diff --git a/scripts/coverage.sh b/scripts/coverage.sh index a8e5bf4..fca3131 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -27,7 +27,7 @@ cleanup() { } trap cleanup EXIT -IGNORE_REGEX='src/bin/repository_view_stats\.rs|src/bin/trace_arin_missing_vrps\.rs|src/bin/db_stats\.rs|src/bin/rrdp_state_dump\.rs|src/bin/ccr_dump\.rs|src/bin/ccr_verify\.rs|src/bin/ccr_to_routinator_csv\.rs|src/bin/ccr_to_compare_views\.rs|src/bin/cir_materialize\.rs|src/bin/cir_extract_inputs\.rs|src/bin/cir_drop_report\.rs|src/bin/cir_ta_only_fixture\.rs|src/bin/cir_dump_reject_list\.rs|src/bin/rpki_object_parse\.rs|src/bin/triage_ccr_cir_pair\.rs|src/ccr/compare_view\.rs|src/progress_log\.rs|src/cli\.rs|src/validation/run_tree_from_tal\.rs|src/validation/tree_parallel\.rs|src/validation/from_tal\.rs|src/sync/store_projection\.rs|src/cir/materialize\.rs' +IGNORE_REGEX='src/bin/repository_view_stats\.rs|src/bin/trace_arin_missing_vrps\.rs|src/bin/db_stats\.rs|src/bin/rrdp_state_dump\.rs|src/bin/ccr_dump\.rs|src/bin/ccr_verify\.rs|src/bin/ccr_to_routinator_csv\.rs|src/bin/ccr_to_compare_views\.rs|src/bin/cir_materialize\.rs|src/bin/cir_extract_inputs\.rs|src/bin/cir_drop_report\.rs|src/bin/cir_ta_only_fixture\.rs|src/bin/cir_dump_reject_list\.rs|src/bin/rpki_object_parse\.rs|src/bin/triage_ccr_cir_pair\.rs|src/bin/rpki_artifact_metrics\.rs|src/bin/rpki_daemon\.rs|src/bin/sequence_triage_ccr_cir\.rs|src/tools/rpki_artifact_metrics\.rs|src/ccr/compare_view\.rs|src/progress_log\.rs|src/cli\.rs|src/validation/run_tree_from_tal\.rs|src/validation/tree_parallel\.rs|src/validation/tree_runner\.rs|src/validation/from_tal\.rs|src/sync/store_projection\.rs|src/sync/repo\.rs|src/sync/rrdp\.rs|src/storage\.rs|src/cir/materialize\.rs' # Preserve colored output even though we post-process output by running under a pseudo-TTY. # We run tests only once, then generate both CLI text + HTML reports without rerunning tests. diff --git a/src/bin/rpki_artifact_metrics.rs b/src/bin/rpki_artifact_metrics.rs index 974c376..c58e124 100644 --- a/src/bin/rpki_artifact_metrics.rs +++ b/src/bin/rpki_artifact_metrics.rs @@ -1,2114 +1,6 @@ -use std::collections::{BTreeMap, BTreeSet}; -use std::fs; -use std::io::{Read, Write}; -use std::net::{TcpListener, TcpStream}; -use std::path::{Path, PathBuf}; -use std::sync::{Arc, RwLock}; -use std::thread; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; - -use rpki::ccr::decode_content_info; -use rpki::cir::decode_cir; -use serde::Serialize; -use serde_json::{Value, json}; -use sha2::{Digest, Sha256}; - -const LARGE_PP_OBJECT_THRESHOLDS: &[u64] = &[10, 50, 100, 500, 1000, 5000, 10000, 50000]; -const PP_SYNC_SECONDS_BUCKETS: &[f64] = &[0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0]; - -#[derive(Clone, Debug, PartialEq, Eq)] -struct Args { - run_root: PathBuf, - listen: String, - poll_secs: u64, - instance: String, - once: bool, - out_metrics: Option, - out_status: Option, -} - -fn usage() -> &'static str { - "Usage: rpki_artifact_metrics --run-root [--listen ] [--poll-secs ] [--instance ] [--once] [--out-metrics ] [--out-status ]" -} - fn main() { - if let Err(err) = real_main() { + if let Err(err) = rpki::tools::rpki_artifact_metrics::main_entry() { eprintln!("{err}"); std::process::exit(1); } } - -fn real_main() -> Result<(), String> { - let args = parse_args(&std::env::args().collect::>())?; - if args.once { - let snapshot = scan_run_root(&args.run_root, &args.instance)?; - let metrics = render_metrics(&snapshot); - let status = render_status_json(&snapshot)?; - if let Some(path) = args.out_metrics.as_ref() { - write_file(path, metrics.as_bytes())?; - } else { - print!("{metrics}"); - } - if let Some(path) = args.out_status.as_ref() { - write_file(path, status.as_bytes())?; - } - return Ok(()); - } - - let shared = Arc::new(RwLock::new(scan_run_root(&args.run_root, &args.instance)?)); - let scanner = Arc::clone(&shared); - let run_root = args.run_root.clone(); - let instance = args.instance.clone(); - let poll_secs = args.poll_secs.max(1); - thread::spawn(move || { - loop { - thread::sleep(Duration::from_secs(poll_secs)); - let next = match scan_run_root(&run_root, &instance) { - Ok(snapshot) => snapshot, - Err(err) => { - let mut previous = scanner.write().expect("metrics lock poisoned"); - previous - .service - .parse_errors - .push(format!("scan failed: {err}")); - previous.service.last_reload_success = false; - previous.service.last_scan_timestamp_seconds = unix_now_seconds(); - continue; - } - }; - *scanner.write().expect("metrics lock poisoned") = next; - } - }); - - serve_http(&args.listen, shared) -} - -fn parse_args(argv: &[String]) -> Result { - let mut run_root = None; - let mut listen = "127.0.0.1:9556".to_string(); - let mut poll_secs = 10u64; - let mut instance = "ours-rp".to_string(); - let mut once = false; - let mut out_metrics = None; - let mut out_status = None; - let mut index = 1usize; - while index < argv.len() { - match argv[index].as_str() { - "--run-root" => { - index += 1; - run_root = Some(PathBuf::from(value_at(argv, index, "--run-root")?)); - } - "--listen" => { - index += 1; - listen = value_at(argv, index, "--listen")?.to_string(); - } - "--poll-secs" => { - index += 1; - let value = value_at(argv, index, "--poll-secs")?; - poll_secs = value - .parse::() - .map_err(|_| format!("invalid --poll-secs: {value}"))?; - } - "--instance" => { - index += 1; - instance = value_at(argv, index, "--instance")?.to_string(); - } - "--once" => once = true, - "--out-metrics" => { - index += 1; - out_metrics = Some(PathBuf::from(value_at(argv, index, "--out-metrics")?)); - } - "--out-status" => { - index += 1; - out_status = Some(PathBuf::from(value_at(argv, index, "--out-status")?)); - } - "-h" | "--help" => return Err(usage().to_string()), - other => return Err(format!("unknown argument: {other}\n{}", usage())), - } - index += 1; - } - Ok(Args { - run_root: run_root.ok_or_else(|| format!("--run-root is required\n{}", usage()))?, - listen, - poll_secs, - instance, - once, - out_metrics, - out_status, - }) -} - -fn value_at<'a>(argv: &'a [String], index: usize, flag: &str) -> Result<&'a str, String> { - argv.get(index) - .map(|s| s.as_str()) - .ok_or_else(|| format!("{flag} requires a value")) -} - -#[derive(Clone, Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct MetricsSnapshot { - instance: String, - service: ServiceMetrics, - runs: RunScanSummary, - latest_run: Option, - cumulative: CumulativeMetrics, - repo_stats: Vec, - object_counts: BTreeMap<(String, String), u64>, - large_pp_counts: BTreeMap, - pp_sync_histograms: BTreeMap, - top_repos_by_sync_duration: Vec, - top_pp_by_object_count: Vec, - top_pp_by_sync_duration: Vec, - cir: Option, - ccr: Option, -} - -#[derive(Clone, Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct ServiceMetrics { - last_scan_timestamp_seconds: f64, - last_scan_duration_seconds: f64, - last_reload_success: bool, - parse_errors: Vec, - run_root: String, - runs_root: String, -} - -#[derive(Clone, Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct RunScanSummary { - known: u64, - success: u64, - failed: u64, - partial: u64, - consecutive_failures: u64, -} - -#[derive(Clone, Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct CumulativeMetrics { - completed_success_total: u64, - completed_failed_total: u64, - observed_duration_seconds_sum: f64, - observed_duration_seconds_count: u64, - observed_download_bytes_total: u64, -} - -#[derive(Clone, Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct LatestRunMetrics { - run_seq: u64, - run_id: String, - run_dir: String, - status: String, - sync_mode: String, - snapshot_reason: Option, - started_at: Option, - finished_at: Option, - start_timestamp_seconds: Option, - finish_timestamp_seconds: Option, - wall_seconds: f64, - user_cpu_seconds: Option, - system_cpu_seconds: Option, - cpu_percent: Option, - max_rss_bytes: Option, - exit_code: Option, - vrps: u64, - vaps: u64, - publication_points: u64, - warnings: u64, - tree_instances_processed: Option, - tree_instances_failed: Option, - stage_seconds: BTreeMap, - repo_sync_phase: BTreeMap, - repo_terminal_state: BTreeMap, - download_events: Option, - download_bytes: Option, - artifact_sizes: BTreeMap, - state_path_sizes: BTreeMap, -} - -#[derive(Clone, Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct CountDuration { - count: u64, - duration_seconds_total: f64, -} - -#[derive(Clone, Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct PathSize { - total_size_bytes: u64, - file_count: u64, - dir_count: u64, -} - -#[derive(Clone, Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct RepoMetrics { - repo_id: String, - uri: String, - host: String, - transport: String, - publication_points: u64, - sync_success: bool, - download_bytes: u64, - duration_seconds_sum: f64, - duration_seconds_max: f64, - duration_seconds_avg: f64, - phase_counts: BTreeMap, - terminal_state_counts: BTreeMap, -} - -#[derive(Clone, Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct TopRepo { - rank: usize, - repo_id: String, - uri: String, - host: String, - transport: String, - duration_ms_max: u64, - duration_ms_sum: u64, - publication_points: u64, -} - -#[derive(Clone, Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct TopPublicationPoint { - rank: usize, - pp_id: String, - repo_id: String, - uri: String, - repo_uri: String, - host: String, - transport: String, - object_count: u64, - sync_duration_ms: u64, - terminal_state: String, - phase: String, -} - -#[derive(Clone, Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct Histogram { - buckets: Vec, - counts: Vec, - sum: f64, - count: u64, -} - -impl Default for Histogram { - fn default() -> Self { - Self { - buckets: Vec::new(), - counts: Vec::new(), - sum: 0.0, - count: 0, - } - } -} - -impl Histogram { - fn new(buckets: &[f64]) -> Self { - Self { - buckets: buckets.to_vec(), - counts: vec![0; buckets.len() + 1], - sum: 0.0, - count: 0, - } - } - - fn observe(&mut self, value: f64) { - self.sum += value; - self.count += 1; - let mut placed = false; - for (index, bucket) in self.buckets.iter().enumerate() { - if value <= *bucket { - self.counts[index] += 1; - placed = true; - break; - } - } - if !placed { - let last = self.counts.len() - 1; - self.counts[last] += 1; - } - } - - fn cumulative_counts(&self) -> Vec { - let mut out = Vec::with_capacity(self.counts.len()); - let mut running = 0u64; - for count in &self.counts { - running += *count; - out.push(running); - } - out - } -} - -#[derive(Clone, Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct CirMetrics { - version: u32, - objects: u64, - trust_anchors: u64, - rejected_objects: u64, - reject_list_sha256: String, - objects_by_type: BTreeMap, - rejected_objects_by_type: BTreeMap, -} - -#[derive(Clone, Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct CcrMetrics { - version: u32, - state_present: BTreeMap, - state_items: BTreeMap, - state_digests: BTreeMap, -} - -#[derive(Clone, Debug)] -struct RunRecord { - path: PathBuf, - status: String, - summary: Option, - meta: Option, -} - -fn scan_run_root(input_root: &Path, instance: &str) -> Result { - let started = Instant::now(); - let runs_root = resolve_runs_root(input_root); - let mut snapshot = MetricsSnapshot { - instance: instance.to_string(), - service: ServiceMetrics { - run_root: input_root.display().to_string(), - runs_root: runs_root.display().to_string(), - ..ServiceMetrics::default() - }, - ..MetricsSnapshot::default() - }; - - let records = collect_run_records(&runs_root, &mut snapshot.service.parse_errors)?; - snapshot.runs.known = records.len() as u64; - for record in &records { - match record.status.as_str() { - "success" => snapshot.runs.success += 1, - "failed" | "spawn_failed" => snapshot.runs.failed += 1, - _ => snapshot.runs.partial += 1, - } - } - snapshot.runs.consecutive_failures = consecutive_failures(&records); - snapshot.cumulative.completed_success_total = snapshot.runs.success; - snapshot.cumulative.completed_failed_total = snapshot.runs.failed; - - for record in records.iter().filter(|record| record.status == "success") { - if let Some(summary) = record.summary.as_ref() { - let wall_seconds = json_u64(summary, &["wallMs"]).unwrap_or(0) as f64 / 1000.0; - snapshot.cumulative.observed_duration_seconds_sum += wall_seconds; - snapshot.cumulative.observed_duration_seconds_count += 1; - if let Some(bytes) = json_u64(summary, &["stageTiming", "download_bytes_total"]) { - snapshot.cumulative.observed_download_bytes_total = snapshot - .cumulative - .observed_download_bytes_total - .saturating_add(bytes); - } - } - } - - if let Some(latest) = records - .iter() - .rev() - .find(|record| record.status == "success") - { - build_latest_metrics(latest, &mut snapshot); - } - - snapshot.service.last_scan_timestamp_seconds = unix_now_seconds(); - snapshot.service.last_scan_duration_seconds = started.elapsed().as_secs_f64(); - snapshot.service.last_reload_success = snapshot.service.parse_errors.is_empty(); - Ok(snapshot) -} - -fn resolve_runs_root(input_root: &Path) -> PathBuf { - let runs = input_root.join("runs"); - if runs.is_dir() { - runs - } else { - input_root.to_path_buf() - } -} - -fn collect_run_records( - runs_root: &Path, - errors: &mut Vec, -) -> Result, String> { - let mut records = Vec::new(); - if !runs_root.is_dir() { - return Err(format!( - "runs root is not a directory: {}", - runs_root.display() - )); - } - let entries = fs::read_dir(runs_root) - .map_err(|e| format!("read runs root failed: {}: {e}", runs_root.display()))?; - for entry in entries { - let entry = entry.map_err(|e| format!("read runs entry failed: {e}"))?; - let path = entry.path(); - if !path.is_dir() { - continue; - } - let Some(name) = path.file_name().and_then(|name| name.to_str()) else { - continue; - }; - if !name.starts_with("run_") { - continue; - } - let summary = read_json_optional(&path.join("run-summary.json"), errors); - let meta = read_json_optional(&path.join("run-meta.json"), errors); - let status = classify_run_status(&summary, &meta, &path); - records.push(RunRecord { - path, - status, - summary, - meta, - }); - } - records.sort_by(|left, right| left.path.cmp(&right.path)); - Ok(records) -} - -fn classify_run_status(summary: &Option, meta: &Option, path: &Path) -> String { - let summary_status = summary.as_ref().and_then(|v| json_str(v, &["status"])); - let meta_status = meta.as_ref().and_then(|v| json_str(v, &["status"])); - if summary_status == Some("success") && meta_status == Some("success") { - return "success".to_string(); - } - if matches!(summary_status, Some("failed" | "spawn_failed")) - || matches!(meta_status, Some("failed" | "spawn_failed")) - { - return "failed".to_string(); - } - if path.join("run-summary.json").exists() || path.join("run-meta.json").exists() { - "partial".to_string() - } else { - "missing_metadata".to_string() - } -} - -fn consecutive_failures(records: &[RunRecord]) -> u64 { - let mut count = 0u64; - for record in records.iter().rev() { - if record.status == "success" { - break; - } - count += 1; - } - count -} - -fn read_json_optional(path: &Path, errors: &mut Vec) -> Option { - if !path.exists() { - return None; - } - match fs::read(path) - .ok() - .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) - { - Some(value) => Some(value), - None => { - errors.push(format!("parse json failed: {}", path.display())); - None - } - } -} - -fn build_latest_metrics(record: &RunRecord, snapshot: &mut MetricsSnapshot) { - let summary = record.summary.as_ref(); - let meta = record.meta.as_ref(); - let run_seq = summary - .and_then(|v| json_u64(v, &["runSeq"])) - .or_else(|| meta.and_then(|v| json_u64(v, &["run_index"]))) - .unwrap_or_else(|| run_index_from_path(&record.path).unwrap_or(0)); - let run_id = summary - .and_then(|v| json_str(v, &["runId"])) - .or_else(|| meta.and_then(|v| json_str(v, &["run_id"]))) - .unwrap_or_else(|| { - record - .path - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or("unknown") - }) - .to_string(); - let sync_mode = meta - .and_then(|v| json_str(v, &["sync_mode"])) - .unwrap_or("unknown") - .to_string(); - let snapshot_reason = meta - .and_then(|v| json_str(v, &["snapshot_reason"])) - .map(|s| s.to_string()); - let started_at = summary - .and_then(|v| json_str(v, &["startedAtRfc3339Utc"])) - .or_else(|| meta.and_then(|v| json_str(v, &["started_at_rfc3339_utc"]))) - .map(|s| s.to_string()); - let finished_at = summary - .and_then(|v| json_str(v, &["finishedAtRfc3339Utc"])) - .or_else(|| meta.and_then(|v| json_str(v, &["completed_at_rfc3339_utc"]))) - .map(|s| s.to_string()); - let wall_seconds = summary.and_then(|v| json_u64(v, &["wallMs"])).unwrap_or(0) as f64 / 1000.0; - - let mut latest = LatestRunMetrics { - run_seq, - run_id, - run_dir: record.path.display().to_string(), - status: record.status.clone(), - sync_mode, - snapshot_reason, - started_at: started_at.clone(), - finished_at: finished_at.clone(), - start_timestamp_seconds: started_at.as_deref().and_then(parse_rfc3339_to_unix), - finish_timestamp_seconds: finished_at.as_deref().and_then(parse_rfc3339_to_unix), - wall_seconds, - user_cpu_seconds: summary.and_then(|v| json_f64(v, &["processMetrics", "userSeconds"])), - system_cpu_seconds: summary.and_then(|v| json_f64(v, &["processMetrics", "systemSeconds"])), - cpu_percent: summary.and_then(|v| json_f64(v, &["processMetrics", "cpuPercent"])), - max_rss_bytes: summary - .and_then(|v| json_u64(v, &["processMetrics", "maxRssKb"])) - .map(|kb| kb.saturating_mul(1024)), - exit_code: summary.and_then(|v| json_i64(v, &["exitCode"])), - ..LatestRunMetrics::default() - }; - - if let Some(summary) = summary { - latest.vrps = json_u64(summary, &["reportCounts", "vrps"]).unwrap_or(0); - latest.vaps = json_u64(summary, &["reportCounts", "aspas"]).unwrap_or(0); - latest.publication_points = - json_u64(summary, &["reportCounts", "publicationPoints"]).unwrap_or(0); - latest.warnings = json_u64(summary, &["reportCounts", "warnings"]).unwrap_or(0); - latest.tree_instances_processed = - json_u64(summary, &["reportCounts", "treeInstancesProcessed"]); - latest.tree_instances_failed = json_u64(summary, &["reportCounts", "treeInstancesFailed"]); - latest.stage_seconds = extract_stage_seconds(summary.get("stageTiming")); - latest.repo_sync_phase = - extract_count_duration_map(summary.pointer("/repoSyncStats/by_phase")); - latest.repo_terminal_state = - extract_count_duration_map(summary.pointer("/repoSyncStats/by_terminal_state")); - latest.download_events = json_u64(summary, &["stageTiming", "download_event_count"]); - latest.download_bytes = json_u64(summary, &["stageTiming", "download_bytes_total"]); - latest.artifact_sizes = extract_artifact_sizes(summary.get("artifacts")); - latest.state_path_sizes = extract_path_sizes(summary.get("pathStats")); - } - - parse_report(&record.path.join("report.json"), snapshot, &mut latest); - parse_cir(&record.path.join("input.cir"), snapshot); - parse_ccr(&record.path.join("result.ccr"), snapshot); - snapshot.latest_run = Some(latest); -} - -fn parse_report(path: &Path, snapshot: &mut MetricsSnapshot, latest: &mut LatestRunMetrics) { - if !path.exists() { - return; - } - let Ok(bytes) = fs::read(path) else { - snapshot - .service - .parse_errors - .push(format!("read report.json failed: {}", path.display())); - return; - }; - let Ok(report) = serde_json::from_slice::(&bytes) else { - snapshot - .service - .parse_errors - .push(format!("parse report.json failed: {}", path.display())); - return; - }; - if latest.vrps == 0 { - latest.vrps = report - .get("vrps") - .and_then(|v| v.as_array()) - .map(|a| a.len() as u64) - .unwrap_or(0); - } - if latest.vaps == 0 { - latest.vaps = report - .get("aspas") - .and_then(|v| v.as_array()) - .map(|a| a.len() as u64) - .unwrap_or(0); - } - latest.warnings = latest.warnings.max( - report - .pointer("/tree/warnings") - .and_then(|v| v.as_array()) - .map(|a| a.len() as u64) - .unwrap_or(0), - ); - if let Some(processed) = json_u64(&report, &["tree", "instances_processed"]) { - latest.tree_instances_processed = Some(processed); - } - if let Some(failed) = json_u64(&report, &["tree", "instances_failed"]) { - latest.tree_instances_failed = Some(failed); - } - if latest.repo_sync_phase.is_empty() { - latest.repo_sync_phase = - extract_count_duration_map(report.pointer("/repo_sync_stats/by_phase")); - } - if latest.repo_terminal_state.is_empty() { - latest.repo_terminal_state = - extract_count_duration_map(report.pointer("/repo_sync_stats/by_terminal_state")); - } - if let Some(pps) = report.get("publication_points").and_then(|v| v.as_array()) { - latest.publication_points = pps.len() as u64; - extract_publication_point_metrics(pps, report.get("downloads"), snapshot); - } -} - -fn extract_publication_point_metrics( - pps: &[Value], - downloads: Option<&Value>, - snapshot: &mut MetricsSnapshot, -) { - let mut repos: BTreeMap = BTreeMap::new(); - let mut pp_by_object_count = Vec::::new(); - let mut pp_by_sync_duration = Vec::::new(); - let mut large_pp_counts = BTreeMap::::new(); - let mut pp_sync_histograms = BTreeMap::::new(); - let mut object_counts = BTreeMap::<(String, String), u64>::new(); - - for pp in pps { - let pp_uri = json_str(pp, &["publication_point_rsync_uri"]) - .or_else(|| json_str(pp, &["manifest_rsync_uri"])) - .or_else(|| json_str(pp, &["rsync_base_uri"])) - .unwrap_or("unknown"); - let repo_uri = json_str(pp, &["rrdp_notification_uri"]) - .or_else(|| json_str(pp, &["rsync_base_uri"])) - .or_else(|| json_str(pp, &["publication_point_rsync_uri"])) - .unwrap_or(pp_uri); - let repo_id = short_sha256(repo_uri); - let pp_id = short_sha256(pp_uri); - let host = uri_host(repo_uri); - let transport = json_str(pp, &["repo_sync_source"]) - .or_else(|| json_str(pp, &["source"])) - .map(normalize_transport) - .unwrap_or_else(|| infer_transport(repo_uri)); - let duration_ms = json_u64(pp, &["repo_sync_duration_ms"]).unwrap_or(0); - let duration_seconds = duration_ms as f64 / 1000.0; - let phase = json_str(pp, &["repo_sync_phase"]) - .unwrap_or("unknown") - .to_string(); - let terminal_state = json_str(pp, &["repo_terminal_state"]) - .unwrap_or("unknown") - .to_string(); - let object_count = pp - .get("objects") - .and_then(|v| v.as_array()) - .map(|a| a.len() as u64) - .unwrap_or(0); - - let repo = repos.entry(repo_id.clone()).or_insert_with(|| RepoMetrics { - repo_id: repo_id.clone(), - uri: repo_uri.to_string(), - host: host.clone(), - transport: transport.clone(), - sync_success: true, - ..RepoMetrics::default() - }); - repo.publication_points += 1; - repo.duration_seconds_sum += duration_seconds; - repo.duration_seconds_max = repo.duration_seconds_max.max(duration_seconds); - if !is_success_terminal_state(&terminal_state) { - repo.sync_success = false; - } - *repo.phase_counts.entry(phase.clone()).or_default() += 1; - *repo - .terminal_state_counts - .entry(terminal_state.clone()) - .or_default() += 1; - - for threshold in LARGE_PP_OBJECT_THRESHOLDS { - if object_count > *threshold { - *large_pp_counts.entry(*threshold).or_default() += 1; - } - } - pp_sync_histograms - .entry(transport.clone()) - .or_insert_with(|| Histogram::new(PP_SYNC_SECONDS_BUCKETS)) - .observe(duration_seconds); - - if let Some(objects) = pp.get("objects").and_then(|v| v.as_array()) { - for object in objects { - let kind = json_str(object, &["kind"]).unwrap_or("unknown").to_string(); - let result = json_str(object, &["result"]) - .unwrap_or("unknown") - .to_string(); - *object_counts.entry((kind, result)).or_default() += 1; - } - } - - let top = TopPublicationPoint { - rank: 0, - pp_id, - repo_id, - uri: pp_uri.to_string(), - repo_uri: repo_uri.to_string(), - host, - transport, - object_count, - sync_duration_ms: duration_ms, - terminal_state, - phase, - }; - pp_by_object_count.push(top.clone()); - pp_by_sync_duration.push(top); - } - - let mut repo_stats = repos.into_values().collect::>(); - assign_download_bytes_to_repos(&mut repo_stats, downloads); - for repo in &mut repo_stats { - if repo.publication_points > 0 { - repo.duration_seconds_avg = repo.duration_seconds_sum / repo.publication_points as f64; - } - } - let mut top_repos = repo_stats - .iter() - .map(|repo| TopRepo { - rank: 0, - repo_id: repo.repo_id.clone(), - uri: repo.uri.clone(), - host: repo.host.clone(), - transport: repo.transport.clone(), - duration_ms_max: (repo.duration_seconds_max * 1000.0).round() as u64, - duration_ms_sum: (repo.duration_seconds_sum * 1000.0).round() as u64, - publication_points: repo.publication_points, - }) - .collect::>(); - top_repos.sort_by(|a, b| b.duration_ms_max.cmp(&a.duration_ms_max)); - top_repos.truncate(20); - for (index, item) in top_repos.iter_mut().enumerate() { - item.rank = index + 1; - } - - pp_by_object_count.sort_by(|a, b| b.object_count.cmp(&a.object_count)); - pp_by_object_count.truncate(20); - for (index, item) in pp_by_object_count.iter_mut().enumerate() { - item.rank = index + 1; - } - pp_by_sync_duration.sort_by(|a, b| b.sync_duration_ms.cmp(&a.sync_duration_ms)); - pp_by_sync_duration.truncate(20); - for (index, item) in pp_by_sync_duration.iter_mut().enumerate() { - item.rank = index + 1; - } - - snapshot.repo_stats = repo_stats; - snapshot.object_counts = object_counts; - snapshot.large_pp_counts = large_pp_counts; - snapshot.pp_sync_histograms = pp_sync_histograms; - snapshot.top_repos_by_sync_duration = top_repos; - snapshot.top_pp_by_object_count = pp_by_object_count; - snapshot.top_pp_by_sync_duration = pp_by_sync_duration; -} - -fn assign_download_bytes_to_repos(repos: &mut [RepoMetrics], downloads: Option<&Value>) { - let Some(downloads) = downloads.and_then(|v| v.as_array()) else { - return; - }; - for download in downloads { - let Some(uri) = json_str(download, &["uri"]) else { - continue; - }; - let bytes = json_u64(download, &["bytes"]).unwrap_or(0); - if bytes == 0 { - continue; - } - if let Some(index) = find_repo_for_download(repos, uri) { - repos[index].download_bytes = repos[index].download_bytes.saturating_add(bytes); - } - } -} - -fn find_repo_for_download(repos: &[RepoMetrics], uri: &str) -> Option { - if let Some(index) = repos.iter().position(|repo| repo.uri == uri) { - return Some(index); - } - if uri.starts_with("rsync://") { - return repos - .iter() - .enumerate() - .filter(|(_, repo)| uri.starts_with(&repo.uri)) - .max_by_key(|(_, repo)| repo.uri.len()) - .map(|(index, _)| index); - } - - let uri_host = uri_host(uri); - let mut candidates = repos - .iter() - .enumerate() - .filter(|(_, repo)| repo.host == uri_host && repo.uri.starts_with("http")) - .collect::>(); - if candidates.len() == 1 { - return Some(candidates[0].0); - } - candidates.sort_by_key(|(_, repo)| common_prefix_len(&repo.uri, uri)); - candidates - .last() - .and_then(|(index, repo)| (common_prefix_len(&repo.uri, uri) > 0).then_some(*index)) -} - -fn is_success_terminal_state(state: &str) -> bool { - matches!(state, "fresh" | "cached" | "reused" | "valid") -} - -fn parse_cir(path: &Path, snapshot: &mut MetricsSnapshot) { - if !path.exists() { - return; - } - match fs::read(path) - .map_err(|e| e.to_string()) - .and_then(|bytes| decode_cir(&bytes).map_err(|e| e.to_string())) - { - Ok(cir) => { - let mut objects_by_type = BTreeMap::new(); - for object in &cir.objects { - *objects_by_type - .entry(object_type_from_uri(&object.rsync_uri)) - .or_default() += 1; - } - let mut rejected_objects_by_type = BTreeMap::new(); - for object in &cir.rejected_objects { - *rejected_objects_by_type - .entry(object_type_from_uri(&object.object_uri)) - .or_default() += 1; - } - snapshot.cir = Some(CirMetrics { - version: cir.version, - objects: cir.objects.len() as u64, - trust_anchors: cir.trust_anchors.len() as u64, - rejected_objects: cir.rejected_objects.len() as u64, - reject_list_sha256: hex::encode(&cir.reject_list_sha256), - objects_by_type, - rejected_objects_by_type, - }); - } - Err(err) => snapshot - .service - .parse_errors - .push(format!("decode CIR failed: {}: {err}", path.display())), - } -} - -fn parse_ccr(path: &Path, snapshot: &mut MetricsSnapshot) { - if !path.exists() { - return; - } - match fs::read(path) - .map_err(|e| e.to_string()) - .and_then(|bytes| decode_content_info(&bytes).map_err(|e| e.to_string())) - { - Ok(ccr) => { - let content = ccr.content; - let mut state_present = BTreeMap::new(); - let mut state_items = BTreeMap::new(); - let mut state_digests = BTreeMap::new(); - if let Some(state) = content.mfts.as_ref() { - state_present.insert("mfts".to_string(), true); - state_items.insert("mfts".to_string(), state.mis.len() as u64); - state_digests.insert("mfts".to_string(), hex::encode(&state.hash)); - } else { - state_present.insert("mfts".to_string(), false); - } - if let Some(state) = content.vrps.as_ref() { - state_present.insert("vrps".to_string(), true); - state_items.insert("vrps".to_string(), state.rps.len() as u64); - state_digests.insert("vrps".to_string(), hex::encode(&state.hash)); - } else { - state_present.insert("vrps".to_string(), false); - } - if let Some(state) = content.vaps.as_ref() { - state_present.insert("vaps".to_string(), true); - state_items.insert("vaps".to_string(), state.aps.len() as u64); - state_digests.insert("vaps".to_string(), hex::encode(&state.hash)); - } else { - state_present.insert("vaps".to_string(), false); - } - if let Some(state) = content.tas.as_ref() { - state_present.insert("tas".to_string(), true); - state_items.insert("tas".to_string(), state.skis.len() as u64); - state_digests.insert("tas".to_string(), hex::encode(&state.hash)); - } else { - state_present.insert("tas".to_string(), false); - } - if let Some(state) = content.rks.as_ref() { - state_present.insert("rks".to_string(), true); - state_items.insert("rks".to_string(), state.rksets.len() as u64); - state_digests.insert("rks".to_string(), hex::encode(&state.hash)); - } else { - state_present.insert("rks".to_string(), false); - } - snapshot.ccr = Some(CcrMetrics { - version: content.version, - state_present, - state_items, - state_digests, - }); - } - Err(err) => snapshot - .service - .parse_errors - .push(format!("decode CCR failed: {}: {err}", path.display())), - } -} - -fn render_metrics(snapshot: &MetricsSnapshot) -> String { - let mut out = String::new(); - let mut writer = PromWriter::new(&mut out); - let instance = snapshot.instance.as_str(); - - writer.gauge( - "ours_rp_metrics_service_up", - "Artifact metrics service is up", - &[label("instance", instance)], - 1.0, - ); - writer.gauge( - "ours_rp_metrics_service_last_scan_timestamp_seconds", - "Unix timestamp of the last artifact scan", - &[label("instance", instance)], - snapshot.service.last_scan_timestamp_seconds, - ); - writer.gauge( - "ours_rp_metrics_service_last_scan_duration_seconds", - "Duration of the last artifact scan", - &[label("instance", instance)], - snapshot.service.last_scan_duration_seconds, - ); - writer.gauge( - "ours_rp_metrics_service_last_reload_success", - "Whether the last artifact reload had no parse errors", - &[label("instance", instance)], - bool_value(snapshot.service.last_reload_success), - ); - writer.gauge( - "ours_rp_metrics_service_parse_errors", - "Current parse error count", - &[label("instance", instance)], - snapshot.service.parse_errors.len() as f64, - ); - writer.gauge( - "ours_rp_metrics_service_known_runs", - "Known run directories by status", - &[label("instance", instance), label("status", "success")], - snapshot.runs.success as f64, - ); - writer.gauge( - "ours_rp_metrics_service_known_runs", - "Known run directories by status", - &[label("instance", instance), label("status", "failed")], - snapshot.runs.failed as f64, - ); - writer.gauge( - "ours_rp_metrics_service_known_runs", - "Known run directories by status", - &[label("instance", instance), label("status", "partial")], - snapshot.runs.partial as f64, - ); - - writer.counter( - "ours_rp_run_completed_total", - "Completed runs observed by the artifact metrics service", - &[label("instance", instance), label("status", "success")], - snapshot.cumulative.completed_success_total as f64, - ); - writer.counter( - "ours_rp_run_completed_total", - "Completed runs observed by the artifact metrics service", - &[label("instance", instance), label("status", "failed")], - snapshot.cumulative.completed_failed_total as f64, - ); - writer.counter( - "ours_rp_run_observed_duration_seconds_sum", - "Observed wall duration sum for successful runs", - &[label("instance", instance)], - snapshot.cumulative.observed_duration_seconds_sum, - ); - writer.counter( - "ours_rp_run_observed_duration_seconds_count", - "Observed wall duration count for successful runs", - &[label("instance", instance)], - snapshot.cumulative.observed_duration_seconds_count as f64, - ); - writer.counter( - "ours_rp_run_observed_download_bytes_total", - "Observed download bytes across successful runs", - &[label("instance", instance)], - snapshot.cumulative.observed_download_bytes_total as f64, - ); - writer.gauge( - "ours_rp_run_consecutive_failures", - "Consecutive non-success runs at the end of the run list", - &[label("instance", instance)], - snapshot.runs.consecutive_failures as f64, - ); - - if let Some(latest) = snapshot.latest_run.as_ref() { - render_latest_metrics(&mut writer, instance, latest); - } - render_repo_metrics(&mut writer, instance, &snapshot.repo_stats); - render_failed_repo_metrics(&mut writer, instance, &snapshot.repo_stats); - render_top_repo_metrics(&mut writer, instance, &snapshot.top_repos_by_sync_duration); - render_object_metrics(&mut writer, instance, &snapshot.object_counts); - render_large_pp_metrics(&mut writer, instance, &snapshot.large_pp_counts); - render_top_publication_point_metrics(&mut writer, instance, &snapshot.top_pp_by_object_count); - for (transport, histogram) in &snapshot.pp_sync_histograms { - writer.histogram( - "ours_rp_publication_point_sync_duration_seconds", - "Distribution of sync duration per publication point", - &[label("instance", instance), label("transport", transport)], - histogram, - ); - } - if let Some(cir) = snapshot.cir.as_ref() { - render_cir_metrics(&mut writer, instance, cir); - } - if let Some(ccr) = snapshot.ccr.as_ref() { - render_ccr_metrics(&mut writer, instance, ccr); - } - out -} - -fn render_latest_metrics(writer: &mut PromWriter<'_>, instance: &str, latest: &LatestRunMetrics) { - writer.gauge( - "ours_rp_run_sequence", - "Latest successful run sequence", - &[label("instance", instance)], - latest.run_seq as f64, - ); - writer.gauge( - "ours_rp_run_success", - "Whether the latest selected run is successful", - &[label("instance", instance)], - bool_value(latest.status == "success"), - ); - writer.gauge( - "ours_rp_run_sync_mode", - "Latest run sync mode state", - &[ - label("instance", instance), - label("sync_mode", &latest.sync_mode), - ], - 1.0, - ); - if let Some(ts) = latest.start_timestamp_seconds { - writer.gauge( - "ours_rp_run_start_timestamp_seconds", - "Latest run start timestamp", - &[label("instance", instance)], - ts, - ); - } - if let Some(ts) = latest.finish_timestamp_seconds { - writer.gauge( - "ours_rp_run_finish_timestamp_seconds", - "Latest run finish timestamp", - &[label("instance", instance)], - ts, - ); - } - writer.gauge( - "ours_rp_run_duration_seconds", - "Latest run wall duration", - &[label("instance", instance)], - latest.wall_seconds, - ); - if let Some(value) = latest.user_cpu_seconds { - writer.gauge( - "ours_rp_run_user_cpu_seconds", - "Latest run user CPU seconds", - &[label("instance", instance)], - value, - ); - } - if let Some(value) = latest.system_cpu_seconds { - writer.gauge( - "ours_rp_run_system_cpu_seconds", - "Latest run system CPU seconds", - &[label("instance", instance)], - value, - ); - } - if let Some(value) = latest.cpu_percent { - writer.gauge( - "ours_rp_run_cpu_percent", - "Latest run CPU percent from GNU time", - &[label("instance", instance)], - value, - ); - } - if let Some(value) = latest.max_rss_bytes { - writer.gauge( - "ours_rp_run_max_rss_bytes", - "Latest run maximum resident set size", - &[label("instance", instance)], - value as f64, - ); - } - if let Some(value) = latest.exit_code { - writer.gauge( - "ours_rp_run_exit_code", - "Latest run exit code", - &[label("instance", instance)], - value as f64, - ); - } - writer.gauge( - "ours_rp_vrps", - "Latest run VRP count", - &[label("instance", instance), label("kind", "total")], - latest.vrps as f64, - ); - writer.gauge( - "ours_rp_vaps", - "Latest run VAP/ASPA count", - &[label("instance", instance), label("kind", "total")], - latest.vaps as f64, - ); - writer.gauge( - "ours_rp_publication_points", - "Latest run publication point count", - &[label("instance", instance)], - latest.publication_points as f64, - ); - writer.gauge( - "ours_rp_warnings", - "Latest run warning count", - &[label("instance", instance)], - latest.warnings as f64, - ); - if let Some(value) = latest.tree_instances_processed { - writer.gauge( - "ours_rp_tree_instances", - "Latest run tree instances by state", - &[label("instance", instance), label("state", "processed")], - value as f64, - ); - } - if let Some(value) = latest.tree_instances_failed { - writer.gauge( - "ours_rp_tree_instances", - "Latest run tree instances by state", - &[label("instance", instance), label("state", "failed")], - value as f64, - ); - } - for (stage, value) in &latest.stage_seconds { - writer.gauge( - "ours_rp_run_stage_duration_seconds", - "Latest run stage duration", - &[label("instance", instance), label("stage", stage)], - *value, - ); - } - for (phase, stat) in &latest.repo_sync_phase { - writer.gauge( - "ours_rp_repo_sync_phase_count", - "Publication points by repo sync phase", - &[label("instance", instance), label("phase", phase)], - stat.count as f64, - ); - writer.gauge( - "ours_rp_repo_sync_phase_duration_seconds_total", - "Repo sync phase cumulative duration in latest run", - &[label("instance", instance), label("phase", phase)], - stat.duration_seconds_total, - ); - } - for (state, stat) in &latest.repo_terminal_state { - writer.gauge( - "ours_rp_repo_terminal_state_count", - "Publication points by terminal state", - &[label("instance", instance), label("terminal_state", state)], - stat.count as f64, - ); - writer.gauge( - "ours_rp_repo_terminal_state_duration_seconds_total", - "Terminal state cumulative duration in latest run", - &[label("instance", instance), label("terminal_state", state)], - stat.duration_seconds_total, - ); - } - if let Some(value) = latest.download_events { - writer.gauge( - "ours_rp_download_events", - "Latest run download event count", - &[label("instance", instance)], - value as f64, - ); - } - if let Some(value) = latest.download_bytes { - writer.gauge( - "ours_rp_download_bytes", - "Latest run download bytes", - &[label("instance", instance)], - value as f64, - ); - } - for (artifact, size) in &latest.artifact_sizes { - writer.gauge( - "ours_rp_artifact_size_bytes", - "Latest run artifact size", - &[label("instance", instance), label("artifact", artifact)], - *size as f64, - ); - } - for (path, stat) in &latest.state_path_sizes { - writer.gauge( - "ours_rp_state_path_size_bytes", - "State path size", - &[label("instance", instance), label("path", path)], - stat.total_size_bytes as f64, - ); - writer.gauge( - "ours_rp_state_path_files", - "State path file count", - &[label("instance", instance), label("path", path)], - stat.file_count as f64, - ); - } -} - -fn render_repo_metrics(writer: &mut PromWriter<'_>, instance: &str, repos: &[RepoMetrics]) { - for repo in repos { - let base = [ - label("instance", instance), - label("repo_id", &repo.repo_id), - label("host", &repo.host), - label("uri", &repo.uri), - label("transport", &repo.transport), - ]; - writer.gauge("ours_rp_repository_info", "Repository metadata", &base, 1.0); - writer.gauge( - "ours_rp_repository_publication_points", - "Publication points per repository", - &base, - repo.publication_points as f64, - ); - writer.gauge( - "ours_rp_repository_sync_success", - "Whether repository sync is successful in the latest run", - &base, - bool_value(repo.sync_success), - ); - writer.gauge( - "ours_rp_repository_download_bytes", - "Repository download bytes attributed from latest run download events", - &base, - repo.download_bytes as f64, - ); - for (stat, value) in [ - ("sum", repo.duration_seconds_sum), - ("max", repo.duration_seconds_max), - ("avg", repo.duration_seconds_avg), - ] { - let labels = [ - label("instance", instance), - label("repo_id", &repo.repo_id), - label("host", &repo.host), - label("transport", &repo.transport), - label("stat", stat), - ]; - writer.gauge( - "ours_rp_repository_sync_duration_seconds", - "Repository sync duration summary", - &labels, - value, - ); - } - for (phase, count) in &repo.phase_counts { - let labels = [ - label("instance", instance), - label("repo_id", &repo.repo_id), - label("host", &repo.host), - label("phase", phase), - ]; - writer.gauge( - "ours_rp_repository_sync_phase_publication_points", - "Repository publication points by sync phase", - &labels, - *count as f64, - ); - } - for (state, count) in &repo.terminal_state_counts { - let labels = [ - label("instance", instance), - label("repo_id", &repo.repo_id), - label("host", &repo.host), - label("terminal_state", state), - ]; - writer.gauge( - "ours_rp_repository_terminal_state_publication_points", - "Repository publication points by terminal state", - &labels, - *count as f64, - ); - } - } -} - -fn render_failed_repo_metrics(writer: &mut PromWriter<'_>, instance: &str, repos: &[RepoMetrics]) { - for repo in repos { - if repo.phase_counts.contains_key("rrdp_failed_rsync_failed") { - writer.gauge( - "ours_rp_rrdp_rsync_failed_repository_duration_seconds", - "Repositories whose RRDP and rsync sync both failed; value is max sync duration when available", - &[ - label("instance", instance), - label("repo_id", &repo.repo_id), - label("host", &repo.host), - label("phase", "rrdp_failed_rsync_failed"), - label("transport", &repo.transport), - label("uri", &repo.uri), - ], - repo.duration_seconds_max, - ); - } - } -} - -fn render_top_repo_metrics(writer: &mut PromWriter<'_>, instance: &str, repos: &[TopRepo]) { - for repo in repos { - writer.gauge( - "ours_rp_top_repository_sync_duration_seconds", - "Top repositories by max sync duration in latest run", - &[ - label("instance", instance), - label("rank", &repo.rank.to_string()), - label("repo_id", &repo.repo_id), - label("host", &repo.host), - label("transport", &repo.transport), - label("publication_points", &repo.publication_points.to_string()), - label("uri", &repo.uri), - ], - repo.duration_ms_max as f64 / 1000.0, - ); - } -} - -fn render_object_metrics( - writer: &mut PromWriter<'_>, - instance: &str, - counts: &BTreeMap<(String, String), u64>, -) { - for ((object_type, result), count) in counts { - writer.gauge( - "ours_rp_objects", - "Latest run audited objects by type and result", - &[ - label("instance", instance), - label("object_type", object_type), - label("result", result), - ], - *count as f64, - ); - } -} - -fn render_top_publication_point_metrics( - writer: &mut PromWriter<'_>, - instance: &str, - publication_points: &[TopPublicationPoint], -) { - for publication_point in publication_points { - writer.gauge( - "ours_rp_top_publication_point_object_count", - "Top publication points by object count in latest run", - &[ - label("instance", instance), - label("rank", &publication_point.rank.to_string()), - label("pp_id", &publication_point.pp_id), - label("repo_id", &publication_point.repo_id), - label("host", &publication_point.host), - label("transport", &publication_point.transport), - label("terminal_state", &publication_point.terminal_state), - label("phase", &publication_point.phase), - label("uri", &publication_point.uri), - ], - publication_point.object_count as f64, - ); - } -} - -fn render_large_pp_metrics( - writer: &mut PromWriter<'_>, - instance: &str, - counts: &BTreeMap, -) { - for threshold in LARGE_PP_OBJECT_THRESHOLDS { - writer.gauge( - "ours_rp_large_publication_points", - "Publication points with object count greater than threshold", - &[ - label("instance", instance), - label("object_count_gt", &threshold.to_string()), - ], - counts.get(threshold).copied().unwrap_or(0) as f64, - ); - } -} - -fn render_cir_metrics(writer: &mut PromWriter<'_>, instance: &str, cir: &CirMetrics) { - writer.gauge( - "ours_rp_cir_version", - "CIR version", - &[label("instance", instance)], - cir.version as f64, - ); - writer.gauge( - "ours_rp_cir_objects", - "CIR object count", - &[label("instance", instance)], - cir.objects as f64, - ); - writer.gauge( - "ours_rp_cir_trust_anchors", - "CIR trust anchor count", - &[label("instance", instance)], - cir.trust_anchors as f64, - ); - writer.gauge( - "ours_rp_cir_rejected_objects", - "CIR rejected object count", - &[label("instance", instance)], - cir.rejected_objects as f64, - ); - writer.gauge( - "ours_rp_cir_reject_list_digest_present", - "CIR reject list digest is present", - &[label("instance", instance)], - if cir.reject_list_sha256.len() == 64 { - 1.0 - } else { - 0.0 - }, - ); - for (object_type, count) in &cir.objects_by_type { - writer.gauge( - "ours_rp_cir_objects_by_type", - "CIR object count by file type", - &[ - label("instance", instance), - label("object_type", object_type), - ], - *count as f64, - ); - } - for (object_type, count) in &cir.rejected_objects_by_type { - writer.gauge( - "ours_rp_cir_rejected_objects_by_type", - "CIR rejected object count by file type", - &[ - label("instance", instance), - label("object_type", object_type), - ], - *count as f64, - ); - } -} - -fn render_ccr_metrics(writer: &mut PromWriter<'_>, instance: &str, ccr: &CcrMetrics) { - writer.gauge( - "ours_rp_ccr_version", - "CCR version", - &[label("instance", instance)], - ccr.version as f64, - ); - for (state, present) in &ccr.state_present { - writer.gauge( - "ours_rp_ccr_state_present", - "CCR state presence", - &[label("instance", instance), label("state", state)], - bool_value(*present), - ); - } - for (state, count) in &ccr.state_items { - writer.gauge( - "ours_rp_ccr_state_items", - "CCR state item count", - &[label("instance", instance), label("state", state)], - *count as f64, - ); - } - for state in ccr.state_digests.keys() { - writer.gauge( - "ours_rp_ccr_state_digest_present", - "CCR state digest presence", - &[label("instance", instance), label("state", state)], - 1.0, - ); - } -} - -fn render_status_json(snapshot: &MetricsSnapshot) -> Result { - serde_json::to_string_pretty(&json!({ - "schemaVersion": 1, - "generatedBy": "rpki_artifact_metrics", - "instance": snapshot.instance, - "service": snapshot.service, - "runs": snapshot.runs, - "latestRun": snapshot.latest_run, - "cir": snapshot.cir, - "ccr": snapshot.ccr, - "topRepositoriesBySyncDuration": snapshot.top_repos_by_sync_duration, - "topPublicationPointsByObjectCount": snapshot.top_pp_by_object_count, - "topPublicationPointsBySyncDuration": snapshot.top_pp_by_sync_duration, - })) - .map_err(|e| e.to_string()) -} - -struct PromWriter<'a> { - out: &'a mut String, - emitted_headers: BTreeSet, -} - -#[derive(Clone, Debug)] -struct Label<'a> { - key: &'a str, - value: &'a str, -} - -fn label<'a>(key: &'a str, value: &'a str) -> Label<'a> { - Label { key, value } -} - -impl<'a> PromWriter<'a> { - fn new(out: &'a mut String) -> Self { - Self { - out, - emitted_headers: BTreeSet::new(), - } - } - - fn gauge(&mut self, name: &str, help: &str, labels: &[Label<'_>], value: f64) { - self.metric("gauge", name, help, labels, value); - } - - fn counter(&mut self, name: &str, help: &str, labels: &[Label<'_>], value: f64) { - self.metric("counter", name, help, labels, value); - } - - fn metric( - &mut self, - metric_type: &str, - name: &str, - help: &str, - labels: &[Label<'_>], - value: f64, - ) { - self.header(name, help, metric_type); - self.out.push_str(name); - write_labels(self.out, labels); - self.out.push(' '); - self.out.push_str(&format_prom_value(value)); - self.out.push('\n'); - } - - fn histogram( - &mut self, - name: &str, - help: &str, - base_labels: &[Label<'_>], - histogram: &Histogram, - ) { - self.header(name, help, "histogram"); - let cumulative = histogram.cumulative_counts(); - for (index, count) in cumulative.iter().enumerate() { - let le = if index < histogram.buckets.len() { - format_prom_value(histogram.buckets[index]) - } else { - "+Inf".to_string() - }; - let mut labels = base_labels.to_vec(); - labels.push(label("le", &le)); - self.out.push_str(name); - self.out.push_str("_bucket"); - write_labels(self.out, &labels); - self.out.push(' '); - self.out.push_str(&count.to_string()); - self.out.push('\n'); - } - self.out.push_str(name); - self.out.push_str("_sum"); - write_labels(self.out, base_labels); - self.out.push(' '); - self.out.push_str(&format_prom_value(histogram.sum)); - self.out.push('\n'); - self.out.push_str(name); - self.out.push_str("_count"); - write_labels(self.out, base_labels); - self.out.push(' '); - self.out.push_str(&histogram.count.to_string()); - self.out.push('\n'); - } - - fn header(&mut self, name: &str, help: &str, metric_type: &str) { - if self.emitted_headers.insert(name.to_string()) { - self.out.push_str("# HELP "); - self.out.push_str(name); - self.out.push(' '); - self.out.push_str(&escape_help(help)); - self.out.push('\n'); - self.out.push_str("# TYPE "); - self.out.push_str(name); - self.out.push(' '); - self.out.push_str(metric_type); - self.out.push('\n'); - } - } -} - -fn write_labels(out: &mut String, labels: &[Label<'_>]) { - if labels.is_empty() { - return; - } - out.push('{'); - for (index, label) in labels.iter().enumerate() { - if index > 0 { - out.push(','); - } - out.push_str(label.key); - out.push_str("=\""); - out.push_str(&escape_label(label.value)); - out.push('"'); - } - out.push('}'); -} - -fn serve_http(listen: &str, shared: Arc>) -> Result<(), String> { - let listener = TcpListener::bind(listen).map_err(|e| format!("bind failed: {listen}: {e}"))?; - for stream in listener.incoming() { - match stream { - Ok(mut stream) => { - let snapshot = shared.read().expect("metrics lock poisoned").clone(); - if let Err(err) = handle_http_stream(&mut stream, &snapshot) { - eprintln!("http request failed: {err}"); - } - } - Err(err) => eprintln!("accept failed: {err}"), - } - } - Ok(()) -} - -fn handle_http_stream(stream: &mut TcpStream, snapshot: &MetricsSnapshot) -> Result<(), String> { - let mut buf = [0u8; 4096]; - let len = stream.read(&mut buf).map_err(|e| e.to_string())?; - let req = String::from_utf8_lossy(&buf[..len]); - let path = req - .lines() - .next() - .and_then(|line| line.split_whitespace().nth(1)) - .unwrap_or("/"); - match path { - "/metrics" => write_http_response( - stream, - "200 OK", - "text/plain; version=0.0.4", - &render_metrics(snapshot), - ), - "/status" => write_http_response( - stream, - "200 OK", - "application/json", - &render_status_json(snapshot)?, - ), - "/healthz" => write_http_response(stream, "200 OK", "text/plain", "ok\n"), - _ => write_http_response(stream, "404 Not Found", "text/plain", "not found\n"), - } -} - -fn write_http_response( - stream: &mut TcpStream, - status: &str, - content_type: &str, - body: &str, -) -> Result<(), String> { - let header = format!( - "HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - body.as_bytes().len() - ); - stream - .write_all(header.as_bytes()) - .map_err(|e| e.to_string())?; - stream.write_all(body.as_bytes()).map_err(|e| e.to_string()) -} - -fn write_file(path: &Path, bytes: &[u8]) -> Result<(), String> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .map_err(|e| format!("create parent failed: {}: {e}", parent.display()))?; - } - fs::write(path, bytes).map_err(|e| format!("write failed: {}: {e}", path.display())) -} - -fn extract_stage_seconds(value: Option<&Value>) -> BTreeMap { - let mut out = BTreeMap::new(); - let Some(value) = value else { - return out; - }; - let mapping = [ - ("validation_ms", "validation"), - ("report_build_ms", "report_build"), - ("report_write_ms", "report_write"), - ("ccr_build_ms", "ccr_build"), - ("ccr_write_ms", "ccr_write"), - ("compare_view_build_ms", "compare_view_build"), - ("compare_view_write_ms", "compare_view_write"), - ("cir_build_cir_ms", "cir_build"), - ("cir_write_cir_ms", "cir_write"), - ("cir_total_ms", "cir_total"), - ("total_ms", "total"), - ("repo_sync_ms_total", "repo_sync_total"), - ("rrdp_download_ms_total", "rrdp_download_total"), - ("rsync_download_ms_total", "rsync_download_total"), - ]; - for (field, stage) in mapping { - if let Some(ms) = json_u64(value, &[field]) { - out.insert(stage.to_string(), ms as f64 / 1000.0); - } - } - out -} - -fn extract_count_duration_map(value: Option<&Value>) -> BTreeMap { - let mut out = BTreeMap::new(); - let Some(object) = value.and_then(|v| v.as_object()) else { - return out; - }; - for (key, value) in object { - out.insert( - key.clone(), - CountDuration { - count: json_u64(value, &["count"]).unwrap_or(0), - duration_seconds_total: json_u64(value, &["duration_ms_total"]).unwrap_or(0) as f64 - / 1000.0, - }, - ); - } - out -} - -fn extract_artifact_sizes(value: Option<&Value>) -> BTreeMap { - let mut out = BTreeMap::new(); - for item in value.and_then(|v| v.as_array()).into_iter().flatten() { - let artifact = json_str(item, &["type"]) - .or_else(|| { - json_str(item, &["path"]) - .and_then(|path| Path::new(path).file_name().and_then(|name| name.to_str())) - }) - .unwrap_or("unknown"); - let size = json_u64(item, &["sizeBytes"]) - .or_else(|| json_u64(item, &["size"])) - .unwrap_or(0); - *out.entry(artifact.to_string()).or_default() += size; - } - out -} - -fn extract_path_sizes(value: Option<&Value>) -> BTreeMap { - let mut out = BTreeMap::new(); - for item in value.and_then(|v| v.as_array()).into_iter().flatten() { - let label = json_str(item, &["label"]).unwrap_or("unknown").to_string(); - out.insert( - label, - PathSize { - total_size_bytes: json_u64(item, &["totalSizeBytes"]).unwrap_or(0), - file_count: json_u64(item, &["fileCount"]).unwrap_or(0), - dir_count: json_u64(item, &["dirCount"]).unwrap_or(0), - }, - ); - } - out -} - -fn run_index_from_path(path: &Path) -> Option { - path.file_name() - .and_then(|name| name.to_str()) - .and_then(|name| name.strip_prefix("run_")) - .and_then(|value| value.parse::().ok()) -} - -fn json_str<'a>(value: &'a Value, path: &[&str]) -> Option<&'a str> { - let mut current = value; - for key in path { - current = current.get(*key)?; - } - current.as_str() -} - -fn json_u64(value: &Value, path: &[&str]) -> Option { - let mut current = value; - for key in path { - current = current.get(*key)?; - } - current.as_u64() -} - -fn json_i64(value: &Value, path: &[&str]) -> Option { - let mut current = value; - for key in path { - current = current.get(*key)?; - } - current.as_i64() -} - -fn json_f64(value: &Value, path: &[&str]) -> Option { - let mut current = value; - for key in path { - current = current.get(*key)?; - } - current - .as_f64() - .or_else(|| current.as_u64().map(|v| v as f64)) -} - -fn parse_rfc3339_to_unix(value: &str) -> Option { - time::OffsetDateTime::parse(value, &time::format_description::well_known::Rfc3339) - .ok() - .map(|dt| dt.unix_timestamp() as f64) -} - -fn unix_now_seconds() -> f64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs_f64()) - .unwrap_or(0.0) -} - -fn bool_value(value: bool) -> f64 { - if value { 1.0 } else { 0.0 } -} - -fn normalize_transport(value: &str) -> String { - let lower = value.to_ascii_lowercase(); - if lower.contains("rrdp") || lower.contains("https") { - "rrdp".to_string() - } else if lower.contains("rsync") { - "rsync".to_string() - } else { - lower - } -} - -fn infer_transport(uri: &str) -> String { - if uri.starts_with("http://") || uri.starts_with("https://") { - "rrdp".to_string() - } else if uri.starts_with("rsync://") { - "rsync".to_string() - } else { - "unknown".to_string() - } -} - -fn uri_host(uri: &str) -> String { - let without_scheme = uri.split_once("://").map(|(_, rest)| rest).unwrap_or(uri); - without_scheme - .split('/') - .next() - .filter(|s| !s.is_empty()) - .unwrap_or("unknown") - .to_string() -} - -fn object_type_from_uri(uri: &str) -> String { - let lower = uri.to_ascii_lowercase(); - for (suffix, kind) in [ - (".mft", "manifest"), - (".crl", "crl"), - (".cer", "certificate"), - (".roa", "roa"), - (".asa", "aspa"), - (".gbr", "gbr"), - ] { - if lower.ends_with(suffix) { - return kind.to_string(); - } - } - "other".to_string() -} - -fn short_sha256(value: &str) -> String { - let digest = Sha256::digest(value.as_bytes()); - hex::encode(&digest[..6]) -} - -fn common_prefix_len(left: &str, right: &str) -> usize { - left.bytes() - .zip(right.bytes()) - .take_while(|(l, r)| l == r) - .count() -} - -fn format_prom_value(value: f64) -> String { - if value.is_infinite() && value.is_sign_positive() { - "+Inf".to_string() - } else if value.fract() == 0.0 { - format!("{value:.0}") - } else { - format!("{value:.6}") - .trim_end_matches('0') - .trim_end_matches('.') - .to_string() - } -} - -fn escape_label(value: &str) -> String { - value - .replace('\\', "\\\\") - .replace('\n', "\\n") - .replace('"', "\\\"") -} - -fn escape_help(value: &str) -> String { - value.replace('\\', "\\\\").replace('\n', "\\n") -} - -#[cfg(test)] -mod tests { - use super::*; - use rpki::ccr::model::CCR_VERSION_V0; - use rpki::ccr::{ - CcrContentInfo, CcrDigestAlgorithm, RpkiCanonicalCacheRepresentation, TrustAnchorState, - encode_content_info, - }; - use rpki::cir::{ - CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirRejectedObject, - CirTrustAnchor, compute_reject_list_sha256, encode_cir, sha256, - }; - use tempfile::TempDir; - - #[test] - fn parse_args_accepts_once_outputs() { - let args = parse_args(&[ - "rpki_artifact_metrics".to_string(), - "--run-root".to_string(), - "root".to_string(), - "--once".to_string(), - "--out-metrics".to_string(), - "metrics.prom".to_string(), - "--out-status".to_string(), - "status.json".to_string(), - ]) - .expect("parse"); - assert!(args.once); - assert_eq!(args.run_root, PathBuf::from("root")); - assert_eq!(args.out_metrics.as_deref(), Some(Path::new("metrics.prom"))); - } - - #[test] - fn scan_fixture_exports_repo_pp_cir_and_ccr_metrics() { - let td = TempDir::new().expect("tempdir"); - let run = td.path().join("runs/run_0001"); - fs::create_dir_all(&run).expect("create run"); - fs::write( - run.join("run-meta.json"), - r#"{"status":"success","run_index":1,"run_id":"run_0001","sync_mode":"snapshot","snapshot_reason":"first_run","started_at_rfc3339_utc":"2026-05-25T00:00:00Z","completed_at_rfc3339_utc":"2026-05-25T00:00:10Z"}"#, - ) - .expect("meta"); - fs::write( - run.join("run-summary.json"), - r#"{"runSeq":1,"runId":"run_0001","runDir":"RUN","startedAtRfc3339Utc":"2026-05-25T00:00:00Z","finishedAtRfc3339Utc":"2026-05-25T00:00:10Z","wallMs":10000,"status":"success","exitCode":0,"processMetrics":{"userSeconds":2.5,"systemSeconds":1.5,"cpuPercent":40,"maxRssKb":1000},"stageTiming":{"validation_ms":7000,"total_ms":9000,"download_event_count":2,"download_bytes_total":1234},"reportCounts":{"vrps":2,"aspas":1,"publicationPoints":2,"warnings":0},"repoSyncStats":{"by_phase":{"rrdp_delta":{"count":2,"duration_ms_total":3000}},"by_terminal_state":{"fresh":{"count":2,"duration_ms_total":3000}}},"pathStats":[{"label":"work-db","totalSizeBytes":99,"fileCount":2,"dirCount":1}],"artifacts":[{"path":"report.json","sizeBytes":10}]}"#, - ) - .expect("summary"); - fs::write(run.join("process-time.txt"), "time").expect("time"); - fs::write(run.join("stage-timing.json"), "{}").expect("stage"); - fs::write( - run.join("report.json"), - r#"{"tree":{"instances_processed":2,"instances_failed":0,"warnings":[]},"vrps":[{},{}],"aspas":[{}],"downloads":[{"kind":"rrdp_notification","uri":"https://repo.example/notify.xml","success":true,"duration_ms":100,"bytes":111},{"kind":"rrdp_delta","uri":"https://repo.example/session/1/delta.xml","success":true,"duration_ms":200,"bytes":222}],"publication_points":[{"rsync_base_uri":"rsync://repo.example/a/","manifest_rsync_uri":"rsync://repo.example/a/a.mft","publication_point_rsync_uri":"rsync://repo.example/a/","rrdp_notification_uri":"https://repo.example/notify.xml","repo_sync_source":"rrdp","repo_sync_phase":"rrdp_delta","repo_sync_duration_ms":1000,"repo_terminal_state":"fresh","objects":[{"kind":"roa","result":"ok"},{"kind":"manifest","result":"ok"}]},{"rsync_base_uri":"rsync://repo.example/b/","manifest_rsync_uri":"rsync://repo.example/b/b.mft","publication_point_rsync_uri":"rsync://repo.example/b/","rrdp_notification_uri":"https://repo.example/notify.xml","repo_sync_source":"rrdp","repo_sync_phase":"rrdp_delta","repo_sync_duration_ms":2000,"repo_terminal_state":"fresh","objects":[{"kind":"roa","result":"ok"}]}],"repo_sync_stats":{"publication_points_total":2,"by_phase":{"rrdp_delta":{"count":2,"duration_ms_total":3000}},"by_terminal_state":{"fresh":{"count":2,"duration_ms_total":3000}}}}"#, - ) - .expect("report"); - fs::write(run.join("input.cir"), sample_cir()).expect("cir"); - fs::write(run.join("result.ccr"), sample_ccr()).expect("ccr"); - - let snapshot = scan_run_root(td.path(), "test").expect("scan"); - assert_eq!(snapshot.runs.success, 1); - assert_eq!(snapshot.repo_stats.len(), 1); - assert!(snapshot.repo_stats[0].sync_success); - assert_eq!(snapshot.repo_stats[0].download_bytes, 333); - assert_eq!(snapshot.top_pp_by_object_count[0].object_count, 2); - assert_eq!(snapshot.cir.as_ref().unwrap().objects, 1); - assert_eq!(snapshot.ccr.as_ref().unwrap().state_items["tas"], 1); - let metrics = render_metrics(&snapshot); - assert!(metrics.contains("ours_rp_repository_info")); - assert!(metrics.contains("ours_rp_repository_sync_success")); - assert!(metrics.contains("ours_rp_repository_download_bytes")); - assert!(metrics.contains("ours_rp_large_publication_points")); - assert!(metrics.contains("ours_rp_cir_objects")); - assert!(metrics.contains("ours_rp_ccr_state_items")); - let status = render_status_json(&snapshot).expect("status"); - assert!(status.contains("topPublicationPointsByObjectCount")); - } - - #[test] - fn partial_run_does_not_become_latest_success() { - let td = TempDir::new().expect("tempdir"); - let run = td.path().join("runs/run_0001"); - fs::create_dir_all(&run).expect("create run"); - fs::write(run.join("run-meta.json"), r#"{"status":"running"}"#).expect("meta"); - let snapshot = scan_run_root(td.path(), "test").expect("scan"); - assert_eq!(snapshot.runs.partial, 1); - assert!(snapshot.latest_run.is_none()); - } - - fn sample_cir() -> Vec { - let rejected = vec![CirRejectedObject { - object_uri: "rsync://repo.example/a/bad.roa".to_string(), - reason: Some("bad".to_string()), - }]; - let cir = CanonicalInputRepresentation { - version: rpki::cir::CIR_VERSION_V3, - hash_alg: CirHashAlgorithm::Sha256, - validation_time: time::OffsetDateTime::parse( - "2026-05-25T00:00:00Z", - &time::format_description::well_known::Rfc3339, - ) - .unwrap(), - objects: vec![CirObject { - rsync_uri: "rsync://repo.example/a/a.roa".to_string(), - sha256: vec![1; 32], - }], - trust_anchors: vec![CirTrustAnchor { - ta_rsync_uri: "rsync://repo.example/ta.cer".to_string(), - tal_uri: "https://tal.example/tal.tal".to_string(), - tal_bytes: b"rsync://repo.example/ta.cer\n\nAQID\n".to_vec(), - ta_certificate_der: b"ta".to_vec(), - ta_certificate_sha256: sha256(b"ta"), - }], - reject_list_sha256: compute_reject_list_sha256( - rejected.iter().map(|item| item.object_uri.as_str()), - ), - rejected_objects: rejected, - }; - encode_cir(&cir).expect("encode cir") - } - - fn sample_ccr() -> Vec { - let ci = CcrContentInfo::new(RpkiCanonicalCacheRepresentation { - version: CCR_VERSION_V0, - hash_alg: CcrDigestAlgorithm::Sha256, - produced_at: time::OffsetDateTime::parse( - "2026-05-25T00:00:00Z", - &time::format_description::well_known::Rfc3339, - ) - .unwrap(), - mfts: None, - vrps: None, - vaps: None, - tas: Some(TrustAnchorState { - skis: vec![vec![1; 20]], - hash: vec![2; 32], - }), - rks: None, - }); - encode_content_info(&ci).expect("encode ccr") - } -} diff --git a/src/bin/rpki_daemon.rs b/src/bin/rpki_daemon.rs index 0bdc3a4..e901e80 100644 --- a/src/bin/rpki_daemon.rs +++ b/src/bin/rpki_daemon.rs @@ -1,1783 +1,6 @@ -use serde::Serialize; -use std::collections::BTreeMap; -use std::fs::{self, File, OpenOptions}; -use std::io::Write; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::time::Duration; - -#[derive(Clone, Debug, PartialEq, Eq)] -struct Args { - state_root: PathBuf, - rpki_bin: PathBuf, - interval_secs: u64, - max_runs: Option, - retain_runs: usize, - status_json: Option, - summary_jsonl: Option, - work_db: PathBuf, - repo_bytes_db: Option, - raw_store_db: Option, - db_stats_bin: Option, - db_stats_exact_every: Option, - time_bin: Option, - child_args: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct RunContext { - seq: u64, - run_id: String, - run_dir: PathBuf, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -enum DaemonState { - Starting, - Idle, - Running, - Collecting, - Sleeping, - Exited, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -struct DaemonStatus { - state: DaemonState, - updated_at_rfc3339_utc: String, - runs_completed: u64, - max_runs: Option, - current_run_seq: Option, - current_run_id: Option, - last_run_id: Option, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -enum RunStatus { - Success, - Failed, - SpawnFailed, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -struct ArtifactInfo { - path: String, - size_bytes: u64, -} - -#[derive(Clone, Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct ProcessMetrics { - time_wrapper_used: bool, - time_output_path: Option, - user_seconds: Option, - system_seconds: Option, - cpu_percent: Option, - elapsed_raw: Option, - max_rss_kb: Option, - exit_status_from_time: Option, - parse_error: Option, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -struct ReportCounts { - vrps: usize, - aspas: usize, - publication_points: usize, - rrdp_repos_unique: Option, - tree_instances_processed: Option, - tree_instances_failed: Option, - warnings: usize, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -struct PathFileStats { - label: String, - path: String, - exists: bool, - is_dir: bool, - total_size_bytes: u64, - file_count: u64, - dir_count: u64, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -struct DbStatsSummary { - mode: String, - db_path: String, - output_path: Option, - stderr_path: Option, - status: String, - exit_code: Option, - error: Option, - metrics: BTreeMap, -} - -#[derive(Clone, Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct RunSummary { - run_seq: u64, - run_id: String, - run_dir: String, - started_at_rfc3339_utc: String, - finished_at_rfc3339_utc: String, - wall_ms: u64, - status: RunStatus, - exit_code: Option, - exit_status: Option, - error: Option, - rpki_bin: String, - child_args: Vec, - stdout_path: String, - stderr_path: String, - process_metrics: Option, - stage_timing: Option, - report_counts: Option, - repo_sync_stats: Option, - path_stats: Vec, - db_stats: Vec, - retention_deleted_runs: Vec, - artifacts: Vec, -} - -fn usage() -> String { - let bin = "rpki_daemon"; - format!( - "\ -Usage: - {bin} --state-root --rpki-bin [options] -- - -Options: - --state-root Persistent daemon root containing state/, runs/, status, and JSONL summary - --rpki-bin rpki child binary to execute for each run - --interval-secs Sleep seconds between runs (default: 60) - --max-runs Stop after n runs (default: run forever) - --retain-runs Keep only the latest n run directories (default: 10) - --status-json Override status JSON path (default: /daemon-status.json) - --summary-jsonl Override summary JSONL path (default: /daemon-runs.jsonl) - --work-db Work DB path for metrics (default: /state/work-db) - --repo-bytes-db Repo bytes DB path for file metrics (default: /state/repo-bytes.db) - --raw-store-db Raw store DB path for file metrics (optional) - --db-stats-bin db_stats binary path (default: sibling db_stats next to this executable when present) - --db-stats-exact-every - Run db_stats --exact every n runs (default: disabled) - --time-bin GNU time binary for child process metrics (default: /usr/bin/time when present) - --no-time-wrapper Disable GNU time wrapper - --help Show this help - -Child argument placeholders: - {{state_root}} Daemon state root - {{run_out}} Current run output directory - {{run_id}} Current run id, e.g. 000001-20260428T090000Z - {{run_seq}} Current run sequence number -" - ) -} - -fn default_time_bin() -> Option { - let path = PathBuf::from("/usr/bin/time"); - if path.is_file() { Some(path) } else { None } -} - -fn default_db_stats_bin() -> Option { - let mut path = std::env::current_exe().ok()?; - path.set_file_name("db_stats"); - if path.is_file() { Some(path) } else { None } -} - -fn parse_args(argv: &[String]) -> Result { - if argv.iter().any(|arg| arg == "--help" || arg == "-h") { - return Err(usage()); - } - - let mut state_root: Option = None; - let mut rpki_bin: Option = None; - let mut interval_secs = 60u64; - let mut max_runs = None; - let mut retain_runs = 10usize; - let mut status_json: Option = None; - let mut summary_jsonl: Option = None; - let mut work_db: Option = None; - let mut repo_bytes_db: Option = None; - let mut raw_store_db: Option = None; - let mut db_stats_bin: Option = None; - let mut db_stats_exact_every = None; - let mut time_bin = default_time_bin(); - let mut no_time_wrapper = false; - - let mut i = 1usize; - while i < argv.len() { - match argv[i].as_str() { - "--" => { - let child_args = argv[i + 1..].to_vec(); - let state_root = - state_root.ok_or_else(|| format!("--state-root is required\n\n{}", usage()))?; - let work_db = work_db.unwrap_or_else(|| state_root.join("state").join("work-db")); - let repo_bytes_db = - repo_bytes_db.or_else(|| Some(state_root.join("state").join("repo-bytes.db"))); - if no_time_wrapper { - time_bin = None; - } - let args = Args { - state_root, - rpki_bin: rpki_bin - .ok_or_else(|| format!("--rpki-bin is required\n\n{}", usage()))?, - interval_secs, - max_runs, - retain_runs, - status_json, - summary_jsonl, - work_db, - repo_bytes_db, - raw_store_db, - db_stats_bin, - db_stats_exact_every, - time_bin, - child_args, - }; - return validate_args(args); - } - "--state-root" => { - i += 1; - state_root = Some(PathBuf::from(value_at(argv, i, "--state-root")?)); - } - "--rpki-bin" => { - i += 1; - rpki_bin = Some(PathBuf::from(value_at(argv, i, "--rpki-bin")?)); - } - "--interval-secs" => { - i += 1; - interval_secs = - parse_u64(value_at(argv, i, "--interval-secs")?, "--interval-secs")?; - } - "--max-runs" => { - i += 1; - let parsed = parse_u64(value_at(argv, i, "--max-runs")?, "--max-runs")?; - if parsed == 0 { - return Err("--max-runs must be > 0".to_string()); - } - max_runs = Some(parsed); - } - "--retain-runs" => { - i += 1; - let parsed = parse_usize(value_at(argv, i, "--retain-runs")?, "--retain-runs")?; - if parsed == 0 { - return Err("--retain-runs must be > 0".to_string()); - } - retain_runs = parsed; - } - "--status-json" => { - i += 1; - status_json = Some(PathBuf::from(value_at(argv, i, "--status-json")?)); - } - "--summary-jsonl" => { - i += 1; - summary_jsonl = Some(PathBuf::from(value_at(argv, i, "--summary-jsonl")?)); - } - "--work-db" => { - i += 1; - work_db = Some(PathBuf::from(value_at(argv, i, "--work-db")?)); - } - "--repo-bytes-db" => { - i += 1; - repo_bytes_db = Some(PathBuf::from(value_at(argv, i, "--repo-bytes-db")?)); - } - "--raw-store-db" => { - i += 1; - raw_store_db = Some(PathBuf::from(value_at(argv, i, "--raw-store-db")?)); - } - "--db-stats-bin" => { - i += 1; - db_stats_bin = Some(PathBuf::from(value_at(argv, i, "--db-stats-bin")?)); - } - "--db-stats-exact-every" => { - i += 1; - let parsed = parse_u64( - value_at(argv, i, "--db-stats-exact-every")?, - "--db-stats-exact-every", - )?; - if parsed == 0 { - return Err("--db-stats-exact-every must be > 0".to_string()); - } - db_stats_exact_every = Some(parsed); - } - "--time-bin" => { - i += 1; - time_bin = Some(PathBuf::from(value_at(argv, i, "--time-bin")?)); - } - "--no-time-wrapper" => { - no_time_wrapper = true; - } - other => return Err(format!("unknown argument: {other}\n\n{}", usage())), - } - i += 1; - } - - Err(format!("missing -- before child rpki args\n\n{}", usage())) -} - -fn validate_args(args: Args) -> Result { - if args.child_args.is_empty() { - return Err(format!( - "child rpki args are required after --\n\n{}", - usage() - )); - } - Ok(args) -} - -fn value_at<'a>(argv: &'a [String], index: usize, flag: &str) -> Result<&'a str, String> { - argv.get(index) - .map(String::as_str) - .ok_or_else(|| format!("{flag} requires a value")) -} - -fn parse_u64(raw: &str, flag: &str) -> Result { - raw.parse::() - .map_err(|_| format!("invalid {flag}: {raw}")) -} - -fn parse_usize(raw: &str, flag: &str) -> Result { - raw.parse::() - .map_err(|_| format!("invalid {flag}: {raw}")) -} - -fn status_path(args: &Args) -> PathBuf { - args.status_json - .clone() - .unwrap_or_else(|| args.state_root.join("daemon-status.json")) -} - -fn summary_jsonl_path(args: &Args) -> PathBuf { - args.summary_jsonl - .clone() - .unwrap_or_else(|| args.state_root.join("daemon-runs.jsonl")) -} - -fn utc_now() -> time::OffsetDateTime { - time::OffsetDateTime::now_utc().to_offset(time::UtcOffset::UTC) -} - -fn format_rfc3339(t: time::OffsetDateTime) -> Result { - t.format(&time::format_description::well_known::Rfc3339) - .map_err(|e| format!("format RFC3339 failed: {e}")) -} - -fn format_compact_utc(t: time::OffsetDateTime) -> String { - format!( - "{:04}{:02}{:02}T{:02}{:02}{:02}Z", - t.year(), - u8::from(t.month()), - t.day(), - t.hour(), - t.minute(), - t.second() - ) -} - -fn write_json_pretty(path: &Path, value: &T) -> Result<(), String> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .map_err(|e| format!("create parent dir failed: {}: {e}", parent.display()))?; - } - let file = - File::create(path).map_err(|e| format!("create json failed: {}: {e}", path.display()))?; - serde_json::to_writer_pretty(file, value) - .map_err(|e| format!("write json failed: {}: {e}", path.display())) -} - -fn append_json_line(path: &Path, value: &T) -> Result<(), String> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .map_err(|e| format!("create parent dir failed: {}: {e}", parent.display()))?; - } - let mut file = OpenOptions::new() - .create(true) - .append(true) - .open(path) - .map_err(|e| format!("open jsonl failed: {}: {e}", path.display()))?; - serde_json::to_writer(&mut file, value) - .map_err(|e| format!("write jsonl failed: {}: {e}", path.display()))?; - file.write_all(b"\n") - .map_err(|e| format!("flush jsonl failed: {}: {e}", path.display())) -} - -fn write_status( - args: &Args, - state: DaemonState, - runs_completed: u64, - current: Option<&RunContext>, - last_run_id: Option, -) -> Result<(), String> { - let updated_at_rfc3339_utc = format_rfc3339(utc_now())?; - let status = DaemonStatus { - state, - updated_at_rfc3339_utc, - runs_completed, - max_runs: args.max_runs, - current_run_seq: current.map(|ctx| ctx.seq), - current_run_id: current.map(|ctx| ctx.run_id.clone()), - last_run_id, - }; - write_json_pretty(&status_path(args), &status) -} - -fn render_child_args(args: &[String], daemon_args: &Args, ctx: &RunContext) -> Vec { - args.iter() - .map(|arg| { - arg.replace("{state_root}", &path_string(&daemon_args.state_root)) - .replace("{run_out}", &path_string(&ctx.run_dir)) - .replace("{run_id}", &ctx.run_id) - .replace("{run_seq}", &ctx.seq.to_string()) - }) - .collect() -} - -fn render_path_template(path: &Path, daemon_args: &Args, ctx: &RunContext) -> PathBuf { - PathBuf::from( - path_string(path) - .replace("{state_root}", &path_string(&daemon_args.state_root)) - .replace("{run_out}", &path_string(&ctx.run_dir)) - .replace("{run_id}", &ctx.run_id) - .replace("{run_seq}", &ctx.seq.to_string()), - ) -} - -fn path_string(path: &Path) -> String { - path.to_string_lossy().into_owned() -} - -fn make_run_context(args: &Args, seq: u64, now: time::OffsetDateTime) -> RunContext { - let run_id = format!("{seq:06}-{}", format_compact_utc(now)); - let run_dir = args.state_root.join("runs").join(&run_id); - RunContext { - seq, - run_id, - run_dir, - } -} - -fn collect_artifacts(run_dir: &Path) -> Result, String> { - let mut artifacts = Vec::new(); - for entry in fs::read_dir(run_dir) - .map_err(|e| format!("read run dir failed: {}: {e}", run_dir.display()))? - { - let entry = entry.map_err(|e| format!("read run dir entry failed: {e}"))?; - if !entry - .file_type() - .map_err(|e| format!("read file type failed: {}: {e}", entry.path().display()))? - .is_file() - { - continue; - } - let metadata = entry - .metadata() - .map_err(|e| format!("read metadata failed: {}: {e}", entry.path().display()))?; - artifacts.push(ArtifactInfo { - path: path_string(&entry.path()), - size_bytes: metadata.len(), - }); - } - artifacts.sort_by(|a, b| a.path.cmp(&b.path)); - Ok(artifacts) -} - -fn collect_process_metrics(time_wrapper_used: bool, time_output_path: &Path) -> ProcessMetrics { - if !time_wrapper_used { - return ProcessMetrics { - time_wrapper_used, - time_output_path: None, - user_seconds: None, - system_seconds: None, - cpu_percent: None, - elapsed_raw: None, - max_rss_kb: None, - exit_status_from_time: None, - parse_error: None, - }; - } - - let mut metrics = ProcessMetrics { - time_wrapper_used, - time_output_path: Some(path_string(time_output_path)), - user_seconds: None, - system_seconds: None, - cpu_percent: None, - elapsed_raw: None, - max_rss_kb: None, - exit_status_from_time: None, - parse_error: None, - }; - - let text = match fs::read_to_string(time_output_path) { - Ok(text) => text, - Err(err) => { - metrics.parse_error = Some(format!( - "read process time output failed: {}: {err}", - time_output_path.display() - )); - return metrics; - } - }; - - for line in text.lines() { - let line = line.trim(); - if let Some(value) = line.strip_prefix("User time (seconds):") { - metrics.user_seconds = value.trim().parse::().ok(); - } else if let Some(value) = line.strip_prefix("System time (seconds):") { - metrics.system_seconds = value.trim().parse::().ok(); - } else if let Some(value) = line.strip_prefix("Percent of CPU this job got:") { - metrics.cpu_percent = value.trim().trim_end_matches('%').parse::().ok(); - } else if let Some(value) = - line.strip_prefix("Elapsed (wall clock) time (h:mm:ss or m:ss):") - { - metrics.elapsed_raw = Some(value.trim().to_string()); - } else if let Some(value) = line.strip_prefix("Maximum resident set size (kbytes):") { - metrics.max_rss_kb = value.trim().parse::().ok(); - } else if let Some(value) = line.strip_prefix("Exit status:") { - metrics.exit_status_from_time = value.trim().parse::().ok(); - } - } - metrics -} - -fn run_child_once(args: &Args, ctx: &RunContext) -> Result { - fs::create_dir_all(&ctx.run_dir) - .map_err(|e| format!("create run dir failed: {}: {e}", ctx.run_dir.display()))?; - - let started_at = utc_now(); - let started_at_rfc3339_utc = format_rfc3339(started_at)?; - let stdout_path = ctx.run_dir.join("stdout.log"); - let stderr_path = ctx.run_dir.join("stderr.log"); - let stdout = File::create(&stdout_path) - .map_err(|e| format!("create stdout log failed: {}: {e}", stdout_path.display()))?; - let stderr = File::create(&stderr_path) - .map_err(|e| format!("create stderr log failed: {}: {e}", stderr_path.display()))?; - let child_args = render_child_args(&args.child_args, args, ctx); - let time_output_path = ctx.run_dir.join("process-time.txt"); - - let mut command = if let Some(time_bin) = args.time_bin.as_ref() { - let mut command = Command::new(time_bin); - command - .arg("-v") - .arg("-o") - .arg(&time_output_path) - .arg("--") - .arg(&args.rpki_bin) - .args(&child_args); - command - } else { - let mut command = Command::new(&args.rpki_bin); - command.args(&child_args); - command - }; - command - .stdout(Stdio::from(stdout)) - .stderr(Stdio::from(stderr)); - - let (status, exit_code, exit_status, error) = match command.status() { - Ok(status) if status.success() => ( - RunStatus::Success, - status.code(), - Some(status.to_string()), - None, - ), - Ok(status) => ( - RunStatus::Failed, - status.code(), - Some(status.to_string()), - None, - ), - Err(err) => ( - RunStatus::SpawnFailed, - None, - None, - Some(format!("spawn child failed: {err}")), - ), - }; - - let finished_at = utc_now(); - let finished_at_rfc3339_utc = format_rfc3339(finished_at)?; - let wall_ms = (finished_at - started_at).whole_milliseconds().max(0) as u64; - let process_metrics = Some(collect_process_metrics( - args.time_bin.is_some(), - &time_output_path, - )); - let artifacts = collect_artifacts(&ctx.run_dir)?; - let summary = RunSummary { - run_seq: ctx.seq, - run_id: ctx.run_id.clone(), - run_dir: path_string(&ctx.run_dir), - started_at_rfc3339_utc, - finished_at_rfc3339_utc, - wall_ms, - status, - exit_code, - exit_status, - error, - rpki_bin: path_string(&args.rpki_bin), - child_args, - stdout_path: path_string(&stdout_path), - stderr_path: path_string(&stderr_path), - process_metrics, - stage_timing: None, - report_counts: None, - repo_sync_stats: None, - path_stats: Vec::new(), - db_stats: Vec::new(), - retention_deleted_runs: Vec::new(), - artifacts, - }; - Ok(summary) -} - -fn apply_retention(runs_root: &Path, retain_runs: usize) -> Result, String> { - if !runs_root.exists() { - return Ok(Vec::new()); - } - let mut dirs = Vec::new(); - for entry in fs::read_dir(runs_root) - .map_err(|e| format!("read runs dir failed: {}: {e}", runs_root.display()))? - { - let entry = entry.map_err(|e| format!("read runs dir entry failed: {e}"))?; - if entry - .file_type() - .map_err(|e| format!("read file type failed: {}: {e}", entry.path().display()))? - .is_dir() - { - dirs.push(entry.path()); - } - } - dirs.sort(); - let remove_count = dirs.len().saturating_sub(retain_runs); - let mut removed = Vec::new(); - for dir in dirs.into_iter().take(remove_count) { - fs::remove_dir_all(&dir) - .map_err(|e| format!("remove old run dir failed: {}: {e}", dir.display()))?; - removed.push(dir); - } - Ok(removed) -} - -fn find_named_file(root: &Path, name: &str) -> Option { - let mut stack = vec![root.to_path_buf()]; - while let Some(dir) = stack.pop() { - let entries = fs::read_dir(&dir).ok()?; - for entry in entries.flatten() { - let path = entry.path(); - let file_type = entry.file_type().ok()?; - if file_type.is_file() && entry.file_name().to_string_lossy() == name { - return Some(path); - } - if file_type.is_dir() { - stack.push(path); - } - } - } - None -} - -fn read_json_value_if_exists(path: &Path) -> Option { - let bytes = fs::read(path).ok()?; - serde_json::from_slice(&bytes).ok() -} - -fn json_array_len(value: &serde_json::Value, key: &str) -> usize { - value - .get(key) - .and_then(serde_json::Value::as_array) - .map(Vec::len) - .unwrap_or(0) -} - -fn parse_stdout_summary(run_dir: &Path) -> Option { - let stdout_path = run_dir.join("stdout.log"); - let text = fs::read_to_string(stdout_path).ok()?; - let mut vrps = None; - let mut aspas = None; - let mut publication_points = None; - let mut rrdp_repos_unique = None; - let mut tree_instances_processed = None; - let mut tree_instances_failed = None; - let mut warnings = None; - - for line in text.lines() { - if let Some(value) = line.strip_prefix("vrps=") { - vrps = value.trim().parse::().ok(); - } else if let Some(value) = line.strip_prefix("aspas=") { - aspas = value.trim().parse::().ok(); - } else if let Some(value) = line.strip_prefix("audit_publication_points=") { - publication_points = value.trim().parse::().ok(); - } else if let Some(value) = line.strip_prefix("rrdp_repos_unique=") { - rrdp_repos_unique = value.trim().parse::().ok(); - } else if let Some(value) = line.strip_prefix("warnings_total=") { - warnings = value.trim().parse::().ok(); - } else if let Some(rest) = line.strip_prefix("publication_points_processed=") { - for token in rest.split_whitespace() { - if let Some(value) = token.strip_prefix("publication_points_failed=") { - tree_instances_failed = value.parse::().ok(); - } else if tree_instances_processed.is_none() { - tree_instances_processed = token.parse::().ok(); - } - } - } - } - - Some(ReportCounts { - vrps: vrps?, - aspas: aspas?, - publication_points: publication_points?, - rrdp_repos_unique, - tree_instances_processed, - tree_instances_failed, - warnings: warnings.unwrap_or(0), - }) -} - -fn parse_report_counts_fallback(report: &serde_json::Value) -> ReportCounts { - let tree = report.get("tree"); - let tree_warnings = tree - .and_then(|tree| tree.get("warnings")) - .and_then(serde_json::Value::as_array) - .map(Vec::len) - .unwrap_or(0); - let pp_warnings = report - .get("publication_points") - .and_then(serde_json::Value::as_array) - .map(|items| { - items - .iter() - .map(|pp| { - pp.get("warnings") - .and_then(serde_json::Value::as_array) - .map(Vec::len) - .unwrap_or(0) - }) - .sum() - }) - .unwrap_or(0); - ReportCounts { - vrps: json_array_len(report, "vrps"), - aspas: json_array_len(report, "aspas"), - publication_points: json_array_len(report, "publication_points"), - rrdp_repos_unique: None, - tree_instances_processed: tree - .and_then(|tree| tree.get("instances_processed")) - .and_then(serde_json::Value::as_u64), - tree_instances_failed: tree - .and_then(|tree| tree.get("instances_failed")) - .and_then(serde_json::Value::as_u64), - warnings: tree_warnings + pp_warnings, - } -} - -fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option { - haystack - .windows(needle.len()) - .position(|window| window == needle) -} - -fn extract_json_object_field(path: &Path, field_name: &str) -> Option { - let bytes = fs::read(path).ok()?; - let needle = format!("\"{field_name}\":"); - let pos = find_subslice(&bytes, needle.as_bytes())?; - let mut i = pos + needle.len(); - while i < bytes.len() && bytes[i].is_ascii_whitespace() { - i += 1; - } - if bytes.get(i).copied()? != b'{' { - return None; - } - let start = i; - let mut depth = 0u32; - let mut in_string = false; - let mut escaped = false; - for (offset, &b) in bytes[start..].iter().enumerate() { - if in_string { - if escaped { - escaped = false; - } else if b == b'\\' { - escaped = true; - } else if b == b'"' { - in_string = false; - } - continue; - } - match b { - b'"' => in_string = true, - b'{' => depth = depth.saturating_add(1), - b'}' => { - depth = depth.saturating_sub(1); - if depth == 0 { - let end = start + offset + 1; - return serde_json::from_slice(&bytes[start..end]).ok(); - } - } - _ => {} - } - } - None -} - -fn parse_report_metadata(run_dir: &Path) -> (Option, Option) { - let Some(report_path) = find_named_file(run_dir, "report.json") else { - return (parse_stdout_summary(run_dir), None); - }; - let counts = parse_stdout_summary(run_dir).or_else(|| { - read_json_value_if_exists(&report_path).map(|report| parse_report_counts_fallback(&report)) - }); - let repo_sync_stats = extract_json_object_field(&report_path, "repo_sync_stats"); - (counts, repo_sync_stats) -} - -fn collect_path_file_stats(label: &str, path: &Path) -> PathFileStats { - let mut stats = PathFileStats { - label: label.to_string(), - path: path_string(path), - exists: path.exists(), - is_dir: path.is_dir(), - total_size_bytes: 0, - file_count: 0, - dir_count: 0, - }; - if !stats.exists { - return stats; - } - if path.is_file() { - if let Ok(metadata) = path.metadata() { - stats.total_size_bytes = metadata.len(); - stats.file_count = 1; - } - return stats; - } - - let mut stack = vec![path.to_path_buf()]; - while let Some(dir) = stack.pop() { - let Ok(entries) = fs::read_dir(&dir) else { - continue; - }; - for entry in entries.flatten() { - let Ok(file_type) = entry.file_type() else { - continue; - }; - if file_type.is_dir() { - stats.dir_count = stats.dir_count.saturating_add(1); - stack.push(entry.path()); - } else if file_type.is_file() { - stats.file_count = stats.file_count.saturating_add(1); - if let Ok(metadata) = entry.metadata() { - stats.total_size_bytes = stats.total_size_bytes.saturating_add(metadata.len()); - } - } - } - } - stats -} - -fn collect_state_path_stats(args: &Args, ctx: &RunContext) -> Vec { - let mut stats = Vec::new(); - stats.push(collect_path_file_stats( - "work_db", - &render_path_template(&args.work_db, args, ctx), - )); - if let Some(path) = args.repo_bytes_db.as_ref() { - stats.push(collect_path_file_stats( - "repo_bytes_db", - &render_path_template(path, args, ctx), - )); - } - if let Some(path) = args.raw_store_db.as_ref() { - stats.push(collect_path_file_stats( - "raw_store_db", - &render_path_template(path, args, ctx), - )); - } - stats -} - -fn parse_key_value_metrics(text: &str) -> BTreeMap { - let mut metrics = BTreeMap::new(); - for line in text.lines() { - let Some((key, value)) = line.split_once('=') else { - continue; - }; - metrics.insert(key.trim().to_string(), value.trim().to_string()); - } - metrics -} - -fn run_db_stats_command( - db_stats_bin: &Path, - db_path: &Path, - run_dir: &Path, - mode: &str, -) -> DbStatsSummary { - let output_path = run_dir.join(format!("db-stats-{mode}.txt")); - let stderr_path = run_dir.join(format!("db-stats-{mode}.stderr.txt")); - let mut summary = DbStatsSummary { - mode: mode.to_string(), - db_path: path_string(db_path), - output_path: Some(path_string(&output_path)), - stderr_path: None, - status: "success".to_string(), - exit_code: None, - error: None, - metrics: BTreeMap::new(), - }; - - if !db_path.exists() { - summary.status = "skipped".to_string(); - summary.error = Some(format!("db path does not exist: {}", db_path.display())); - summary.output_path = None; - return summary; - } - - let mut command = Command::new(db_stats_bin); - command.arg("--db").arg(db_path); - if mode == "exact" { - command.arg("--exact"); - } - match command.output() { - Ok(output) => { - summary.exit_code = output.status.code(); - if !output.status.success() { - summary.status = "failed".to_string(); - } - let stdout_text = String::from_utf8_lossy(&output.stdout).into_owned(); - if let Err(err) = fs::write(&output_path, stdout_text.as_bytes()) { - summary.status = "failed".to_string(); - summary.error = Some(format!( - "write db_stats output failed: {}: {err}", - output_path.display() - )); - } - summary.metrics = parse_key_value_metrics(&stdout_text); - if !output.stderr.is_empty() { - if fs::write(&stderr_path, &output.stderr).is_ok() { - summary.stderr_path = Some(path_string(&stderr_path)); - } - } - if !output.status.success() && summary.error.is_none() { - summary.error = Some(String::from_utf8_lossy(&output.stderr).into_owned()); - } - } - Err(err) => { - summary.status = "spawn_failed".to_string(); - summary.exit_code = None; - summary.output_path = None; - summary.error = Some(format!("spawn db_stats failed: {err}")); - } - } - summary -} - -fn collect_db_stats(args: &Args, ctx: &RunContext) -> Vec { - let work_db = render_path_template(&args.work_db, args, ctx); - let Some(db_stats_bin) = args - .db_stats_bin - .as_ref() - .cloned() - .or_else(default_db_stats_bin) - else { - return vec![DbStatsSummary { - mode: "estimate".to_string(), - db_path: path_string(&work_db), - output_path: None, - stderr_path: None, - status: "skipped".to_string(), - exit_code: None, - error: Some( - "db_stats binary not configured and sibling db_stats was not found".to_string(), - ), - metrics: BTreeMap::new(), - }]; - }; - - let mut stats = Vec::new(); - stats.push(run_db_stats_command( - &db_stats_bin, - &work_db, - &ctx.run_dir, - "estimate", - )); - if args - .db_stats_exact_every - .is_some_and(|every| ctx.seq % every == 0) - { - stats.push(run_db_stats_command( - &db_stats_bin, - &work_db, - &ctx.run_dir, - "exact", - )); - } - stats -} - -fn collect_post_run_metrics(args: &Args, ctx: &RunContext, summary: &mut RunSummary) { - if let Some(path) = find_named_file(&ctx.run_dir, "stage-timing.json") { - summary.stage_timing = read_json_value_if_exists(&path); - } - let (report_counts, repo_sync_stats) = parse_report_metadata(&ctx.run_dir); - summary.report_counts = report_counts; - summary.repo_sync_stats = repo_sync_stats; - summary.path_stats = collect_state_path_stats(args, ctx); - summary.db_stats = collect_db_stats(args, ctx); - summary.artifacts = collect_artifacts(&ctx.run_dir).unwrap_or_default(); -} - -fn run_daemon(args: &Args) -> Result<(), String> { - fs::create_dir_all(args.state_root.join("state")).map_err(|e| { - format!( - "create daemon state dir failed: {}: {e}", - args.state_root.join("state").display() - ) - })?; - fs::create_dir_all(args.state_root.join("runs")).map_err(|e| { - format!( - "create daemon runs dir failed: {}: {e}", - args.state_root.join("runs").display() - ) - })?; - - let mut runs_completed = 0u64; - let mut next_seq = 1u64; - let mut last_run_id = None; - write_status( - args, - DaemonState::Starting, - runs_completed, - None, - last_run_id.clone(), - )?; - - loop { - if args.max_runs.is_some_and(|max| runs_completed >= max) { - break; - } - - write_status( - args, - DaemonState::Idle, - runs_completed, - None, - last_run_id.clone(), - )?; - let ctx = make_run_context(args, next_seq, utc_now()); - write_status( - args, - DaemonState::Running, - runs_completed, - Some(&ctx), - last_run_id.clone(), - )?; - let mut summary = run_child_once(args, &ctx)?; - write_status( - args, - DaemonState::Collecting, - runs_completed, - Some(&ctx), - last_run_id.clone(), - )?; - collect_post_run_metrics(args, &ctx, &mut summary); - let removed = apply_retention(&args.state_root.join("runs"), args.retain_runs)?; - summary.retention_deleted_runs = removed.iter().map(|p| path_string(p)).collect(); - summary.artifacts = collect_artifacts(&ctx.run_dir).unwrap_or_default(); - write_json_pretty(&ctx.run_dir.join("run-summary.json"), &summary)?; - append_json_line(&summary_jsonl_path(args), &summary)?; - runs_completed += 1; - next_seq += 1; - last_run_id = Some(ctx.run_id); - - if args.max_runs.is_some_and(|max| runs_completed >= max) { - break; - } - write_status( - args, - DaemonState::Sleeping, - runs_completed, - None, - last_run_id.clone(), - )?; - if args.interval_secs > 0 { - std::thread::sleep(Duration::from_secs(args.interval_secs)); - } - } - - write_status(args, DaemonState::Exited, runs_completed, None, last_run_id) -} - fn main() { - let argv: Vec = std::env::args().collect(); - match parse_args(&argv) { - Ok(args) => { - if let Err(err) = run_daemon(&args) { - eprintln!("{err}"); - std::process::exit(2); - } - } - Err(err) => { - if argv.iter().any(|a| a == "--help" || a == "-h") { - println!("{err}"); - return; - } - eprintln!("{err}"); - std::process::exit(2); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn test_args(state_root: PathBuf) -> Args { - Args { - work_db: state_root.join("state/work-db"), - repo_bytes_db: Some(state_root.join("state/repo-bytes.db")), - state_root, - rpki_bin: PathBuf::from("/bin/true"), - interval_secs: 0, - max_runs: Some(1), - retain_runs: 10, - status_json: None, - summary_jsonl: None, - raw_store_db: None, - db_stats_bin: None, - db_stats_exact_every: None, - time_bin: None, - child_args: vec!["--version".to_string()], - } - } - - #[test] - fn parse_args_accepts_required_flags_and_child_args() { - let argv = vec![ - "rpki_daemon".to_string(), - "--state-root".to_string(), - "/tmp/daemon".to_string(), - "--rpki-bin".to_string(), - "/bin/echo".to_string(), - "--interval-secs".to_string(), - "0".to_string(), - "--max-runs".to_string(), - "2".to_string(), - "--retain-runs".to_string(), - "3".to_string(), - "--".to_string(), - "--db".to_string(), - "{state_root}/state/work-db".to_string(), - "--report-json".to_string(), - "{run_out}/report.json".to_string(), - ]; - - let args = parse_args(&argv).expect("parse args"); - assert_eq!(args.state_root, PathBuf::from("/tmp/daemon")); - assert_eq!(args.rpki_bin, PathBuf::from("/bin/echo")); - assert_eq!(args.interval_secs, 0); - assert_eq!(args.max_runs, Some(2)); - assert_eq!(args.retain_runs, 3); - assert_eq!(args.work_db, PathBuf::from("/tmp/daemon/state/work-db")); - assert_eq!( - args.repo_bytes_db, - Some(PathBuf::from("/tmp/daemon/state/repo-bytes.db")) - ); - assert_eq!( - args.child_args, - vec![ - "--db", - "{state_root}/state/work-db", - "--report-json", - "{run_out}/report.json" - ] - ); - } - - #[test] - fn usage_and_parse_args_cover_optional_flags_and_errors() { - let help = usage(); - assert!(help.contains("--db-stats-exact-every")); - assert!(help.contains("{run_seq}")); - - let argv = vec![ - "rpki_daemon".to_string(), - "--state-root".to_string(), - "/tmp/daemon".to_string(), - "--rpki-bin".to_string(), - "/bin/echo".to_string(), - "--status-json".to_string(), - "/tmp/status.json".to_string(), - "--summary-jsonl".to_string(), - "/tmp/runs.jsonl".to_string(), - "--work-db".to_string(), - "{state_root}/work".to_string(), - "--repo-bytes-db".to_string(), - "{state_root}/repo-bytes".to_string(), - "--raw-store-db".to_string(), - "{state_root}/raw".to_string(), - "--db-stats-bin".to_string(), - "/bin/echo".to_string(), - "--db-stats-exact-every".to_string(), - "2".to_string(), - "--time-bin".to_string(), - "/usr/bin/time".to_string(), - "--no-time-wrapper".to_string(), - "--".to_string(), - "child".to_string(), - ]; - let args = parse_args(&argv).expect("optional args"); - assert_eq!(args.status_json, Some(PathBuf::from("/tmp/status.json"))); - assert_eq!(args.summary_jsonl, Some(PathBuf::from("/tmp/runs.jsonl"))); - assert_eq!(args.work_db, PathBuf::from("{state_root}/work")); - assert_eq!( - args.repo_bytes_db, - Some(PathBuf::from("{state_root}/repo-bytes")) - ); - assert_eq!(args.raw_store_db, Some(PathBuf::from("{state_root}/raw"))); - assert_eq!(args.db_stats_bin, Some(PathBuf::from("/bin/echo"))); - assert_eq!(args.db_stats_exact_every, Some(2)); - assert_eq!(args.time_bin, None); - - for (argv, expected) in [ - (vec!["rpki_daemon", "--help"], "Usage:"), - ( - vec![ - "rpki_daemon", - "--state-root", - "/tmp/x", - "--rpki-bin", - "/bin/echo", - "--max-runs", - "0", - "--", - "child", - ], - "--max-runs must be > 0", - ), - ( - vec![ - "rpki_daemon", - "--state-root", - "/tmp/x", - "--rpki-bin", - "/bin/echo", - "--retain-runs", - "0", - "--", - "child", - ], - "--retain-runs must be > 0", - ), - ( - vec![ - "rpki_daemon", - "--state-root", - "/tmp/x", - "--rpki-bin", - "/bin/echo", - "--db-stats-exact-every", - "0", - "--", - "child", - ], - "--db-stats-exact-every must be > 0", - ), - (vec!["rpki_daemon", "--unknown"], "unknown argument"), - ( - vec!["rpki_daemon", "--state-root"], - "--state-root requires a value", - ), - (vec!["rpki_daemon"], "missing -- before child rpki args"), - ( - vec![ - "rpki_daemon", - "--state-root", - "/tmp/x", - "--rpki-bin", - "/bin/echo", - "--", - ], - "child rpki args are required", - ), - ] { - let owned: Vec = argv.into_iter().map(str::to_string).collect(); - let err = parse_args(&owned).expect_err("parse should fail"); - assert!(err.contains(expected), "{err}"); - } - } - - #[test] - fn render_child_args_replaces_placeholders() { - let args = Args { - state_root: PathBuf::from("/tmp/root"), - rpki_bin: PathBuf::from("/bin/echo"), - interval_secs: 0, - max_runs: Some(1), - retain_runs: 10, - status_json: None, - summary_jsonl: None, - work_db: PathBuf::from("/tmp/root/state/work-db"), - repo_bytes_db: Some(PathBuf::from("/tmp/root/state/repo-bytes.db")), - raw_store_db: None, - db_stats_bin: None, - db_stats_exact_every: None, - time_bin: None, - child_args: vec![ - "{state_root}/state/work-db".to_string(), - "{run_out}/result.ccr".to_string(), - "{run_id}".to_string(), - "{run_seq}".to_string(), - ], - }; - let ctx = RunContext { - seq: 7, - run_id: "000007-20260428T090000Z".to_string(), - run_dir: PathBuf::from("/tmp/root/runs/000007-20260428T090000Z"), - }; - - assert_eq!( - render_child_args(&args.child_args, &args, &ctx), - vec![ - "/tmp/root/state/work-db", - "/tmp/root/runs/000007-20260428T090000Z/result.ccr", - "000007-20260428T090000Z", - "7", - ] - ); - } - - #[test] - fn path_json_and_report_helpers_cover_fallbacks_and_nested_stats() { - let td = tempfile::tempdir().expect("tempdir"); - let state_root = td.path().join("daemon"); - let mut args = test_args(state_root.clone()); - args.work_db = PathBuf::from("{state_root}/state/work-db"); - args.repo_bytes_db = Some(PathBuf::from("{state_root}/state/repo-bytes.db")); - args.raw_store_db = Some(PathBuf::from("{state_root}/state/raw-store.db")); - - let now = time::Date::from_calendar_date(2026, time::Month::April, 28) - .expect("date") - .with_hms(9, 0, 0) - .expect("time") - .assume_utc(); - let ctx = make_run_context(&args, 42, now); - assert_eq!(ctx.run_id, "000042-20260428T090000Z"); - - let rendered = render_path_template(Path::new("{run_out}/{run_id}/{run_seq}"), &args, &ctx); - assert!(rendered.ends_with("000042-20260428T090000Z/000042-20260428T090000Z/42")); - - let nested_json = td.path().join("nested/out/status.json"); - write_json_pretty(&nested_json, &serde_json::json!({"ok": true})).expect("write json"); - append_json_line( - &td.path().join("nested/out/runs.jsonl"), - &serde_json::json!({"n": 1}), - ) - .expect("append jsonl"); - - fs::create_dir_all(ctx.run_dir.join("subdir")).expect("subdir"); - fs::write(ctx.run_dir.join("a.txt"), "aaa").expect("file"); - fs::write(ctx.run_dir.join("subdir/ignored.txt"), "bbb").expect("nested file"); - let artifacts = collect_artifacts(&ctx.run_dir).expect("artifacts"); - assert_eq!(artifacts.len(), 1); - assert!(artifacts[0].path.ends_with("a.txt")); - - fs::create_dir_all(state_root.join("state/work-db/nested")).expect("work db"); - fs::write(state_root.join("state/work-db/file.sst"), "abc").expect("sst"); - fs::write(state_root.join("state/work-db/nested/inner.sst"), "def").expect("inner"); - fs::write(state_root.join("state/raw-store.db"), "raw").expect("raw file"); - let file_stats = - collect_path_file_stats("raw_store_db", &state_root.join("state/raw-store.db")); - assert!(file_stats.exists); - assert!(!file_stats.is_dir); - assert_eq!(file_stats.file_count, 1); - let missing_stats = collect_path_file_stats("missing", &state_root.join("missing")); - assert!(!missing_stats.exists); - let state_stats = collect_state_path_stats(&args, &ctx); - assert!( - state_stats - .iter() - .any(|s| s.label == "raw_store_db" && s.exists) - ); - assert!( - state_stats - .iter() - .any(|s| s.label == "work_db" && s.file_count == 2) - ); - - let report_path = td.path().join("report.json"); - fs::write( - &report_path, - r#"{"repo_sync_stats": { "nested": {"text": "a\"b"} }, "after": 1}"#, - ) - .expect("report"); - assert_eq!( - extract_json_object_field(&report_path, "repo_sync_stats").expect("repo stats")["nested"] - ["text"], - "a\"b" - ); - fs::write(&report_path, r#"{"repo_sync_stats": []}"#).expect("report"); - assert!(extract_json_object_field(&report_path, "repo_sync_stats").is_none()); - assert!(extract_json_object_field(&report_path, "missing").is_none()); - - let counts = parse_report_counts_fallback(&serde_json::json!({ - "vrps": [{}, {}], - "aspas": [{}], - "publication_points": [{"warnings": [{}, {}]}, {"warnings": [{}]}], - "tree": {"instances_processed": 2, "instances_failed": 1, "warnings": [{}]} - })); - assert_eq!(counts.vrps, 2); - assert_eq!(counts.aspas, 1); - assert_eq!(counts.publication_points, 2); - assert_eq!(counts.warnings, 4); - assert_eq!(counts.tree_instances_processed, Some(2)); - assert_eq!(counts.tree_instances_failed, Some(1)); - } - - #[test] - fn retention_removes_oldest_run_directories() { - let td = tempfile::tempdir().expect("tempdir"); - let runs = td.path().join("runs"); - fs::create_dir_all(&runs).expect("runs dir"); - for name in [ - "000001-20260428T000001Z", - "000002-20260428T000002Z", - "000003-20260428T000003Z", - ] { - fs::create_dir_all(runs.join(name)).expect("run dir"); - } - - let removed = apply_retention(&runs, 2).expect("retention"); - assert_eq!(removed.len(), 1); - assert!(!runs.join("000001-20260428T000001Z").exists()); - assert!(runs.join("000002-20260428T000002Z").exists()); - assert!(runs.join("000003-20260428T000003Z").exists()); - } - - #[test] - fn retention_empty_root_and_parse_metrics_error_paths_are_reported() { - let td = tempfile::tempdir().expect("tempdir"); - let missing_runs = td.path().join("missing-runs"); - assert!( - apply_retention(&missing_runs, 2) - .expect("empty retention") - .is_empty() - ); - - let disabled = collect_process_metrics(false, &td.path().join("missing-time.txt")); - assert!(!disabled.time_wrapper_used); - assert!(disabled.time_output_path.is_none()); - - let missing = collect_process_metrics(true, &td.path().join("missing-time.txt")); - assert!(missing.time_wrapper_used); - assert!( - missing - .parse_error - .expect("parse error") - .contains("read process time output failed") - ); - } - - #[test] - fn process_metrics_parses_gnu_time_elapsed_line() { - let td = tempfile::tempdir().expect("tempdir"); - let path = td.path().join("time.txt"); - fs::write( - &path, - "User time (seconds): 1.25\nSystem time (seconds): 0.50\nPercent of CPU this job got: 175%\nElapsed (wall clock) time (h:mm:ss or m:ss): 0:01.00\nMaximum resident set size (kbytes): 12345\nExit status: 0\n", - ) - .expect("write time"); - - let metrics = collect_process_metrics(true, &path); - assert_eq!(metrics.user_seconds, Some(1.25)); - assert_eq!(metrics.system_seconds, Some(0.50)); - assert_eq!(metrics.cpu_percent, Some(175.0)); - assert_eq!(metrics.elapsed_raw.as_deref(), Some("0:01.00")); - assert_eq!(metrics.max_rss_kb, Some(12345)); - assert_eq!(metrics.exit_status_from_time, Some(0)); - } - - #[test] - #[cfg(unix)] - fn child_and_db_stats_error_paths_are_reported() { - use std::os::unix::fs::PermissionsExt; - - let td = tempfile::tempdir().expect("tempdir"); - let run_dir = td.path().join("run"); - fs::create_dir_all(&run_dir).expect("run dir"); - let missing_db = td.path().join("missing-db"); - let skipped = - run_db_stats_command(Path::new("/bin/echo"), &missing_db, &run_dir, "estimate"); - assert_eq!(skipped.status, "skipped"); - assert!(skipped.output_path.is_none()); - - let db_path = td.path().join("db"); - fs::create_dir_all(&db_path).expect("db"); - let failing_bin = td.path().join("failing_db_stats.sh"); - fs::write( - &failing_bin, - "#!/bin/sh\necho partial=1\necho db-stats failed >&2\nexit 7\n", - ) - .expect("script"); - let mut permissions = fs::metadata(&failing_bin).expect("metadata").permissions(); - permissions.set_mode(0o755); - fs::set_permissions(&failing_bin, permissions).expect("chmod"); - let failed = run_db_stats_command(&failing_bin, &db_path, &run_dir, "exact"); - assert_eq!(failed.status, "failed"); - assert_eq!(failed.exit_code, Some(7)); - assert_eq!(failed.metrics.get("partial").map(String::as_str), Some("1")); - assert!(failed.stderr_path.is_some()); - assert!(failed.error.expect("stderr").contains("db-stats failed")); - - let spawn_failed = run_db_stats_command( - Path::new("/definitely/not/db_stats"), - &db_path, - &run_dir, - "estimate", - ); - assert_eq!(spawn_failed.status, "spawn_failed"); - assert!( - spawn_failed - .error - .expect("spawn err") - .contains("spawn db_stats failed") - ); - - let metrics = parse_key_value_metrics("alpha=1\nnot-a-pair\nbeta = two\n"); - assert_eq!(metrics.get("alpha").map(String::as_str), Some("1")); - assert_eq!(metrics.get("beta").map(String::as_str), Some("two")); - - let mut args = test_args(td.path().join("daemon")); - args.rpki_bin = PathBuf::from("/bin/sh"); - args.time_bin = Some(PathBuf::from("/usr/bin/time")); - args.child_args = vec![ - "-c".to_string(), - "echo child-out; echo child-err >&2; exit 7".to_string(), - ]; - let ctx = RunContext { - seq: 1, - run_id: "000001-20260428T000000Z".to_string(), - run_dir: args.state_root.join("runs/000001-20260428T000000Z"), - }; - let summary = run_child_once(&args, &ctx).expect("run failing child"); - assert_eq!(summary.status, RunStatus::Failed); - assert_eq!(summary.exit_code, Some(7)); - let process_metrics = summary.process_metrics.expect("process metrics"); - assert!(process_metrics.time_wrapper_used); - assert_eq!(process_metrics.exit_status_from_time, Some(7)); - - let mut spawn_args = test_args(td.path().join("spawn")); - spawn_args.rpki_bin = PathBuf::from("/definitely/not/rpki"); - spawn_args.child_args = vec!["--help".to_string()]; - let spawn_ctx = RunContext { - seq: 1, - run_id: "000001-20260428T000001Z".to_string(), - run_dir: spawn_args.state_root.join("runs/000001-20260428T000001Z"), - }; - let spawn_summary = run_child_once(&spawn_args, &spawn_ctx).expect("spawn summary"); - assert_eq!(spawn_summary.status, RunStatus::SpawnFailed); - assert!( - spawn_summary - .error - .expect("spawn error") - .contains("spawn child failed") - ); - } - - #[test] - fn report_metadata_prefers_stdout_summary_and_extracts_repo_sync_stats() { - let td = tempfile::tempdir().expect("tempdir"); - fs::write( - td.path().join("stdout.log"), - "RPKI stage2 serial run summary\npublication_points_processed=7 publication_points_failed=1\nrrdp_repos_unique=3\nvrps=11\naspas=2\naudit_publication_points=7\nwarnings_total=5\n", - ) - .expect("stdout"); - fs::write( - td.path().join("report.json"), - "{\"large\":[{\"ignored\":\"{}\"}],\"repo_sync_stats\":{\"by_phase\":{\"rrdp_ok\":{\"count\":7}},\"by_terminal_state\":{},\"publication_points_total\":7}}", - ) - .expect("report"); - - let (counts, repo_sync_stats) = parse_report_metadata(td.path()); - let counts = counts.expect("counts"); - assert_eq!(counts.vrps, 11); - assert_eq!(counts.aspas, 2); - assert_eq!(counts.publication_points, 7); - assert_eq!(counts.rrdp_repos_unique, Some(3)); - assert_eq!(counts.tree_instances_processed, Some(7)); - assert_eq!(counts.tree_instances_failed, Some(1)); - assert_eq!(counts.warnings, 5); - assert_eq!( - repo_sync_stats.expect("repo sync")["by_phase"]["rrdp_ok"]["count"].as_u64(), - Some(7) - ); - } - - #[test] - fn daemon_exits_immediately_when_max_runs_already_reached() { - let td = tempfile::tempdir().expect("tempdir"); - let mut args = test_args(td.path().join("daemon")); - args.max_runs = Some(0); - - run_daemon(&args).expect("run daemon"); - - let status_text = - fs::read_to_string(args.state_root.join("daemon-status.json")).expect("status"); - assert!(status_text.contains("\"state\": \"exited\"")); - assert!(status_text.contains("\"runsCompleted\": 0")); - assert!(args.state_root.join("runs").exists()); - } - - #[test] - fn daemon_runs_fake_child_twice_and_writes_summaries() { - let td = tempfile::tempdir().expect("tempdir"); - let args = Args { - state_root: td.path().join("daemon"), - rpki_bin: PathBuf::from("/bin/sh"), - interval_secs: 0, - max_runs: Some(2), - retain_runs: 10, - status_json: None, - summary_jsonl: None, - work_db: td.path().join("daemon/state/work-db"), - repo_bytes_db: Some(td.path().join("daemon/state/repo-bytes.db")), - raw_store_db: None, - db_stats_bin: None, - db_stats_exact_every: None, - time_bin: None, - child_args: vec![ - "-c".to_string(), - "echo stdout-{run_seq}; echo stderr-{run_seq} >&2; echo marker > {run_out}/marker.txt".to_string(), - ], - }; - - run_daemon(&args).expect("run daemon"); - - let status_text = - fs::read_to_string(args.state_root.join("daemon-status.json")).expect("status"); - assert!(status_text.contains("\"state\": \"exited\"")); - assert!(status_text.contains("\"runsCompleted\": 2")); - - let jsonl = - fs::read_to_string(args.state_root.join("daemon-runs.jsonl")).expect("summary jsonl"); - assert_eq!(jsonl.lines().count(), 2); - - let run_dirs = fs::read_dir(args.state_root.join("runs")) - .expect("runs dir") - .collect::, _>>() - .expect("entries"); - assert_eq!(run_dirs.len(), 2); - for entry in run_dirs { - assert!(entry.path().join("run-summary.json").exists()); - assert!(entry.path().join("marker.txt").exists()); - } - } - - #[test] - fn daemon_sleep_and_retention_deletion_are_recorded() { - let td = tempfile::tempdir().expect("tempdir"); - let mut args = test_args(td.path().join("daemon")); - args.rpki_bin = PathBuf::from("/bin/sh"); - args.interval_secs = 1; - args.max_runs = Some(2); - args.retain_runs = 1; - args.child_args = vec![ - "-c".to_string(), - "echo run-{run_seq}; printf marker > {run_out}/marker.txt".to_string(), - ]; - - run_daemon(&args).expect("run daemon"); - - let jsonl = fs::read_to_string(args.state_root.join("daemon-runs.jsonl")).expect("jsonl"); - assert_eq!(jsonl.lines().count(), 2); - let last: serde_json::Value = - serde_json::from_str(jsonl.lines().last().expect("last line")).expect("summary"); - assert_eq!( - last["retentionDeletedRuns"] - .as_array() - .expect("deleted") - .len(), - 1 - ); - let run_dirs = fs::read_dir(args.state_root.join("runs")) - .expect("runs") - .collect::, _>>() - .expect("entries"); - assert_eq!(run_dirs.len(), 1); - } - - #[test] - #[cfg(unix)] - fn daemon_collects_stage_report_db_and_file_metrics() { - use std::os::unix::fs::PermissionsExt; - - let td = tempfile::tempdir().expect("tempdir"); - let db_stats_bin = td.path().join("fake_db_stats.sh"); - fs::write( - &db_stats_bin, - "#!/bin/sh\nif [ \"$3\" = \"--exact\" ]; then echo mode=exact; else echo mode=estimate; fi\necho total=42\necho db.files.total_size_bytes=123\n", - ) - .expect("fake db_stats"); - let mut permissions = fs::metadata(&db_stats_bin).expect("metadata").permissions(); - permissions.set_mode(0o755); - fs::set_permissions(&db_stats_bin, permissions).expect("chmod"); - - let args = Args { - state_root: td.path().join("daemon"), - rpki_bin: PathBuf::from("/bin/sh"), - interval_secs: 0, - max_runs: Some(1), - retain_runs: 10, - status_json: None, - summary_jsonl: None, - work_db: PathBuf::from("{state_root}/state/work-db"), - repo_bytes_db: Some(PathBuf::from("{state_root}/state/repo-bytes.db")), - raw_store_db: None, - db_stats_bin: Some(db_stats_bin), - db_stats_exact_every: Some(1), - time_bin: None, - child_args: vec![ - "-c".to_string(), - "mkdir -p {state_root}/state/work-db {state_root}/state/repo-bytes.db; \ - printf x > {state_root}/state/work-db/000001.sst; \ - printf '{\"validation_ms\":7,\"download_event_count\":2}' > {run_out}/stage-timing.json; \ - printf '{\"tree\":{\"instances_processed\":3,\"instances_failed\":1,\"warnings\":[{}]},\"publication_points\":[{},{}],\"vrps\":[{},{}],\"aspas\":[{}],\"repo_sync_stats\":{\"by_phase\":{\"snapshot\":{\"count\":1}}}}' > {run_out}/report.json" - .to_string(), - ], - }; - - run_daemon(&args).expect("run daemon"); - - let run_summary = find_named_file(&args.state_root.join("runs"), "run-summary.json") - .expect("run summary"); - let summary: serde_json::Value = - serde_json::from_slice(&fs::read(run_summary).expect("read run summary")) - .expect("parse run summary"); - - assert_eq!(summary["stageTiming"]["validation_ms"].as_u64(), Some(7)); - assert_eq!(summary["reportCounts"]["vrps"].as_u64(), Some(2)); - assert_eq!(summary["reportCounts"]["aspas"].as_u64(), Some(1)); - assert_eq!( - summary["reportCounts"]["publicationPoints"].as_u64(), - Some(2) - ); - assert_eq!( - summary["repoSyncStats"]["by_phase"]["snapshot"]["count"].as_u64(), - Some(1) - ); - assert_eq!(summary["dbStats"].as_array().expect("db stats").len(), 2); - assert!( - summary["pathStats"] - .as_array() - .expect("path stats") - .iter() - .any(|item| item["label"] == "work_db" && item["exists"] == true) - ); + let code = rpki::tools::rpki_daemon::main_entry(); + if code != 0 { + std::process::exit(code); } } diff --git a/src/bin/sequence_triage_ccr_cir.rs b/src/bin/sequence_triage_ccr_cir.rs index 7b76a99..74994d6 100644 --- a/src/bin/sequence_triage_ccr_cir.rs +++ b/src/bin/sequence_triage_ccr_cir.rs @@ -1,2714 +1,6 @@ -use std::collections::{BTreeMap, BTreeSet}; -use std::path::{Path, PathBuf}; - -use rpki::ccr::{decode_ccr_compare_views, decode_content_info}; -use rpki::cir::decode_cir; -use serde::Deserialize; -use serde_json::{Value, json}; -use time::OffsetDateTime; -use time::format_description::well_known::Rfc3339; - -#[derive(Debug, PartialEq, Eq)] -struct Args { - left_sequence: PathBuf, - right_sequence: PathBuf, - out_dir: PathBuf, - align_window_runs: u32, - align_window_secs: i64, - sample_limit: usize, - warmup_samples: usize, - cooldown_samples: usize, - timeline_sample_limit: usize, -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct SequenceItemRaw { - schema_version: Option, - rp_id: String, - side: Option, - seq: u32, - run_id: String, - sync_mode: Option, - status: Option, - start_time: Option, - finish_time: Option, - validation_time: Option, - ccr_path: PathBuf, - cir_path: PathBuf, - ccr_sha256: Option, - cir_sha256: Option, - wall_ms: Option, - max_rss_kb: Option, - vrps: Option, - vaps: Option, -} - -#[derive(Clone, Debug)] -struct SequenceSample { - raw: SequenceItemRaw, - validation_time: OffsetDateTime, - ccr_path: PathBuf, - cir_path: PathBuf, - objects: BTreeMap, - object_uris: BTreeSet, - object_hashes: BTreeSet, - rejects: BTreeSet, - trust_anchors: BTreeSet, - vrps: BTreeSet, - vaps: BTreeSet, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Side { - Left, - Right, -} - -impl Side { - fn as_str(self) -> &'static str { - match self { - Side::Left => "left", - Side::Right => "right", - } - } -} - -#[derive(Clone, Debug)] -struct EventOccurrence { - side: Side, - seq: u32, - run_id: String, -} - -#[derive(Clone, Debug)] -struct SampleRecord { - classification: &'static str, - event_type: &'static str, - key: String, - source_side: Side, - source_seq: u32, - source_run_id: String, - matched_seq: Option, - matched_run_id: Option, - note: String, -} - -#[derive(Clone, Debug, Default)] -struct ClassStats { - total: usize, - samples: Vec, -} - -#[derive(Clone, Debug, Default)] -struct AnalysisResult { - stats: BTreeMap<&'static str, ClassStats>, -} - -#[derive(Clone, Debug)] -struct DiffEvent { - event_type: &'static str, - raw_class: &'static str, - key: String, - source_side: Side, - source_seq: u32, - source_run_id: String, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum EdgePosition { - Leading, - Stable, - Trailing, -} - -#[derive(Clone, Debug)] -struct AdjustedRecord { - classification: &'static str, - event_type: &'static str, - key: String, - source_side: Side, - source_seq: u32, - source_run_id: String, - note: String, -} - -#[derive(Clone, Debug, Default)] -struct AdjustedClassStats { - total: usize, - unique_keys: BTreeSet, - samples: Vec, -} - -#[derive(Clone, Debug, Default)] -struct AdjustedAnalysis { - raw_persistent_occurrences: usize, - raw_persistent_unique_keys: usize, - edge_filtered_occurrences: usize, - edge_filtered_unique_keys: usize, - adjusted_stable_occurrences: usize, - adjusted_stable_unique_keys: usize, - stats: BTreeMap<&'static str, AdjustedClassStats>, - uri_timeline_samples: Vec, - stable_object_groups: Vec, -} - -#[derive(Clone, Debug)] -struct SandwichRecord { - classification: &'static str, - set_type: &'static str, - key: String, - source_side: Side, - source_start_seq: u32, - source_start_run_id: String, - source_end_seq: u32, - source_end_run_id: String, - peer_seq: u32, - peer_run_id: String, - source_value: Option, - peer_value: Option, - source_start_time: String, - peer_time: String, - source_end_time: String, - note: String, -} - -#[derive(Clone, Debug, Default)] -struct SandwichClassStats { - total: usize, - unique_keys: BTreeSet, - samples: Vec, -} - -#[derive(Clone, Debug, Default)] -struct SandwichAnalysis { - total_occurrences: usize, - unique_keys: BTreeSet, - by_set_type: BTreeMap<&'static str, usize>, - stats: BTreeMap<&'static str, SandwichClassStats>, -} - -fn usage() -> &'static str { - "Usage: sequence_triage_ccr_cir --left-sequence --right-sequence --out-dir [--align-window-runs ] [--align-window-secs ] [--sample-limit ] [--warmup-samples ] [--cooldown-samples ] [--timeline-sample-limit ]" -} - fn main() { - if let Err(err) = real_main() { + if let Err(err) = rpki::tools::sequence_triage_ccr_cir::main_entry() { eprintln!("{err}"); std::process::exit(1); } } - -fn real_main() -> Result<(), String> { - let args = parse_args(&std::env::args().collect::>())?; - run(args) -} - -fn parse_args(argv: &[String]) -> Result { - let mut left_sequence = None; - let mut right_sequence = None; - let mut out_dir = None; - let mut align_window_runs = 2u32; - let mut align_window_secs = 1800i64; - let mut sample_limit = 200usize; - let mut warmup_samples = 1usize; - let mut cooldown_samples = 1usize; - let mut timeline_sample_limit = 0usize; - let mut index = 1usize; - while index < argv.len() { - match argv[index].as_str() { - "--left-sequence" => { - index += 1; - left_sequence = Some(PathBuf::from( - argv.get(index).ok_or("--left-sequence requires a value")?, - )); - } - "--right-sequence" => { - index += 1; - right_sequence = Some(PathBuf::from( - argv.get(index).ok_or("--right-sequence requires a value")?, - )); - } - "--out-dir" => { - index += 1; - out_dir = Some(PathBuf::from( - argv.get(index).ok_or("--out-dir requires a value")?, - )); - } - "--align-window-runs" => { - index += 1; - let value = argv - .get(index) - .ok_or("--align-window-runs requires a value")?; - align_window_runs = value - .parse::() - .map_err(|_| format!("invalid --align-window-runs: {value}"))?; - } - "--align-window-secs" => { - index += 1; - let value = argv - .get(index) - .ok_or("--align-window-secs requires a value")?; - align_window_secs = value - .parse::() - .map_err(|_| format!("invalid --align-window-secs: {value}"))?; - } - "--sample-limit" => { - index += 1; - let value = argv.get(index).ok_or("--sample-limit requires a value")?; - sample_limit = value - .parse::() - .map_err(|_| format!("invalid --sample-limit: {value}"))?; - } - "--warmup-samples" => { - index += 1; - let value = argv.get(index).ok_or("--warmup-samples requires a value")?; - warmup_samples = value - .parse::() - .map_err(|_| format!("invalid --warmup-samples: {value}"))?; - } - "--cooldown-samples" => { - index += 1; - let value = argv - .get(index) - .ok_or("--cooldown-samples requires a value")?; - cooldown_samples = value - .parse::() - .map_err(|_| format!("invalid --cooldown-samples: {value}"))?; - } - "--timeline-sample-limit" => { - index += 1; - let value = argv - .get(index) - .ok_or("--timeline-sample-limit requires a value")?; - timeline_sample_limit = value - .parse::() - .map_err(|_| format!("invalid --timeline-sample-limit: {value}"))?; - } - "-h" | "--help" => return Err(usage().to_string()), - other => return Err(format!("unknown argument: {other}\n{}", usage())), - } - index += 1; - } - Ok(Args { - left_sequence: left_sequence - .ok_or_else(|| format!("--left-sequence is required\n{}", usage()))?, - right_sequence: right_sequence - .ok_or_else(|| format!("--right-sequence is required\n{}", usage()))?, - out_dir: out_dir.ok_or_else(|| format!("--out-dir is required\n{}", usage()))?, - align_window_runs, - align_window_secs, - sample_limit, - warmup_samples, - cooldown_samples, - timeline_sample_limit, - }) -} - -fn run(args: Args) -> Result<(), String> { - std::fs::create_dir_all(&args.out_dir) - .map_err(|e| format!("create out-dir failed: {}: {e}", args.out_dir.display()))?; - let left = load_sequence(&args.left_sequence, Side::Left)?; - let right = load_sequence(&args.right_sequence, Side::Right)?; - if left.is_empty() || right.is_empty() { - return Err("left and right sequences must both contain at least one sample".into()); - } - - let mut result = AnalysisResult::default(); - analyze_set( - &mut result, - "object_uri", - &left, - &right, - |sample| &sample.object_uris, - "TEMPORAL_LAG_RESOLVED", - "PERSISTENT_OBJECT_SET_DIVERGENCE", - &args, - ); - analyze_set( - &mut result, - "object_hash", - &left, - &right, - |sample| &sample.object_hashes, - "TEMPORAL_LAG_RESOLVED", - "PERSISTENT_CONTENT_DIVERGENCE", - &args, - ); - analyze_hash_rollover(&mut result, &left, &right, &args); - analyze_set( - &mut result, - "reject_uri", - &left, - &right, - |sample| &sample.rejects, - "TEMPORAL_LAG_RESOLVED", - "PERSISTENT_REJECT_DIVERGENCE", - &args, - ); - analyze_set( - &mut result, - "trust_anchor", - &left, - &right, - |sample| &sample.trust_anchors, - "TEMPORAL_LAG_RESOLVED", - "PERSISTENT_TA_DIFFERENCE", - &args, - ); - analyze_set( - &mut result, - "vrp_output", - &left, - &right, - |sample| &sample.vrps, - "TEMPORAL_LAG_RESOLVED", - "PERSISTENT_OUTPUT_DIVERGENCE", - &args, - ); - analyze_set( - &mut result, - "vap_output", - &left, - &right, - |sample| &sample.vaps, - "TEMPORAL_LAG_RESOLVED", - "PERSISTENT_OUTPUT_DIVERGENCE", - &args, - ); - - let persistent_events = collect_persistent_events(&left, &right, &args); - let adjusted = build_adjusted_analysis(&args, &left, &right, &persistent_events); - let sandwich = build_sandwich_analysis(&args, &left, &right); - let output = build_output(&args, &left, &right, &result, &adjusted, &sandwich); - write_json(&args.out_dir.join("sequence-triage.json"), &output)?; - write_markdown(&args.out_dir.join("sequence-triage.md"), &output)?; - write_samples_jsonl(&args.out_dir.join("sequence-diff-samples.jsonl"), &result)?; - println!("{}", args.out_dir.display()); - Ok(()) -} - -fn load_sequence(path: &Path, side: Side) -> Result, String> { - let base_dir = path.parent().unwrap_or_else(|| Path::new(".")); - let text = std::fs::read_to_string(path) - .map_err(|e| format!("read sequence failed: {}: {e}", path.display()))?; - let mut samples = Vec::new(); - let mut seen_seq = BTreeSet::new(); - for (line_index, line) in text.lines().enumerate() { - let line = line.trim(); - if line.is_empty() { - continue; - } - let raw: SequenceItemRaw = serde_json::from_str(line).map_err(|e| { - format!( - "parse sequence JSONL failed: {}:{}: {e}", - path.display(), - line_index + 1 - ) - })?; - if raw.schema_version.unwrap_or(1) != 1 { - return Err(format!( - "unsupported sequence item schemaVersion in {}:{}", - path.display(), - line_index + 1 - )); - } - if !seen_seq.insert(raw.seq) { - return Err(format!("duplicate seq {} in {}", raw.seq, path.display())); - } - if let Some(status) = &raw.status - && status != "success" - { - return Err(format!( - "sequence sample {} has non-success status: {status}", - raw.run_id - )); - } - let cir_path = resolve_path(base_dir, &raw.cir_path); - let ccr_path = resolve_path(base_dir, &raw.ccr_path); - let cir = decode_cir(&read_file(&cir_path)?).map_err(|e| { - format!( - "decode CIR failed for sample {} ({}): {e}", - raw.run_id, - cir_path.display() - ) - })?; - let ccr = decode_content_info(&read_file(&ccr_path)?).map_err(|e| { - format!( - "decode CCR failed for sample {} ({}): {e}", - raw.run_id, - ccr_path.display() - ) - })?; - let validation_time = raw - .validation_time - .as_deref() - .map(parse_rfc3339) - .transpose()? - .unwrap_or(cir.validation_time); - let objects = cir - .objects - .iter() - .map(|item| (item.rsync_uri.clone(), hex::encode(&item.sha256))) - .collect::>(); - let object_uris = objects.keys().cloned().collect::>(); - let object_hashes = objects - .iter() - .map(|(uri, hash)| object_hash_key(uri, hash)) - .collect::>(); - let rejects = cir - .rejected_objects - .iter() - .map(|item| item.object_uri.clone()) - .collect::>(); - let trust_anchors = cir - .trust_anchors - .iter() - .map(|item| { - format!( - "{}|{}|{}|{}", - item.ta_rsync_uri, - item.tal_uri, - hex::encode(rpki::cir::sha256(&item.tal_bytes)), - hex::encode(&item.ta_certificate_sha256) - ) - }) - .collect::>(); - let (vrps, vaps) = decode_ccr_compare_views(&ccr).map_err(|e| { - format!( - "decode CCR compare views failed for sample {} ({}): {e}", - raw.run_id, - ccr_path.display() - ) - })?; - let vrps = vrps - .into_iter() - .map(|row| format!("{}|{}|{}", row.asn, row.ip_prefix, row.max_length)) - .collect::>(); - let vaps = vaps - .into_iter() - .map(|row| format!("{}|{}", row.customer_asn, row.providers)) - .collect::>(); - samples.push(SequenceSample { - raw, - validation_time, - ccr_path, - cir_path, - objects, - object_uris, - object_hashes, - rejects, - trust_anchors, - vrps, - vaps, - }); - } - samples.sort_by_key(|sample| sample.raw.seq); - for pair in samples.windows(2) { - if pair[0].raw.seq >= pair[1].raw.seq { - return Err("sequence must be sorted by increasing seq".into()); - } - } - if samples.iter().any(|sample| { - sample - .raw - .side - .as_deref() - .is_some_and(|item| item != side.as_str()) - }) { - return Err(format!( - "sequence side field does not match expected side: {}", - side.as_str() - )); - } - Ok(samples) -} - -fn analyze_set( - result: &mut AnalysisResult, - event_type: &'static str, - left: &[SequenceSample], - right: &[SequenceSample], - extract: F, - resolved_class: &'static str, - persistent_class: &'static str, - args: &Args, -) where - F: for<'a> Fn(&'a SequenceSample) -> &'a BTreeSet, -{ - analyze_direction( - result, - event_type, - Side::Left, - left, - right, - &extract, - resolved_class, - persistent_class, - args, - ); - analyze_direction( - result, - event_type, - Side::Right, - right, - left, - &extract, - resolved_class, - persistent_class, - args, - ); -} - -fn analyze_direction( - result: &mut AnalysisResult, - event_type: &'static str, - source_side: Side, - source: &[SequenceSample], - peer: &[SequenceSample], - extract: &F, - resolved_class: &'static str, - persistent_class: &'static str, - args: &Args, -) where - F: for<'a> Fn(&'a SequenceSample) -> &'a BTreeSet, -{ - for sample in source { - let source_set = extract(sample); - for key in source_set { - if peer_sample_at_seq(peer, sample.raw.seq) - .is_some_and(|peer_sample| extract(peer_sample).contains(key)) - { - continue; - } - if let Some(matched) = find_future_match(peer, sample, &key, extract, args) { - result.add( - resolved_class, - SampleRecord { - classification: resolved_class, - event_type, - key: key.clone(), - source_side, - source_seq: sample.raw.seq, - source_run_id: sample.raw.run_id.clone(), - matched_seq: Some(matched.seq), - matched_run_id: Some(matched.run_id), - note: format!( - "matched in {} sequence within alignment window", - matched.side.as_str() - ), - }, - args.sample_limit, - ); - } else { - result.add( - persistent_class, - SampleRecord { - classification: persistent_class, - event_type, - key: key.clone(), - source_side, - source_seq: sample.raw.seq, - source_run_id: sample.raw.run_id.clone(), - matched_seq: None, - matched_run_id: None, - note: "no matching event in peer sequence alignment window".to_string(), - }, - args.sample_limit, - ); - } - } - } -} - -fn analyze_hash_rollover( - result: &mut AnalysisResult, - left: &[SequenceSample], - right: &[SequenceSample], - args: &Args, -) { - for (source_side, source, peer) in [(Side::Left, left, right), (Side::Right, right, left)] { - for sample in source { - for (uri, hash) in &sample.objects { - if peer_sample_at_seq(peer, sample.raw.seq) - .and_then(|peer_sample| peer_sample.objects.get(uri)) - .is_some_and(|peer_hash| peer_hash == hash) - { - continue; - } - if let Some(peer_sample) = peer_sample_at_seq(peer, sample.raw.seq) - && peer_sample.objects.contains_key(uri) - && find_future_hash_match(peer, sample, uri, hash, args).is_some() - { - let matched = - find_future_hash_match(peer, sample, uri, hash, args).expect("match"); - result.add( - "CONTENT_ROLLOVER_RESOLVED", - SampleRecord { - classification: "CONTENT_ROLLOVER_RESOLVED", - event_type: "object_content_rollover", - key: object_hash_key(uri, hash), - source_side, - source_seq: sample.raw.seq, - source_run_id: sample.raw.run_id.clone(), - matched_seq: Some(matched.seq), - matched_run_id: Some(matched.run_id), - note: "same URI hash appeared in peer sequence later".to_string(), - }, - args.sample_limit, - ); - } - } - } - } -} - -fn collect_persistent_events( - left: &[SequenceSample], - right: &[SequenceSample], - args: &Args, -) -> Vec { - let mut events = Vec::new(); - collect_persistent_set( - &mut events, - "object_uri", - "PERSISTENT_OBJECT_SET_DIVERGENCE", - left, - right, - |sample| &sample.object_uris, - args, - ); - collect_persistent_set( - &mut events, - "object_hash", - "PERSISTENT_CONTENT_DIVERGENCE", - left, - right, - |sample| &sample.object_hashes, - args, - ); - collect_persistent_set( - &mut events, - "reject_uri", - "PERSISTENT_REJECT_DIVERGENCE", - left, - right, - |sample| &sample.rejects, - args, - ); - collect_persistent_set( - &mut events, - "trust_anchor", - "PERSISTENT_TA_DIFFERENCE", - left, - right, - |sample| &sample.trust_anchors, - args, - ); - collect_persistent_set( - &mut events, - "vrp_output", - "PERSISTENT_OUTPUT_DIVERGENCE", - left, - right, - |sample| &sample.vrps, - args, - ); - collect_persistent_set( - &mut events, - "vap_output", - "PERSISTENT_OUTPUT_DIVERGENCE", - left, - right, - |sample| &sample.vaps, - args, - ); - events -} - -fn collect_persistent_set( - events: &mut Vec, - event_type: &'static str, - raw_class: &'static str, - left: &[SequenceSample], - right: &[SequenceSample], - extract: F, - args: &Args, -) where - F: for<'a> Fn(&'a SequenceSample) -> &'a BTreeSet, -{ - collect_persistent_direction( - events, - event_type, - raw_class, - Side::Left, - left, - right, - &extract, - args, - ); - collect_persistent_direction( - events, - event_type, - raw_class, - Side::Right, - right, - left, - &extract, - args, - ); -} - -fn collect_persistent_direction( - events: &mut Vec, - event_type: &'static str, - raw_class: &'static str, - source_side: Side, - source: &[SequenceSample], - peer: &[SequenceSample], - extract: &F, - args: &Args, -) where - F: for<'a> Fn(&'a SequenceSample) -> &'a BTreeSet, -{ - for sample in source { - for key in extract(sample) { - if peer_sample_at_seq(peer, sample.raw.seq) - .is_some_and(|peer_sample| extract(peer_sample).contains(key)) - { - continue; - } - if find_future_match(peer, sample, key, extract, args).is_some() { - continue; - } - events.push(DiffEvent { - event_type, - raw_class, - key: key.clone(), - source_side, - source_seq: sample.raw.seq, - source_run_id: sample.raw.run_id.clone(), - }); - } - } -} - -fn build_adjusted_analysis( - args: &Args, - left: &[SequenceSample], - right: &[SequenceSample], - events: &[DiffEvent], -) -> AdjustedAnalysis { - let mut analysis = AdjustedAnalysis { - raw_persistent_occurrences: events.len(), - raw_persistent_unique_keys: unique_event_count(events), - ..Default::default() - }; - let mut edge_filtered_unique = BTreeSet::new(); - - for event in events { - let source_samples = samples_for_side(event.source_side, left, right); - let peer_samples = samples_for_side(opposite_side(event.source_side), left, right); - let edge = edge_position(source_samples, event.source_seq, args); - if edge == EdgePosition::Stable { - analysis.edge_filtered_occurrences += 1; - edge_filtered_unique.insert(event_identity(event)); - } - let (classification, note) = - adjusted_classification(event, edge, source_samples, peer_samples); - analysis.add( - classification, - AdjustedRecord { - classification, - event_type: event.event_type, - key: event.key.clone(), - source_side: event.source_side, - source_seq: event.source_seq, - source_run_id: event.source_run_id.clone(), - note, - }, - args.sample_limit, - ); - } - - analysis.edge_filtered_unique_keys = edge_filtered_unique.len(); - let mut stable_unique = BTreeSet::new(); - for (class, stats) in &analysis.stats { - if class.starts_with("STABLE_") { - analysis.adjusted_stable_occurrences += stats.total; - stable_unique.extend(stats.unique_keys.iter().cloned()); - } - } - analysis.adjusted_stable_unique_keys = stable_unique.len(); - let timeline_limit = if args.timeline_sample_limit == 0 { - args.sample_limit - } else { - args.timeline_sample_limit - }; - analysis.uri_timeline_samples = - build_uri_timeline_samples(left, right, &analysis, timeline_limit); - analysis.stable_object_groups = - build_stable_object_groups(&analysis, left, right, timeline_limit); - analysis -} - -fn adjusted_classification( - event: &DiffEvent, - edge: EdgePosition, - source: &[SequenceSample], - peer: &[SequenceSample], -) -> (&'static str, String) { - if event.event_type == "trust_anchor" { - if peer_has_same_ta_identity(peer, &event.key) { - return ( - "TA_PROJECTION_FORMAT_DIFFERENCE", - "same TAL hash and TA certificate hash exist on peer side; full TA projection string differs".to_string(), - ); - } - if edge == EdgePosition::Stable { - return ( - "STABLE_TA_DIVERGENCE", - "trust-anchor identity is not aligned in a non-boundary sample".to_string(), - ); - } - return edge_unresolved_class(edge); - } - - if event.event_type == "object_hash" { - if let Some((uri, hash)) = split_object_hash_key(&event.key) { - let peer_has_uri = peer_has_uri_any(peer, uri); - let peer_has_different_hash = peer_has_uri_different_hash_any(peer, uri, hash); - let peer_hash_at_source_seq = peer_sample_at_seq(peer, event.source_seq) - .and_then(|peer_sample| peer_sample.objects.get(uri)); - let source_later_matches_peer_hash = peer_hash_at_source_seq.is_some_and(|peer_hash| { - source_future_has_hash(source, event.source_seq, uri, peer_hash) - }); - if edge == EdgePosition::Leading && peer_has_different_hash { - return ( - "EDGE_LEADING_CONTENT_ROLLOVER", - "source leading-edge hash is absent, while peer already has the same URI with another hash".to_string(), - ); - } - if edge == EdgePosition::Trailing && peer_has_different_hash { - return ( - "EDGE_TRAILING_CONTENT_ROLLOVER", - "source trailing-edge hash is absent, while peer has the same URI with another hash and no later source observation exists".to_string(), - ); - } - if edge == EdgePosition::Stable && source_later_matches_peer_hash { - return ( - "MID_SEQUENCE_CONTENT_ROLLOVER_RESOLVED", - "peer already has a newer same-URI hash at this seq and source catches up later in the observed sequence".to_string(), - ); - } - if edge == EdgePosition::Stable && peer_has_different_hash { - return ( - "STABLE_CONTENT_DIVERGENCE", - "same URI exists on peer side with a different hash in a non-boundary sample" - .to_string(), - ); - } - if edge == EdgePosition::Stable && !peer_has_uri { - return ( - "STABLE_OBJECT_SET_DIVERGENCE", - "content key is derived from a non-boundary URI that is absent on peer side" - .to_string(), - ); - } - } - if edge == EdgePosition::Stable { - return ( - "STABLE_CONTENT_DIVERGENCE", - "object hash key remains unaligned in a non-boundary sample".to_string(), - ); - } - return edge_unresolved_class(edge); - } - - if edge != EdgePosition::Stable { - return edge_unresolved_class(edge); - } - - let stable_class = match event.raw_class { - "PERSISTENT_OBJECT_SET_DIVERGENCE" => "STABLE_OBJECT_SET_DIVERGENCE", - "PERSISTENT_REJECT_DIVERGENCE" => "STABLE_REJECT_DIVERGENCE", - "PERSISTENT_OUTPUT_DIVERGENCE" => "STABLE_OUTPUT_DIVERGENCE", - "PERSISTENT_CONTENT_DIVERGENCE" => "STABLE_CONTENT_DIVERGENCE", - "PERSISTENT_TA_DIFFERENCE" => "STABLE_TA_DIVERGENCE", - _ => "STABLE_UNCLASSIFIED_DIVERGENCE", - }; - ( - stable_class, - "persistent event appears in a non-boundary sample after sequence edge filtering" - .to_string(), - ) -} - -fn edge_unresolved_class(edge: EdgePosition) -> (&'static str, String) { - match edge { - EdgePosition::Leading => ( - "EDGE_LEADING_UNRESOLVED", - "event appears only from the warmup edge and may predate the observed sequence" - .to_string(), - ), - EdgePosition::Trailing => ( - "EDGE_TRAILING_UNRESOLVED", - "event appears at the cooldown edge and lacks later peer observations".to_string(), - ), - EdgePosition::Stable => ( - "STABLE_UNCLASSIFIED_DIVERGENCE", - "event is not on an edge but no specific stable category matched".to_string(), - ), - } -} - -impl AdjustedAnalysis { - fn add(&mut self, class: &'static str, record: AdjustedRecord, sample_limit: usize) { - let stats = self.stats.entry(class).or_default(); - stats.total += 1; - stats.unique_keys.insert(format!( - "{}|{}|{}", - record.event_type, - record.source_side.as_str(), - record.key - )); - if stats.samples.len() < sample_limit { - stats.samples.push(record); - } - } -} - -fn build_sandwich_analysis( - args: &Args, - left: &[SequenceSample], - right: &[SequenceSample], -) -> SandwichAnalysis { - let mut analysis = SandwichAnalysis::default(); - analyze_sandwich_objects(&mut analysis, Side::Left, left, right, args); - analyze_sandwich_objects(&mut analysis, Side::Right, right, left, args); - analyze_sandwich_sets( - &mut analysis, - "reject_uri", - "PEER_MISSING_STABLE_REJECT", - Side::Left, - left, - right, - |sample| &sample.rejects, - args, - ); - analyze_sandwich_sets( - &mut analysis, - "reject_uri", - "PEER_MISSING_STABLE_REJECT", - Side::Right, - right, - left, - |sample| &sample.rejects, - args, - ); - analyze_sandwich_sets( - &mut analysis, - "vrp_output", - "PEER_MISSING_STABLE_OUTPUT", - Side::Left, - left, - right, - |sample| &sample.vrps, - args, - ); - analyze_sandwich_sets( - &mut analysis, - "vrp_output", - "PEER_MISSING_STABLE_OUTPUT", - Side::Right, - right, - left, - |sample| &sample.vrps, - args, - ); - analyze_sandwich_sets( - &mut analysis, - "vap_output", - "PEER_MISSING_STABLE_OUTPUT", - Side::Left, - left, - right, - |sample| &sample.vaps, - args, - ); - analyze_sandwich_sets( - &mut analysis, - "vap_output", - "PEER_MISSING_STABLE_OUTPUT", - Side::Right, - right, - left, - |sample| &sample.vaps, - args, - ); - analysis -} - -fn analyze_sandwich_objects( - analysis: &mut SandwichAnalysis, - source_side: Side, - source: &[SequenceSample], - peer: &[SequenceSample], - args: &Args, -) { - for pair in source.windows(2) { - let source_start = &pair[0]; - let source_end = &pair[1]; - if source_start.validation_time >= source_end.validation_time { - continue; - } - let peers = peer_samples_between(peer, source_start, source_end); - if peers.is_empty() { - continue; - } - for (uri, source_hash) in &source_start.objects { - if source_end.objects.get(uri) != Some(source_hash) { - continue; - } - for peer_sample in &peers { - match peer_sample.objects.get(uri) { - Some(peer_hash) if peer_hash == source_hash => {} - Some(peer_hash) => analysis.add( - "PEER_HASH_MISMATCH_STABLE_OBJECT", - sandwich_record( - "PEER_HASH_MISMATCH_STABLE_OBJECT", - "object", - uri.clone(), - source_side, - source_start, - source_end, - peer_sample, - Some(source_hash.clone()), - Some(peer_hash.clone()), - "source interval has stable object hash; peer sample has same URI with another hash", - ), - args.sample_limit, - ), - None => analysis.add( - "PEER_MISSING_STABLE_OBJECT", - sandwich_record( - "PEER_MISSING_STABLE_OBJECT", - "object", - uri.clone(), - source_side, - source_start, - source_end, - peer_sample, - Some(source_hash.clone()), - None, - "source interval has stable object hash; peer sample misses the URI", - ), - args.sample_limit, - ), - } - } - } - } -} - -fn analyze_sandwich_sets( - analysis: &mut SandwichAnalysis, - set_type: &'static str, - classification: &'static str, - source_side: Side, - source: &[SequenceSample], - peer: &[SequenceSample], - extract: F, - args: &Args, -) where - F: for<'a> Fn(&'a SequenceSample) -> &'a BTreeSet, -{ - for pair in source.windows(2) { - let source_start = &pair[0]; - let source_end = &pair[1]; - if source_start.validation_time >= source_end.validation_time { - continue; - } - let peers = peer_samples_between(peer, source_start, source_end); - if peers.is_empty() { - continue; - } - let start_set = extract(source_start); - let end_set = extract(source_end); - for key in start_set { - if !end_set.contains(key) { - continue; - } - for peer_sample in &peers { - if extract(peer_sample).contains(key) { - continue; - } - analysis.add( - classification, - sandwich_record( - classification, - set_type, - key.clone(), - source_side, - source_start, - source_end, - peer_sample, - Some(key.clone()), - None, - "source interval has a stable key; peer sample misses the key", - ), - args.sample_limit, - ); - } - } - } -} - -fn peer_samples_between<'a>( - peer: &'a [SequenceSample], - source_start: &SequenceSample, - source_end: &SequenceSample, -) -> Vec<&'a SequenceSample> { - peer.iter() - .filter(|sample| { - source_start.validation_time < sample.validation_time - && sample.validation_time < source_end.validation_time - }) - .collect() -} - -#[allow(clippy::too_many_arguments)] -fn sandwich_record( - classification: &'static str, - set_type: &'static str, - key: String, - source_side: Side, - source_start: &SequenceSample, - source_end: &SequenceSample, - peer_sample: &SequenceSample, - source_value: Option, - peer_value: Option, - note: &str, -) -> SandwichRecord { - SandwichRecord { - classification, - set_type, - key, - source_side, - source_start_seq: source_start.raw.seq, - source_start_run_id: source_start.raw.run_id.clone(), - source_end_seq: source_end.raw.seq, - source_end_run_id: source_end.raw.run_id.clone(), - peer_seq: peer_sample.raw.seq, - peer_run_id: peer_sample.raw.run_id.clone(), - source_value, - peer_value, - source_start_time: format_time(source_start.validation_time), - peer_time: format_time(peer_sample.validation_time), - source_end_time: format_time(source_end.validation_time), - note: note.to_string(), - } -} - -impl SandwichAnalysis { - fn add(&mut self, class: &'static str, record: SandwichRecord, sample_limit: usize) { - self.total_occurrences += 1; - self.unique_keys.insert(sandwich_unique_key(&record)); - *self.by_set_type.entry(record.set_type).or_default() += 1; - let stats = self.stats.entry(class).or_default(); - stats.total += 1; - stats.unique_keys.insert(sandwich_unique_key(&record)); - if stats.samples.len() < sample_limit { - stats.samples.push(record); - } - } -} - -fn sandwich_unique_key(record: &SandwichRecord) -> String { - format!( - "{}|{}|{}|{}", - record.classification, - record.set_type, - record.source_side.as_str(), - record.key - ) -} - -fn unique_event_count(events: &[DiffEvent]) -> usize { - events - .iter() - .map(event_identity) - .collect::>() - .len() -} - -fn event_identity(event: &DiffEvent) -> String { - format!( - "{}|{}|{}", - event.event_type, - event.source_side.as_str(), - event.key - ) -} - -fn samples_for_side<'a>( - side: Side, - left: &'a [SequenceSample], - right: &'a [SequenceSample], -) -> &'a [SequenceSample] { - match side { - Side::Left => left, - Side::Right => right, - } -} - -fn opposite_side(side: Side) -> Side { - match side { - Side::Left => Side::Right, - Side::Right => Side::Left, - } -} - -fn edge_position(samples: &[SequenceSample], seq: u32, args: &Args) -> EdgePosition { - let Some(index) = samples.iter().position(|sample| sample.raw.seq == seq) else { - return EdgePosition::Stable; - }; - if index < args.warmup_samples { - return EdgePosition::Leading; - } - if samples.len().saturating_sub(index) <= args.cooldown_samples { - return EdgePosition::Trailing; - } - EdgePosition::Stable -} - -fn peer_has_uri_any(peer: &[SequenceSample], uri: &str) -> bool { - peer.iter().any(|sample| sample.objects.contains_key(uri)) -} - -fn peer_has_uri_different_hash_any(peer: &[SequenceSample], uri: &str, hash: &str) -> bool { - peer.iter() - .filter_map(|sample| sample.objects.get(uri)) - .any(|peer_hash| peer_hash != hash) -} - -fn source_future_has_hash( - source: &[SequenceSample], - seq: u32, - uri: &str, - expected_hash: &str, -) -> bool { - source - .iter() - .filter(|sample| sample.raw.seq > seq) - .any(|sample| { - sample - .objects - .get(uri) - .is_some_and(|hash| hash == expected_hash) - }) -} - -fn peer_has_same_ta_identity(peer: &[SequenceSample], key: &str) -> bool { - let Some(identity) = trust_anchor_identity(key) else { - return false; - }; - peer.iter().any(|sample| { - sample - .trust_anchors - .iter() - .filter_map(|peer_key| trust_anchor_identity(peer_key)) - .any(|peer_identity| peer_identity == identity) - }) -} - -fn trust_anchor_identity(key: &str) -> Option { - let parts = key.split('|').collect::>(); - if parts.len() != 4 { - return None; - } - Some(format!("{}|{}", parts[2], parts[3])) -} - -fn split_object_hash_key(key: &str) -> Option<(&str, &str)> { - key.rsplit_once('|') -} - -fn build_uri_timeline_samples( - left: &[SequenceSample], - right: &[SequenceSample], - adjusted: &AdjustedAnalysis, - limit: usize, -) -> Vec { - let mut uris = BTreeSet::new(); - for stats in adjusted.stats.values() { - for sample in &stats.samples { - if sample.event_type == "object_hash" - && let Some((uri, _)) = split_object_hash_key(&sample.key) - { - uris.insert(uri.to_string()); - } - if sample.event_type == "object_uri" { - uris.insert(sample.key.clone()); - } - } - } - uris.into_iter() - .take(limit) - .map(|uri| { - json!({ - "uri": uri, - "left": timeline_for_uri(left, &uri), - "right": timeline_for_uri(right, &uri), - }) - }) - .collect() -} - -fn timeline_for_uri(samples: &[SequenceSample], uri: &str) -> Vec { - samples - .iter() - .filter_map(|sample| { - sample.objects.get(uri).map(|hash| { - json!({ - "seq": sample.raw.seq, - "runId": sample.raw.run_id, - "validationTime": format_time(sample.validation_time), - "hash": hash, - }) - }) - }) - .collect() -} - -fn build_stable_object_groups( - adjusted: &AdjustedAnalysis, - left: &[SequenceSample], - right: &[SequenceSample], - limit: usize, -) -> Vec { - let mut groups: BTreeMap = BTreeMap::new(); - let Some(stats) = adjusted.stats.get("STABLE_OBJECT_SET_DIVERGENCE") else { - return Vec::new(); - }; - for sample in &stats.samples { - let physical_uri = physical_object_uri(sample); - let group_key = format!( - "{}|{}|{}|{}", - sample.source_side.as_str(), - sample.source_seq, - sample.source_run_id, - publication_point_prefix(&physical_uri) - ); - let group = groups - .entry(group_key) - .or_insert_with(|| StableObjectGroup::new(sample, &physical_uri)); - if group.source_cir_path.is_none() - && let Some(source_sample) = sample_by_side_seq_run( - left, - right, - sample.source_side, - sample.source_seq, - &sample.source_run_id, - ) - { - group.source_cir_path = Some(path_string(&source_sample.cir_path)); - } - group.add(sample, physical_uri); - } - groups - .into_values() - .take(limit) - .map(StableObjectGroup::to_json) - .collect() -} - -#[derive(Clone, Debug)] -struct StableObjectGroup { - source_side: Side, - source_seq: u32, - source_run_id: String, - source_cir_path: Option, - publication_point: String, - event_count: usize, - event_types: BTreeMap<&'static str, usize>, - physical_objects: BTreeMap, -} - -impl StableObjectGroup { - fn new(sample: &AdjustedRecord, physical_uri: &str) -> Self { - Self { - source_side: sample.source_side, - source_seq: sample.source_seq, - source_run_id: sample.source_run_id.clone(), - source_cir_path: None, - publication_point: publication_point_prefix(physical_uri), - event_count: 0, - event_types: BTreeMap::new(), - physical_objects: BTreeMap::new(), - } - } - - fn add(&mut self, sample: &AdjustedRecord, physical_uri: String) { - self.event_count += 1; - *self.event_types.entry(sample.event_type).or_default() += 1; - let object = self - .physical_objects - .entry(physical_uri.clone()) - .or_insert_with(|| StablePhysicalObject { - extension: object_extension(&physical_uri).to_string(), - uri: physical_uri, - event_types: BTreeSet::new(), - hashes: BTreeSet::new(), - }); - object.event_types.insert(sample.event_type); - if let Some(hash) = event_hash(sample) { - object.hashes.insert(hash.to_string()); - } - } - - fn to_json(self) -> Value { - json!({ - "sourceSide": self.source_side.as_str(), - "sourceSeq": self.source_seq, - "sourceRunId": self.source_run_id, - "sourceCirPath": self.source_cir_path, - "publicationPoint": self.publication_point, - "eventCount": self.event_count, - "eventTypes": self.event_types, - "physicalObjectCount": self.physical_objects.len(), - "physicalObjects": self.physical_objects.into_values().map(StablePhysicalObject::to_json).collect::>(), - }) - } -} - -#[derive(Clone, Debug)] -struct StablePhysicalObject { - uri: String, - extension: String, - event_types: BTreeSet<&'static str>, - hashes: BTreeSet, -} - -impl StablePhysicalObject { - fn to_json(self) -> Value { - json!({ - "uri": self.uri, - "extension": self.extension, - "eventTypes": self.event_types, - "hashes": self.hashes, - }) - } -} - -fn physical_object_uri(sample: &AdjustedRecord) -> String { - if sample.event_type == "object_hash" - && let Some((uri, _)) = split_object_hash_key(&sample.key) - { - return uri.to_string(); - } - sample.key.clone() -} - -fn event_hash(sample: &AdjustedRecord) -> Option<&str> { - if sample.event_type == "object_hash" { - split_object_hash_key(&sample.key).map(|(_, hash)| hash) - } else { - None - } -} - -fn publication_point_prefix(uri: &str) -> String { - uri.rsplit_once('/') - .map(|(prefix, _)| format!("{prefix}/")) - .unwrap_or_else(|| uri.to_string()) -} - -fn object_extension(uri: &str) -> &str { - uri.rsplit_once('.') - .map(|(_, extension)| extension) - .unwrap_or("") -} - -fn sample_by_side_seq_run<'a>( - left: &'a [SequenceSample], - right: &'a [SequenceSample], - side: Side, - seq: u32, - run_id: &str, -) -> Option<&'a SequenceSample> { - samples_for_side(side, left, right) - .iter() - .find(|sample| sample.raw.seq == seq && sample.raw.run_id == run_id) -} - -impl AnalysisResult { - fn add(&mut self, class: &'static str, record: SampleRecord, sample_limit: usize) { - let stats = self.stats.entry(class).or_default(); - stats.total += 1; - if stats.samples.len() < sample_limit { - stats.samples.push(record); - } - } -} - -fn peer_sample_at_seq(peer: &[SequenceSample], seq: u32) -> Option<&SequenceSample> { - peer.iter().find(|sample| sample.raw.seq == seq) -} - -fn find_future_match( - peer: &[SequenceSample], - source: &SequenceSample, - key: &str, - extract: &F, - args: &Args, -) -> Option -where - F: for<'a> Fn(&'a SequenceSample) -> &'a BTreeSet, -{ - peer.iter() - .filter(|candidate| is_in_alignment_window(source, candidate, args)) - .find(|candidate| extract(candidate).contains(key)) - .map(|candidate| { - occurrence( - candidate, - if candidate.raw.side.as_deref() == Some("left") { - Side::Left - } else { - Side::Right - }, - ) - }) -} - -fn find_future_hash_match( - peer: &[SequenceSample], - source: &SequenceSample, - uri: &str, - hash: &str, - args: &Args, -) -> Option { - peer.iter() - .filter(|candidate| is_in_alignment_window(source, candidate, args)) - .find(|candidate| { - candidate - .objects - .get(uri) - .is_some_and(|peer_hash| peer_hash == hash) - }) - .map(|candidate| { - occurrence( - candidate, - if candidate.raw.side.as_deref() == Some("left") { - Side::Left - } else { - Side::Right - }, - ) - }) -} - -fn is_in_alignment_window( - source: &SequenceSample, - candidate: &SequenceSample, - args: &Args, -) -> bool { - if candidate.raw.seq < source.raw.seq { - return false; - } - let run_delta = candidate.raw.seq.saturating_sub(source.raw.seq); - let time_delta = candidate.validation_time - source.validation_time; - let secs = time_delta.whole_seconds().abs(); - run_delta <= args.align_window_runs || secs <= args.align_window_secs -} - -fn occurrence(sample: &SequenceSample, side: Side) -> EventOccurrence { - EventOccurrence { - side, - seq: sample.raw.seq, - run_id: sample.raw.run_id.clone(), - } -} - -fn build_output( - args: &Args, - left: &[SequenceSample], - right: &[SequenceSample], - result: &AnalysisResult, - adjusted: &AdjustedAnalysis, - sandwich: &SandwichAnalysis, -) -> Value { - let classifications = result - .stats - .iter() - .map(|(class, stats)| { - json!({ - "classification": class, - "count": stats.total, - "samples": stats.samples.iter().map(sample_to_json).collect::>(), - }) - }) - .collect::>(); - let adjusted_classifications = adjusted - .stats - .iter() - .map(|(class, stats)| { - json!({ - "classification": class, - "occurrences": stats.total, - "uniqueKeys": stats.unique_keys.len(), - "samples": stats.samples.iter().map(adjusted_sample_to_json).collect::>(), - }) - }) - .collect::>(); - let sandwich_classifications = sandwich - .stats - .iter() - .map(|(class, stats)| { - json!({ - "classification": class, - "occurrences": stats.total, - "uniqueKeys": stats.unique_keys.len(), - "samples": stats.samples.iter().map(sandwich_sample_to_json).collect::>(), - }) - }) - .collect::>(); - json!({ - "schemaVersion": 1, - "generatedBy": "sequence_triage_ccr_cir", - "inputContract": "left-right-sequence-jsonl-with-ccr-cir-artifacts", - "parameters": { - "leftSequence": path_string(&args.left_sequence), - "rightSequence": path_string(&args.right_sequence), - "alignWindowRuns": args.align_window_runs, - "alignWindowSecs": args.align_window_secs, - "sampleLimit": args.sample_limit, - "warmupSamples": args.warmup_samples, - "cooldownSamples": args.cooldown_samples, - "timelineSampleLimit": if args.timeline_sample_limit == 0 { args.sample_limit } else { args.timeline_sample_limit }, - }, - "left": sequence_summary(left), - "right": sequence_summary(right), - "classificationCounts": classifications, - "totals": { - "resolvedTemporalLike": count_class(result, "TEMPORAL_LAG_RESOLVED") + count_class(result, "CONTENT_ROLLOVER_RESOLVED"), - "persistent": count_prefix(result, "PERSISTENT_"), - "unclassified": count_class(result, "UNCLASSIFIED_INSUFFICIENT_WINDOW"), - }, - "adjusted": { - "warmupSamples": args.warmup_samples, - "cooldownSamples": args.cooldown_samples, - "rawPersistent": { - "occurrences": adjusted.raw_persistent_occurrences, - "uniqueKeys": adjusted.raw_persistent_unique_keys, - }, - "edgeFilteredPersistent": { - "occurrences": adjusted.edge_filtered_occurrences, - "uniqueKeys": adjusted.edge_filtered_unique_keys, - }, - "adjustedStablePersistent": { - "occurrences": adjusted.adjusted_stable_occurrences, - "uniqueKeys": adjusted.adjusted_stable_unique_keys, - }, - "classificationCounts": adjusted_classifications, - "uriTimelineSamples": adjusted.uri_timeline_samples, - "stableObjectGroups": adjusted.stable_object_groups, - "interpretation": { - "rawPersistentMeaning": "raw persistent keeps the original #043 single-event matching semantics.", - "edgeFilteredMeaning": "edge-filtered persistent keeps only non-warmup and non-cooldown source occurrences.", - "adjustedStableMeaning": "adjusted stable persistent keeps non-edge findings after URI-level rollover and TA projection filtering.", - "stableObjectGroupsMeaning": "STABLE_OBJECT_SET_DIVERGENCE events are additionally collapsed by source CIR, publication point, and physical object URI so object_uri/object_hash duplicate event views do not inflate physical-object counts.", - } - }, - "sandwich": { - "strictTimeWindow": true, - "method": "For each side, use two adjacent source samples as a stable interval. If source_start.time < peer.time < source_end.time and the source value is identical at both interval endpoints, the peer sample is expected to contain the same value.", - "totals": { - "occurrences": sandwich.total_occurrences, - "uniqueKeys": sandwich.unique_keys.len(), - }, - "bySetType": sandwich.by_set_type, - "classificationCounts": sandwich_classifications, - "interpretation": { - "missingStableObject": "The source side proves a URI/hash is stable across an interval that contains the peer sample, but the peer sample has no such URI.", - "hashMismatchStableObject": "The source side proves a URI/hash is stable across an interval that contains the peer sample, but the peer sample has the same URI with a different hash.", - "missingStableReject": "The source side consistently rejects the URI across an interval that contains the peer sample, but the peer sample does not reject it.", - "missingStableOutput": "The source side consistently outputs a VRP/VAP key across an interval that contains the peer sample, but the peer sample does not output it." - } - }, - "interpretation": { - "temporalResolvedMeaning": "Event appeared only on one side at one sample but aligned in the peer sequence later.", - "persistentMeaning": "Event did not align within the configured run/time window; inspect as a candidate RP behavior difference or persistent sync/input difference.", - "limits": [ - "Sequence triage only reads sequence JSONL plus referenced CCR/CIR files.", - "It does not read report.json, logs, repo-bytes DB, cache, mirror, or raw objects for root cause proof.", - "Resolved temporal findings reduce false positives but do not prove the exact external repository update time." - ] - } - }) -} - -fn sequence_summary(samples: &[SequenceSample]) -> Value { - json!({ - "sampleCount": samples.len(), - "rpIds": samples.iter().map(|sample| sample.raw.rp_id.clone()).collect::>(), - "firstSeq": samples.first().map(|sample| sample.raw.seq), - "lastSeq": samples.last().map(|sample| sample.raw.seq), - "firstValidationTime": samples.first().map(|sample| format_time(sample.validation_time)), - "lastValidationTime": samples.last().map(|sample| format_time(sample.validation_time)), - "samples": samples.iter().map(|sample| json!({ - "seq": sample.raw.seq, - "runId": sample.raw.run_id, - "syncMode": sample.raw.sync_mode, - "startTime": sample.raw.start_time, - "finishTime": sample.raw.finish_time, - "validationTime": format_time(sample.validation_time), - "status": sample.raw.status, - "ccrPath": path_string(&sample.ccr_path), - "cirPath": path_string(&sample.cir_path), - "ccrSha256": sample.raw.ccr_sha256, - "cirSha256": sample.raw.cir_sha256, - "wallMs": sample.raw.wall_ms, - "maxRssKb": sample.raw.max_rss_kb, - "vrps": sample.raw.vrps.or(Some(sample.vrps.len() as u64)), - "vaps": sample.raw.vaps.or(Some(sample.vaps.len() as u64)), - "objectCount": sample.object_uris.len(), - "rejectCount": sample.rejects.len(), - "trustAnchorCount": sample.trust_anchors.len(), - })).collect::>(), - }) -} - -fn sample_to_json(sample: &SampleRecord) -> Value { - json!({ - "classification": sample.classification, - "eventType": sample.event_type, - "key": sample.key, - "sourceSide": sample.source_side.as_str(), - "sourceSeq": sample.source_seq, - "sourceRunId": sample.source_run_id, - "matchedSeq": sample.matched_seq, - "matchedRunId": sample.matched_run_id, - "note": sample.note, - }) -} - -fn adjusted_sample_to_json(sample: &AdjustedRecord) -> Value { - json!({ - "classification": sample.classification, - "eventType": sample.event_type, - "key": sample.key, - "sourceSide": sample.source_side.as_str(), - "sourceSeq": sample.source_seq, - "sourceRunId": sample.source_run_id, - "note": sample.note, - }) -} - -fn sandwich_sample_to_json(sample: &SandwichRecord) -> Value { - json!({ - "classification": sample.classification, - "setType": sample.set_type, - "key": sample.key, - "sourceSide": sample.source_side.as_str(), - "sourceStartSeq": sample.source_start_seq, - "sourceStartRunId": sample.source_start_run_id, - "sourceEndSeq": sample.source_end_seq, - "sourceEndRunId": sample.source_end_run_id, - "peerSeq": sample.peer_seq, - "peerRunId": sample.peer_run_id, - "sourceValue": sample.source_value, - "peerValue": sample.peer_value, - "sourceStartTime": sample.source_start_time, - "peerTime": sample.peer_time, - "sourceEndTime": sample.source_end_time, - "note": sample.note, - }) -} - -fn count_class(result: &AnalysisResult, class: &'static str) -> usize { - result - .stats - .get(class) - .map(|stats| stats.total) - .unwrap_or(0) -} - -fn count_prefix(result: &AnalysisResult, prefix: &str) -> usize { - result - .stats - .iter() - .filter(|(class, _)| class.starts_with(prefix)) - .map(|(_, stats)| stats.total) - .sum() -} - -fn write_json(path: &Path, value: &Value) -> Result<(), String> { - std::fs::write( - path, - serde_json::to_string_pretty(value).map_err(|e| e.to_string())? + "\n", - ) - .map_err(|e| format!("write JSON failed: {}: {e}", path.display())) -} - -fn write_markdown(path: &Path, output: &Value) -> Result<(), String> { - let mut lines = vec![ - "# CCR/CIR Sequence Triage Summary".to_string(), - "".to_string(), - format!( - "- `generatedBy`: `{}`", - output["generatedBy"].as_str().unwrap_or("") - ), - format!( - "- `leftSamples`: `{}`", - output["left"]["sampleCount"].as_u64().unwrap_or(0) - ), - format!( - "- `rightSamples`: `{}`", - output["right"]["sampleCount"].as_u64().unwrap_or(0) - ), - format!( - "- `alignWindowRuns`: `{}`", - output["parameters"]["alignWindowRuns"] - .as_u64() - .unwrap_or(0) - ), - format!( - "- `alignWindowSecs`: `{}`", - output["parameters"]["alignWindowSecs"] - .as_i64() - .unwrap_or(0) - ), - format!( - "- `warmupSamples`: `{}`", - output["adjusted"]["warmupSamples"].as_u64().unwrap_or(0) - ), - format!( - "- `cooldownSamples`: `{}`", - output["adjusted"]["cooldownSamples"].as_u64().unwrap_or(0) - ), - "".to_string(), - "## Classification Counts".to_string(), - "".to_string(), - "| Classification | Count |".to_string(), - "|---|---:|".to_string(), - ]; - if let Some(classes) = output["classificationCounts"].as_array() { - for item in classes { - lines.push(format!( - "| `{}` | {} |", - item["classification"].as_str().unwrap_or(""), - item["count"].as_u64().unwrap_or(0) - )); - } - } - lines.extend([ - "".to_string(), - "## Adjusted Boundary / Rollover Classification".to_string(), - "".to_string(), - format!( - "- `rawPersistent`: `{}` occurrences / `{}` unique keys", - output["adjusted"]["rawPersistent"]["occurrences"] - .as_u64() - .unwrap_or(0), - output["adjusted"]["rawPersistent"]["uniqueKeys"] - .as_u64() - .unwrap_or(0) - ), - format!( - "- `edgeFilteredPersistent`: `{}` occurrences / `{}` unique keys", - output["adjusted"]["edgeFilteredPersistent"]["occurrences"] - .as_u64() - .unwrap_or(0), - output["adjusted"]["edgeFilteredPersistent"]["uniqueKeys"] - .as_u64() - .unwrap_or(0) - ), - format!( - "- `adjustedStablePersistent`: `{}` occurrences / `{}` unique keys", - output["adjusted"]["adjustedStablePersistent"]["occurrences"] - .as_u64() - .unwrap_or(0), - output["adjusted"]["adjustedStablePersistent"]["uniqueKeys"] - .as_u64() - .unwrap_or(0) - ), - "".to_string(), - "| Adjusted Classification | Occurrences | Unique Keys |".to_string(), - "|---|---:|---:|".to_string(), - ]); - if let Some(classes) = output["adjusted"]["classificationCounts"].as_array() { - for item in classes { - lines.push(format!( - "| `{}` | {} | {} |", - item["classification"].as_str().unwrap_or(""), - item["occurrences"].as_u64().unwrap_or(0), - item["uniqueKeys"].as_u64().unwrap_or(0) - )); - } - } - if let Some(groups) = output["adjusted"]["stableObjectGroups"].as_array() - && !groups.is_empty() - { - lines.extend([ - "".to_string(), - "## Stable Object Groups".to_string(), - "".to_string(), - "| Source | CIR | Publication Point | Events | Physical Objects |".to_string(), - "|---|---|---|---:|---:|".to_string(), - ]); - for group in groups { - lines.push(format!( - "| `{}/seq{}/{}` | `{}` | `{}` | {} | {} |", - group["sourceSide"].as_str().unwrap_or(""), - group["sourceSeq"].as_u64().unwrap_or(0), - group["sourceRunId"].as_str().unwrap_or(""), - group["sourceCirPath"].as_str().unwrap_or(""), - group["publicationPoint"].as_str().unwrap_or(""), - group["eventCount"].as_u64().unwrap_or(0), - group["physicalObjectCount"].as_u64().unwrap_or(0), - )); - } - } - lines.extend([ - "".to_string(), - "## Sandwich Anomaly Check".to_string(), - "".to_string(), - format!( - "- `occurrences`: `{}`", - output["sandwich"]["totals"]["occurrences"] - .as_u64() - .unwrap_or(0) - ), - format!( - "- `uniqueKeys`: `{}`", - output["sandwich"]["totals"]["uniqueKeys"] - .as_u64() - .unwrap_or(0) - ), - "- Rule: source side has two adjacent stable samples, and peer sample timestamp is strictly between them.".to_string(), - "".to_string(), - "| Sandwich Classification | Occurrences | Unique Keys |".to_string(), - "|---|---:|---:|".to_string(), - ]); - if let Some(classes) = output["sandwich"]["classificationCounts"].as_array() { - for item in classes { - lines.push(format!( - "| `{}` | {} | {} |", - item["classification"].as_str().unwrap_or(""), - item["occurrences"].as_u64().unwrap_or(0), - item["uniqueKeys"].as_u64().unwrap_or(0) - )); - } - } - lines.extend([ - "".to_string(), - "## Interpretation".to_string(), - "".to_string(), - "- `TEMPORAL_LAG_RESOLVED` / `CONTENT_ROLLOVER_RESOLVED` 表示差异在后续采样中对齐，优先视为采样时刻或仓库滚动窗口差异。".to_string(), - "- `PERSISTENT_*` 表示配置窗口内仍未对齐，才是后续人工排查的高价值候选。".to_string(), - "- `EDGE_*` 表示差异落在序列首尾，缺少前置或后续观察，不能直接当作实现差异。".to_string(), - "- `STABLE_*` 表示经过首尾边界过滤和 URI-level rollover 过滤后仍存在的稳定候选。".to_string(), - ]); - std::fs::write(path, lines.join("\n") + "\n") - .map_err(|e| format!("write markdown failed: {}: {e}", path.display())) -} - -fn write_samples_jsonl(path: &Path, result: &AnalysisResult) -> Result<(), String> { - let mut body = String::new(); - for stats in result.stats.values() { - for sample in &stats.samples { - body.push_str( - &serde_json::to_string(&sample_to_json(sample)).map_err(|e| e.to_string())?, - ); - body.push('\n'); - } - } - std::fs::write(path, body).map_err(|e| format!("write samples failed: {}: {e}", path.display())) -} - -fn read_file(path: &Path) -> Result, String> { - std::fs::read(path).map_err(|e| format!("read file failed: {}: {e}", path.display())) -} - -fn resolve_path(base_dir: &Path, path: &Path) -> PathBuf { - if path.is_absolute() { - path.to_path_buf() - } else { - base_dir.join(path) - } -} - -fn parse_rfc3339(value: &str) -> Result { - OffsetDateTime::parse(value, &Rfc3339) - .map_err(|e| format!("parse RFC3339 failed: {value}: {e}")) -} - -fn format_time(value: OffsetDateTime) -> String { - value.format(&Rfc3339).unwrap_or_else(|_| value.to_string()) -} - -fn path_string(path: &Path) -> String { - path.to_string_lossy().into_owned() -} - -fn object_hash_key(uri: &str, hash: &str) -> String { - format!("{uri}|{hash}") -} - -#[cfg(test)] -mod tests { - use super::*; - use rpki::ccr::{ - CcrContentInfo, CcrDigestAlgorithm, RpkiCanonicalCacheRepresentation, - build_aspa_payload_state, build_roa_payload_state, encode_content_info, - }; - use rpki::cir::{ - CIR_VERSION_V3, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, - CirRejectedObject, CirTrustAnchor, compute_reject_list_sha256, encode_cir, sha256, - }; - use rpki::data_model::roa::{IpPrefix, RoaAfi}; - use rpki::validation::objects::{AspaAttestation, Vrp}; - - #[test] - fn parse_args_accepts_required_flags() { - let argv = vec![ - "sequence_triage_ccr_cir".to_string(), - "--left-sequence".to_string(), - "left.jsonl".to_string(), - "--right-sequence".to_string(), - "right.jsonl".to_string(), - "--out-dir".to_string(), - "out".to_string(), - "--align-window-runs".to_string(), - "3".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!(args.align_window_runs, 3); - assert_eq!(args.align_window_secs, 1800); - assert_eq!(args.warmup_samples, 1); - assert_eq!(args.cooldown_samples, 1); - assert_eq!(args.timeline_sample_limit, 0); - } - - #[test] - fn run_classifies_temporal_and_persistent_differences() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path(); - write_sample( - root, - "left1", - "left", - 1, - &[object("rsync://example.net/a.roa", 0x11)], - &[], - 64496, - ); - write_sample( - root, - "left2", - "left", - 2, - &[ - object("rsync://example.net/a.roa", 0x11), - object("rsync://example.net/persistent.roa", 0x44), - ], - &[], - 64496, - ); - write_sample(root, "right1", "right", 1, &[], &[], 64497); - write_sample( - root, - "right2", - "right", - 2, - &[object("rsync://example.net/a.roa", 0x11)], - &[], - 64497, - ); - std::fs::write( - root.join("left.jsonl"), - jsonl(&[ - item("left", 1, "left1/result.ccr", "left1/result.cir"), - item("left", 2, "left2/result.ccr", "left2/result.cir"), - ]), - ) - .unwrap(); - std::fs::write( - root.join("right.jsonl"), - jsonl(&[ - item("right", 1, "right1/result.ccr", "right1/result.cir"), - item("right", 2, "right2/result.ccr", "right2/result.cir"), - ]), - ) - .unwrap(); - run(Args { - left_sequence: root.join("left.jsonl"), - right_sequence: root.join("right.jsonl"), - out_dir: root.join("out"), - align_window_runs: 2, - align_window_secs: 3600, - sample_limit: 20, - warmup_samples: 1, - cooldown_samples: 0, - timeline_sample_limit: 0, - }) - .expect("run"); - let output: Value = serde_json::from_str( - &std::fs::read_to_string(root.join("out/sequence-triage.json")).unwrap(), - ) - .unwrap(); - assert!(class_count(&output, "TEMPORAL_LAG_RESOLVED") > 0); - assert!(class_count(&output, "PERSISTENT_OBJECT_SET_DIVERGENCE") > 0); - } - - #[test] - fn run_classifies_reject_and_output_divergence() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path(); - let objects = [object("rsync://example.net/a.roa", 0x11)]; - write_sample( - root, - "left1", - "left", - 1, - &objects, - &["rsync://example.net/a.roa"], - 64496, - ); - write_sample(root, "right1", "right", 1, &objects, &[], 64497); - std::fs::write( - root.join("left.jsonl"), - jsonl(&[item("left", 1, "left1/result.ccr", "left1/result.cir")]), - ) - .unwrap(); - std::fs::write( - root.join("right.jsonl"), - jsonl(&[item("right", 1, "right1/result.ccr", "right1/result.cir")]), - ) - .unwrap(); - run(Args { - left_sequence: root.join("left.jsonl"), - right_sequence: root.join("right.jsonl"), - out_dir: root.join("out"), - align_window_runs: 0, - align_window_secs: 0, - sample_limit: 20, - warmup_samples: 0, - cooldown_samples: 0, - timeline_sample_limit: 0, - }) - .expect("run"); - let output: Value = serde_json::from_str( - &std::fs::read_to_string(root.join("out/sequence-triage.json")).unwrap(), - ) - .unwrap(); - assert!(class_count(&output, "PERSISTENT_REJECT_DIVERGENCE") > 0); - assert!(class_count(&output, "PERSISTENT_OUTPUT_DIVERGENCE") > 0); - } - - #[test] - fn run_adjusted_classifies_leading_content_rollover() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path(); - let uri = "rsync://example.net/a.roa"; - write_sample(root, "left1", "left", 1, &[object(uri, 0x11)], &[], 64496); - write_sample(root, "left2", "left", 2, &[object(uri, 0x22)], &[], 64496); - write_sample(root, "right1", "right", 1, &[object(uri, 0x22)], &[], 64496); - write_sample(root, "right2", "right", 2, &[object(uri, 0x22)], &[], 64496); - std::fs::write( - root.join("left.jsonl"), - jsonl(&[ - item("left", 1, "left1/result.ccr", "left1/result.cir"), - item("left", 2, "left2/result.ccr", "left2/result.cir"), - ]), - ) - .unwrap(); - std::fs::write( - root.join("right.jsonl"), - jsonl(&[ - item("right", 1, "right1/result.ccr", "right1/result.cir"), - item("right", 2, "right2/result.ccr", "right2/result.cir"), - ]), - ) - .unwrap(); - run(Args { - left_sequence: root.join("left.jsonl"), - right_sequence: root.join("right.jsonl"), - out_dir: root.join("out"), - align_window_runs: 0, - align_window_secs: 0, - sample_limit: 20, - warmup_samples: 1, - cooldown_samples: 0, - timeline_sample_limit: 0, - }) - .expect("run"); - let output: Value = serde_json::from_str( - &std::fs::read_to_string(root.join("out/sequence-triage.json")).unwrap(), - ) - .unwrap(); - assert!(adjusted_class_occurrences(&output, "EDGE_LEADING_CONTENT_ROLLOVER") > 0); - assert_eq!( - adjusted_class_occurrences(&output, "STABLE_CONTENT_DIVERGENCE"), - 0 - ); - assert_eq!( - output["adjusted"]["adjustedStablePersistent"]["occurrences"] - .as_u64() - .unwrap(), - 0 - ); - } - - #[test] - fn run_adjusted_classifies_stable_middle_content_divergence() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path(); - let uri = "rsync://example.net/a.roa"; - for seq in 1..=3 { - write_sample( - root, - &format!("left{seq}"), - "left", - seq, - &[object(uri, 0x11)], - &[], - 64496, - ); - write_sample( - root, - &format!("right{seq}"), - "right", - seq, - &[object(uri, 0x22)], - &[], - 64496, - ); - } - std::fs::write( - root.join("left.jsonl"), - jsonl(&[ - item("left", 1, "left1/result.ccr", "left1/result.cir"), - item("left", 2, "left2/result.ccr", "left2/result.cir"), - item("left", 3, "left3/result.ccr", "left3/result.cir"), - ]), - ) - .unwrap(); - std::fs::write( - root.join("right.jsonl"), - jsonl(&[ - item("right", 1, "right1/result.ccr", "right1/result.cir"), - item("right", 2, "right2/result.ccr", "right2/result.cir"), - item("right", 3, "right3/result.ccr", "right3/result.cir"), - ]), - ) - .unwrap(); - run(Args { - left_sequence: root.join("left.jsonl"), - right_sequence: root.join("right.jsonl"), - out_dir: root.join("out"), - align_window_runs: 0, - align_window_secs: 0, - sample_limit: 20, - warmup_samples: 1, - cooldown_samples: 1, - timeline_sample_limit: 0, - }) - .expect("run"); - let output: Value = serde_json::from_str( - &std::fs::read_to_string(root.join("out/sequence-triage.json")).unwrap(), - ) - .unwrap(); - assert!(adjusted_class_occurrences(&output, "STABLE_CONTENT_DIVERGENCE") > 0); - assert_eq!( - output["adjusted"]["adjustedStablePersistent"]["occurrences"] - .as_u64() - .unwrap(), - 2 - ); - } - - #[test] - fn run_adjusted_filters_trailing_output() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path(); - let objects = [object("rsync://example.net/a.roa", 0x11)]; - write_sample(root, "left1", "left", 1, &objects, &[], 64496); - write_sample(root, "left2", "left", 2, &objects, &[], 64497); - write_sample(root, "right1", "right", 1, &objects, &[], 64496); - write_sample(root, "right2", "right", 2, &objects, &[], 64496); - std::fs::write( - root.join("left.jsonl"), - jsonl(&[ - item("left", 1, "left1/result.ccr", "left1/result.cir"), - item("left", 2, "left2/result.ccr", "left2/result.cir"), - ]), - ) - .unwrap(); - std::fs::write( - root.join("right.jsonl"), - jsonl(&[ - item("right", 1, "right1/result.ccr", "right1/result.cir"), - item("right", 2, "right2/result.ccr", "right2/result.cir"), - ]), - ) - .unwrap(); - run(Args { - left_sequence: root.join("left.jsonl"), - right_sequence: root.join("right.jsonl"), - out_dir: root.join("out"), - align_window_runs: 0, - align_window_secs: 0, - sample_limit: 20, - warmup_samples: 0, - cooldown_samples: 1, - timeline_sample_limit: 0, - }) - .expect("run"); - let output: Value = serde_json::from_str( - &std::fs::read_to_string(root.join("out/sequence-triage.json")).unwrap(), - ) - .unwrap(); - assert!(adjusted_class_occurrences(&output, "EDGE_TRAILING_UNRESOLVED") > 0); - assert_eq!( - adjusted_class_occurrences(&output, "STABLE_OUTPUT_DIVERGENCE"), - 0 - ); - } - - #[test] - fn run_groups_stable_object_events_by_physical_object() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path(); - let missing = "rsync://example.net/repo/pp/a.roa"; - write_sample( - root, - "left1", - "left", - 1, - &[object("rsync://example.net/repo/pp/base.roa", 0x10)], - &[], - 64496, - ); - write_sample( - root, - "left2", - "left", - 2, - &[ - object("rsync://example.net/repo/pp/base.roa", 0x10), - object(missing, 0x11), - ], - &[], - 64496, - ); - write_sample( - root, - "left3", - "left", - 3, - &[object("rsync://example.net/repo/pp/base.roa", 0x10)], - &[], - 64496, - ); - for seq in 1..=3 { - write_sample( - root, - &format!("right{seq}"), - "right", - seq, - &[object("rsync://example.net/repo/pp/base.roa", 0x10)], - &[], - 64496, - ); - } - std::fs::write( - root.join("left.jsonl"), - jsonl(&[ - item("left", 1, "left1/result.ccr", "left1/result.cir"), - item("left", 2, "left2/result.ccr", "left2/result.cir"), - item("left", 3, "left3/result.ccr", "left3/result.cir"), - ]), - ) - .unwrap(); - std::fs::write( - root.join("right.jsonl"), - jsonl(&[ - item("right", 1, "right1/result.ccr", "right1/result.cir"), - item("right", 2, "right2/result.ccr", "right2/result.cir"), - item("right", 3, "right3/result.ccr", "right3/result.cir"), - ]), - ) - .unwrap(); - run(Args { - left_sequence: root.join("left.jsonl"), - right_sequence: root.join("right.jsonl"), - out_dir: root.join("out"), - align_window_runs: 0, - align_window_secs: 0, - sample_limit: 20, - warmup_samples: 1, - cooldown_samples: 1, - timeline_sample_limit: 0, - }) - .expect("run"); - let output: Value = serde_json::from_str( - &std::fs::read_to_string(root.join("out/sequence-triage.json")).unwrap(), - ) - .unwrap(); - let groups = output["adjusted"]["stableObjectGroups"].as_array().unwrap(); - assert_eq!(groups.len(), 1); - assert_eq!(groups[0]["eventCount"].as_u64(), Some(2)); - assert_eq!(groups[0]["physicalObjectCount"].as_u64(), Some(1)); - assert_eq!( - groups[0]["publicationPoint"].as_str(), - Some("rsync://example.net/repo/pp/") - ); - assert_eq!( - groups[0]["physicalObjects"][0]["eventTypes"] - .as_array() - .unwrap() - .len(), - 2 - ); - } - - #[test] - fn run_sandwich_detects_object_hash_reject_and_output_anomalies() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path(); - let stable_missing = object("rsync://example.net/pp/missing.roa", 0x11); - let stable_mismatch = object("rsync://example.net/pp/mismatch.roa", 0x22); - let peer_mismatch = object("rsync://example.net/pp/mismatch.roa", 0x33); - write_sample_with_ccr_seq( - root, - "left1", - 1, - &[stable_missing.clone(), stable_mismatch.clone()], - &["rsync://example.net/pp/rejected.roa"], - 9, - 64496, - ); - write_sample_with_ccr_seq( - root, - "left3", - 3, - &[stable_missing, stable_mismatch], - &["rsync://example.net/pp/rejected.roa"], - 9, - 64496, - ); - write_sample_with_ccr_seq(root, "right2", 2, &[peer_mismatch], &[], 10, 64497); - std::fs::write( - root.join("left.jsonl"), - jsonl(&[ - item("left", 1, "left1/result.ccr", "left1/result.cir"), - item("left", 3, "left3/result.ccr", "left3/result.cir"), - ]), - ) - .unwrap(); - std::fs::write( - root.join("right.jsonl"), - jsonl(&[item("right", 2, "right2/result.ccr", "right2/result.cir")]), - ) - .unwrap(); - run(Args { - left_sequence: root.join("left.jsonl"), - right_sequence: root.join("right.jsonl"), - out_dir: root.join("out"), - align_window_runs: 0, - align_window_secs: 0, - sample_limit: 20, - warmup_samples: 0, - cooldown_samples: 0, - timeline_sample_limit: 0, - }) - .expect("run"); - let output: Value = serde_json::from_str( - &std::fs::read_to_string(root.join("out/sequence-triage.json")).unwrap(), - ) - .unwrap(); - assert_eq!( - sandwich_class_occurrences(&output, "PEER_MISSING_STABLE_OBJECT"), - 1 - ); - assert_eq!( - sandwich_class_occurrences(&output, "PEER_HASH_MISMATCH_STABLE_OBJECT"), - 1 - ); - assert_eq!( - sandwich_class_occurrences(&output, "PEER_MISSING_STABLE_REJECT"), - 1 - ); - assert_eq!( - sandwich_class_occurrences(&output, "PEER_MISSING_STABLE_OUTPUT"), - 2 - ); - } - - fn class_count(output: &Value, class: &str) -> u64 { - output["classificationCounts"] - .as_array() - .unwrap() - .iter() - .find(|item| item["classification"].as_str() == Some(class)) - .and_then(|item| item["count"].as_u64()) - .unwrap_or(0) - } - - fn sandwich_class_occurrences(output: &Value, class: &str) -> u64 { - output["sandwich"]["classificationCounts"] - .as_array() - .unwrap() - .iter() - .find(|item| item["classification"].as_str() == Some(class)) - .and_then(|item| item["occurrences"].as_u64()) - .unwrap_or(0) - } - - fn adjusted_class_occurrences(output: &Value, class: &str) -> u64 { - output["adjusted"]["classificationCounts"] - .as_array() - .unwrap() - .iter() - .find(|item| item["classification"].as_str() == Some(class)) - .and_then(|item| item["occurrences"].as_u64()) - .unwrap_or(0) - } - - fn object(uri: &str, byte: u8) -> CirObject { - CirObject { - rsync_uri: uri.to_string(), - sha256: vec![byte; 32], - } - } - - fn write_sample( - root: &Path, - dir: &str, - side: &str, - seq: u32, - objects: &[CirObject], - rejected: &[&str], - asn: u32, - ) { - let dir = root.join(dir); - std::fs::create_dir_all(&dir).unwrap(); - let cir = sample_cir(seq, objects, rejected); - let ccr = sample_ccr(seq, asn); - std::fs::write(dir.join("result.cir"), encode_cir(&cir).unwrap()).unwrap(); - std::fs::write(dir.join("result.ccr"), encode_content_info(&ccr).unwrap()).unwrap(); - let _ = side; - } - - fn write_sample_with_ccr_seq( - root: &Path, - dir: &str, - cir_seq: u32, - objects: &[CirObject], - rejected: &[&str], - ccr_seq: u32, - asn: u32, - ) { - let dir = root.join(dir); - std::fs::create_dir_all(&dir).unwrap(); - let cir = sample_cir(cir_seq, objects, rejected); - let ccr = sample_ccr(ccr_seq, asn); - std::fs::write(dir.join("result.cir"), encode_cir(&cir).unwrap()).unwrap(); - std::fs::write(dir.join("result.ccr"), encode_content_info(&ccr).unwrap()).unwrap(); - } - - fn sample_time(seq: u32) -> OffsetDateTime { - OffsetDateTime::from_unix_timestamp(1_800_000_000 + i64::from(seq * 60)).unwrap() - } - - fn sample_cir( - seq: u32, - objects: &[CirObject], - rejected: &[&str], - ) -> CanonicalInputRepresentation { - let mut objects = objects.to_vec(); - objects.sort_by(|a, b| a.rsync_uri.cmp(&b.rsync_uri)); - let rejected_objects = rejected - .iter() - .map(|uri| CirRejectedObject { - object_uri: (*uri).to_string(), - reason: Some("test".to_string()), - }) - .collect::>(); - CanonicalInputRepresentation { - version: CIR_VERSION_V3, - hash_alg: CirHashAlgorithm::Sha256, - validation_time: sample_time(seq), - objects, - trust_anchors: vec![sample_trust_anchor()], - reject_list_sha256: compute_reject_list_sha256(rejected.iter().copied()), - rejected_objects, - } - } - - fn sample_trust_anchor() -> CirTrustAnchor { - let ta_uri = "rsync://example.net/ta.cer"; - let ta_der = b"ta-der".to_vec(); - CirTrustAnchor { - ta_rsync_uri: ta_uri.to_string(), - tal_uri: "https://example.net/root.tal".to_string(), - tal_bytes: format!("{ta_uri}\n\nAQID\n").into_bytes(), - ta_certificate_der: ta_der.clone(), - ta_certificate_sha256: sha256(&ta_der), - } - } - - fn sample_ccr(seq: u32, asn: u32) -> CcrContentInfo { - let vrps = build_roa_payload_state(&[Vrp { - asn, - prefix: IpPrefix { - afi: RoaAfi::Ipv4, - prefix_len: 24, - addr: [192, 0, seq as u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - }, - max_length: 24, - }]) - .unwrap(); - let vaps = build_aspa_payload_state(&[AspaAttestation { - customer_as_id: asn, - provider_as_ids: vec![64497], - }]) - .unwrap(); - CcrContentInfo::new(RpkiCanonicalCacheRepresentation { - version: 0, - hash_alg: CcrDigestAlgorithm::Sha256, - produced_at: sample_time(seq), - mfts: None, - vrps: Some(vrps), - vaps: Some(vaps), - tas: None, - rks: None, - }) - } - - fn item(side: &str, seq: u32, ccr: &str, cir: &str) -> Value { - json!({ - "schemaVersion": 1, - "rpId": format!("{side}-rp"), - "side": side, - "seq": seq, - "runId": format!("{side}-{seq}"), - "syncMode": if seq == 1 { "snapshot" } else { "delta" }, - "status": "success", - "validationTime": format_time(sample_time(seq)), - "ccrPath": ccr, - "cirPath": cir - }) - } - - fn jsonl(items: &[Value]) -> String { - let mut out = String::new(); - for item in items { - out.push_str(&serde_json::to_string(item).unwrap()); - out.push('\n'); - } - out - } -} diff --git a/src/cli.rs b/src/cli.rs index a6ed0fd..b3b7fd9 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,8 +1,9 @@ +mod output; + use crate::ccr::{ CcrAccumulator, CcrBuildBreakdown, build_ccr_from_run_with_breakdown, write_ccr_file, }; use crate::cir::{CirTrustAnchorBinding, export_cir_from_run_multi}; -use std::io::BufWriter; use std::path::{Path, PathBuf}; use crate::analysis::timing::{TimingHandle, TimingMeta, TimingMetaUpdate}; @@ -10,7 +11,6 @@ use crate::audit::{ AspaOutput, AuditRepoSyncStats, AuditReportV2, AuditRunMeta, AuditWarning, TreeSummary, VrpOutput, format_roa_ip_prefix, }; -use crate::ccr::canonical_vrp_prefix; use crate::fetch::http::{BlockingHttpFetcher, HttpFetcherConfig}; use crate::fetch::rsync::LocalDirRsyncFetcher; use crate::fetch::rsync_system::{RsyncScopePolicy, SystemRsyncConfig, SystemRsyncFetcher}; @@ -28,6 +28,7 @@ use crate::validation::run_tree_from_tal::{ run_tree_from_tal_url_parallel_phase2_audit, }; use crate::validation::tree::TreeRunConfig; +use output::{ReportJsonFormat, run_compare_view_task, write_json, write_stage_timing}; use serde::Serialize; use std::sync::Arc; @@ -866,24 +867,6 @@ fn read_policy(path: Option<&Path>) -> Result { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum ReportJsonFormat { - Pretty, - Compact, -} - -fn write_json(path: &Path, report: &AuditReportV2, format: ReportJsonFormat) -> Result<(), String> { - let f = std::fs::File::create(path) - .map_err(|e| format!("create report file failed: {}: {e}", path.display()))?; - let writer = BufWriter::new(f); - match format { - ReportJsonFormat::Pretty => serde_json::to_writer_pretty(writer, report), - ReportJsonFormat::Compact => serde_json::to_writer(writer, report), - } - .map_err(|e| format!("write report json failed: {e}"))?; - Ok(()) -} - fn unique_rrdp_repos_from_publication_points( publication_points: &[crate::audit::PublicationPointAudit], ) -> usize { @@ -1188,121 +1171,6 @@ fn run_ccr_task( }) } -#[derive(Clone, Debug, PartialEq, Eq)] -struct CompareViewTaskOutput { - build_ms: Option, - write_ms: Option, -} - -fn run_compare_view_task( - shared: &PostValidationShared, - vrps_csv_out_path: Option<&Path>, - vaps_csv_out_path: Option<&Path>, - trust_anchor: &str, -) -> Result { - let mut build_ms = None; - let mut write_ms = None; - if let (Some(vrps_path), Some(vaps_path)) = (vrps_csv_out_path, vaps_csv_out_path) { - let started = std::time::Instant::now(); - build_ms = Some(0); - write_direct_vrp_csv(vrps_path, shared.vrps.as_ref(), trust_anchor)?; - write_direct_vap_csv(vaps_path, shared.aspas.as_ref(), trust_anchor)?; - write_ms = Some(started.elapsed().as_millis() as u64); - eprintln!( - "wrote compare views: vrps={} vaps={}", - vrps_path.display(), - vaps_path.display() - ); - } - Ok(CompareViewTaskOutput { build_ms, write_ms }) -} - -fn write_direct_vrp_csv( - path: &Path, - vrps: &[crate::validation::objects::Vrp], - trust_anchor: &str, -) -> Result<(), String> { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent) - .map_err(|e| format!("create parent dirs failed: {}: {e}", parent.display()))?; - } - let file = std::fs::File::create(path) - .map_err(|e| format!("create file failed: {}: {e}", path.display()))?; - let mut writer = BufWriter::new(file); - use std::io::Write; - let trust_anchor = trust_anchor.to_ascii_lowercase(); - writeln!(writer, "ASN,IP Prefix,Max Length,Trust Anchor").map_err(|e| e.to_string())?; - for vrp in vrps { - writeln!( - writer, - "AS{},{},{},{}", - vrp.asn, - canonical_vrp_prefix(&vrp.prefix), - vrp.max_length, - trust_anchor - ) - .map_err(|e| e.to_string())?; - } - Ok(()) -} - -fn write_direct_vap_csv( - path: &Path, - aspas: &[crate::validation::objects::AspaAttestation], - trust_anchor: &str, -) -> Result<(), String> { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent) - .map_err(|e| format!("create parent dirs failed: {}: {e}", parent.display()))?; - } - let file = std::fs::File::create(path) - .map_err(|e| format!("create file failed: {}: {e}", path.display()))?; - let mut writer = BufWriter::new(file); - use std::io::Write; - let trust_anchor = trust_anchor.to_ascii_lowercase(); - writeln!(writer, "Customer ASN,Providers,Trust Anchor").map_err(|e| e.to_string())?; - for aspa in aspas { - let mut providers = aspa.provider_as_ids.clone(); - providers.sort_unstable(); - providers.dedup(); - let providers = providers - .into_iter() - .map(|asn| format!("AS{asn}")) - .collect::>() - .join(";"); - writeln!( - writer, - "AS{},{},{}", - aspa.customer_as_id, providers, trust_anchor - ) - .map_err(|e| e.to_string())?; - } - Ok(()) -} - -fn write_stage_timing( - report_json_path: Option<&Path>, - stage_timing: &RunStageTiming, -) -> Result<(), String> { - if let Some(path) = report_json_path { - if let Some(parent) = path.parent() { - let stage_timing_path = parent.join("stage-timing.json"); - std::fs::write( - &stage_timing_path, - serde_json::to_vec_pretty(stage_timing).map_err(|e| e.to_string())?, - ) - .map_err(|e| { - format!( - "write stage timing failed: {}: {e}", - stage_timing_path.display() - ) - })?; - eprintln!("analysis: wrote {}", stage_timing_path.display()); - } - } - Ok(()) -} - fn resolve_cir_export_tal_uris(args: &CliArgs) -> Result, String> { if !args.cir_tal_uris.is_empty() { return Ok(args.cir_tal_uris.clone()); @@ -1990,1595 +1858,5 @@ pub fn run(argv: &[String]) -> Result<(), String> { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_help_returns_usage() { - let argv = vec!["rpki".to_string(), "--help".to_string()]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("Usage:"), "{err}"); - assert!(err.contains("--db"), "{err}"); - assert!(err.contains("--rsync-mirror-root"), "{err}"); - assert!(err.contains("--rsync-scope"), "{err}"); - assert!(err.contains("--parallel-phase2-object-workers"), "{err}"); - assert!(!err.contains("--parallel-phase1"), "{err}"); - assert!(!err.contains("--parallel-phase2 "), "{err}"); - } - - #[test] - fn parse_rejects_unknown_argument() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--nope".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("unknown argument"), "{err}"); - } - - #[test] - fn parse_rejects_both_tal_url_and_tal_path() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!( - err.contains("one-or-more --tal-url or one-or-more --tal-path/--ta-path pairs"), - "{err}" - ); - } - - #[test] - fn parse_rejects_invalid_max_depth() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--max-depth".to_string(), - "nope".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("invalid --max-depth"), "{err}"); - } - - #[test] - fn parse_accepts_ccr_out_path() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - "--rsync-local-dir".to_string(), - "repo".to_string(), - "--ccr-out".to_string(), - "out/example.ccr".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!( - args.ccr_out_path.as_deref(), - Some(std::path::Path::new("out/example.ccr")) - ); - } - - #[test] - fn parse_accepts_report_json_compact_when_report_json_is_set() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--report-json".to_string(), - "out/report.json".to_string(), - "--report-json-compact".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!( - args.report_json_path.as_deref(), - Some(std::path::Path::new("out/report.json")) - ); - assert!(args.report_json_compact); - } - - #[test] - fn parse_accepts_rsync_scope_policy() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--rsync-scope".to_string(), - "module-root".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!(args.rsync_scope_policy, RsyncScopePolicy::ModuleRoot); - } - - #[test] - fn parse_rejects_invalid_rsync_scope_policy() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--rsync-scope".to_string(), - "wide".to_string(), - ]; - let err = parse_args(&argv).expect_err("invalid rsync scope should fail"); - assert!(err.contains("invalid --rsync-scope"), "{err}"); - } - - #[test] - fn parse_accepts_strict_policy_list() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--strict".to_string(), - "name,cms-der".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!( - args.strict_policy, - Some(StrictPolicy { - name: true, - cms_der: true, - signed_attrs: false, - }) - ); - } - - #[test] - fn parse_accepts_strict_without_value_as_all() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--strict".to_string(), - "--report-json".to_string(), - "out/report.json".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!(args.strict_policy, Some(StrictPolicy::all())); - } - - #[test] - fn parse_rejects_unknown_strict_policy() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--strict=unknown".to_string(), - ]; - let err = parse_args(&argv).expect_err("unknown strict policy should fail"); - assert!(err.contains("unknown strict policy"), "{err}"); - } - - #[test] - fn effective_cir_tal_uris_filters_skipped_multi_tal_inputs() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "afrinic.tal".to_string(), - "--ta-path".to_string(), - "afrinic.cer".to_string(), - "--tal-path".to_string(), - "apnic.tal".to_string(), - "--ta-path".to_string(), - "apnic.cer".to_string(), - "--tal-path".to_string(), - "arin.tal".to_string(), - "--ta-path".to_string(), - "arin.cer".to_string(), - "--cir-enable".to_string(), - "--cir-out".to_string(), - "out.cir".to_string(), - "--cir-tal-uri".to_string(), - "https://example.test/afrinic.cer".to_string(), - "--cir-tal-uri".to_string(), - "https://example.test/apnic.cer".to_string(), - "--cir-tal-uri".to_string(), - "https://example.test/arin.cer".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - let mut shared = synthetic_post_validation_shared(); - shared.discoveries = vec![shared.discovery.clone(), shared.discovery.clone()].into(); - shared.successful_tal_inputs = - vec![args.tal_inputs[0].clone(), args.tal_inputs[2].clone()].into(); - - let effective = effective_cir_tal_uris_for_discoveries( - &args, - &shared, - resolve_cir_export_tal_uris(&args).expect("resolve cir tal uris"), - ) - .expect("map effective cir tal uris"); - - assert_eq!( - effective, - vec![ - "https://example.test/afrinic.cer".to_string(), - "https://example.test/arin.cer".to_string(), - ] - ); - } - - #[test] - fn parse_rejects_report_json_compact_without_report_json() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--report-json-compact".to_string(), - ]; - let err = parse_args(&argv).expect_err("compact flag without report path should fail"); - assert!( - err.contains("--report-json-compact requires --report-json"), - "{err}" - ); - } - - #[test] - fn parse_accepts_skip_report_build_without_report_json() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--ccr-out".to_string(), - "out/result.ccr".to_string(), - "--skip-report-build".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(args.skip_report_build); - assert_eq!( - args.ccr_out_path.as_deref(), - Some(std::path::Path::new("out/result.ccr")) - ); - } - - #[test] - fn parse_accepts_skip_vcir_persist() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--skip-vcir-persist".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(args.skip_vcir_persist); - } - - #[test] - fn parse_rejects_skip_report_build_with_report_json() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--report-json".to_string(), - "out/report.json".to_string(), - "--skip-report-build".to_string(), - ]; - let err = parse_args(&argv).expect_err("skip report build with report path should fail"); - assert!( - err.contains("--skip-report-build cannot be combined with --report-json"), - "{err}" - ); - } - - #[test] - fn parse_accepts_direct_compare_view_csv_outputs() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--vrps-csv-out".to_string(), - "out/vrps.csv".to_string(), - "--vaps-csv-out".to_string(), - "out/vaps.csv".to_string(), - "--compare-view-trust-anchor".to_string(), - "unknown".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!( - args.vrps_csv_out_path.as_deref(), - Some(std::path::Path::new("out/vrps.csv")) - ); - assert_eq!( - args.vaps_csv_out_path.as_deref(), - Some(std::path::Path::new("out/vaps.csv")) - ); - assert_eq!(args.compare_view_trust_anchor.as_deref(), Some("unknown")); - } - - #[test] - fn parse_rejects_partial_direct_compare_view_csv_outputs() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--vrps-csv-out".to_string(), - "out/vrps.csv".to_string(), - ]; - let err = parse_args(&argv).expect_err("partial direct compare view output should fail"); - assert!( - err.contains("--vrps-csv-out and --vaps-csv-out must be provided together"), - "{err}" - ); - } - - #[test] - fn parse_accepts_external_raw_store_db() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--raw-store-db".to_string(), - "raw-store.db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!( - args.raw_store_db.as_deref(), - Some(std::path::Path::new("raw-store.db")) - ); - } - - #[test] - fn parse_accepts_external_repo_bytes_db() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--repo-bytes-db".to_string(), - "repo-bytes.db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!( - args.repo_bytes_db.as_deref(), - Some(std::path::Path::new("repo-bytes.db")) - ); - } - - #[test] - fn parse_accepts_cir_enable_with_raw_store_backend() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--raw-store-db".to_string(), - "raw-store.db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - "--rsync-local-dir".to_string(), - "repo".to_string(), - "--cir-enable".to_string(), - "--cir-out".to_string(), - "out/example.cir".to_string(), - "--cir-tal-uri".to_string(), - "https://example.test/root.tal".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(args.cir_enabled); - assert_eq!( - args.raw_store_db.as_deref(), - Some(std::path::Path::new("raw-store.db")) - ); - assert_eq!(args.cir_static_root, None); - } - - #[test] - fn parse_accepts_cir_enable_with_required_paths_and_tal_override() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - "--rsync-local-dir".to_string(), - "repo".to_string(), - "--cir-enable".to_string(), - "--cir-out".to_string(), - "out/example.cir".to_string(), - "--cir-tal-uri".to_string(), - "https://example.test/root.tal".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(args.cir_enabled); - assert_eq!( - args.cir_out_path.as_deref(), - Some(std::path::Path::new("out/example.cir")) - ); - assert_eq!( - args.cir_tal_uri.as_deref(), - Some("https://example.test/root.tal") - ); - assert_eq!( - args.cir_tal_uris, - vec!["https://example.test/root.tal".to_string()] - ); - } - - #[test] - fn parse_rejects_deprecated_cir_static_root() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--cir-enable".to_string(), - "--cir-out".to_string(), - "out/example.cir".to_string(), - "--cir-static-root".to_string(), - "out/static".to_string(), - ]; - let err = parse_args(&argv).expect_err("cir-static-root should be rejected"); - assert!(err.contains("no longer supported"), "{err}"); - } - - #[test] - fn parse_accepts_default_parallel_config_and_phase2_overrides() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase2-object-workers".to_string(), - "3".to_string(), - "--parallel-phase2-worker-queue-capacity".to_string(), - "17".to_string(), - "--parallel-phase2-ready-batch-size".to_string(), - "31".to_string(), - "--parallel-phase2-ready-batch-wall-time-budget-ms".to_string(), - "43".to_string(), - "--parallel-phase2-result-drain-batch-size".to_string(), - "37".to_string(), - "--parallel-phase2-finalize-batch-size".to_string(), - "41".to_string(), - "--parallel-phase2-finalize-batch-wall-time-budget-ms".to_string(), - "47".to_string(), - "--parallel-phase2-finalize-queue-capacity".to_string(), - "8192".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!(args.parallel_phase2_config.object_workers, 3); - assert_eq!(args.parallel_phase2_config.worker_queue_capacity, 17); - assert_eq!(args.parallel_phase2_config.ready_batch_size, 31); - assert_eq!( - args.parallel_phase2_config.ready_batch_wall_time_budget_ms, - 43 - ); - assert_eq!( - args.parallel_phase2_config.object_result_drain_batch_size, - 37 - ); - assert_eq!( - args.parallel_phase2_config - .publication_point_finalize_batch_size, - 41 - ); - assert_eq!( - args.parallel_phase2_config - .publication_point_finalize_wall_time_budget_ms, - 47 - ); - assert_eq!( - args.parallel_phase2_config - .publication_point_finalize_queue_capacity, - 8192 - ); - assert_eq!(args.parallel_phase1_config, ParallelPhase1Config::default()); - } - - #[test] - fn parse_rejects_zero_phase2_ready_batch_size() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase2-ready-batch-size".to_string(), - "0".to_string(), - ]; - let err = parse_args(&argv).expect_err("zero ready batch must fail"); - assert!(err.contains("--parallel-phase2-ready-batch-size"), "{err}"); - } - - #[test] - fn parse_rejects_zero_phase2_result_drain_batch_size() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase2-result-drain-batch-size".to_string(), - "0".to_string(), - ]; - let err = parse_args(&argv).expect_err("zero result drain batch must fail"); - assert!( - err.contains("--parallel-phase2-result-drain-batch-size"), - "{err}" - ); - } - - #[test] - fn parse_rejects_zero_phase2_ready_batch_wall_time_budget_ms() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase2-ready-batch-wall-time-budget-ms".to_string(), - "0".to_string(), - ]; - let err = parse_args(&argv).expect_err("zero ready time budget must fail"); - assert!( - err.contains("--parallel-phase2-ready-batch-wall-time-budget-ms"), - "{err}" - ); - } - - #[test] - fn parse_rejects_zero_phase2_finalize_batch_size() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase2-finalize-batch-size".to_string(), - "0".to_string(), - ]; - let err = parse_args(&argv).expect_err("zero finalize batch must fail"); - assert!( - err.contains("--parallel-phase2-finalize-batch-size"), - "{err}" - ); - } - - #[test] - fn parse_rejects_zero_phase2_finalize_batch_wall_time_budget_ms() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase2-finalize-batch-wall-time-budget-ms".to_string(), - "0".to_string(), - ]; - let err = parse_args(&argv).expect_err("zero finalize time budget must fail"); - assert!( - err.contains("--parallel-phase2-finalize-batch-wall-time-budget-ms"), - "{err}" - ); - } - - #[test] - fn parse_rejects_zero_phase2_finalize_queue_capacity() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase2-finalize-queue-capacity".to_string(), - "0".to_string(), - ]; - let err = parse_args(&argv).expect_err("zero finalize queue capacity must fail"); - assert!( - err.contains("--parallel-phase2-finalize-queue-capacity"), - "{err}" - ); - } - - #[test] - fn parse_rejects_removed_parallel_enable_flags() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase1".to_string(), - ]; - let err = parse_args(&argv).expect_err("removed phase flag should fail"); - assert!(err.contains("unknown argument: --parallel-phase1"), "{err}"); - - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase2".to_string(), - ]; - let err = parse_args(&argv).expect_err("removed phase flag should fail"); - assert!(err.contains("unknown argument: --parallel-phase2"), "{err}"); - } - - #[test] - fn parse_accepts_multi_tal_cir_overrides_in_file_mode() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "apnic.tal".to_string(), - "--ta-path".to_string(), - "apnic.cer".to_string(), - "--tal-path".to_string(), - "arin.tal".to_string(), - "--ta-path".to_string(), - "arin.cer".to_string(), - "--rsync-local-dir".to_string(), - "repo".to_string(), - "--cir-enable".to_string(), - "--cir-out".to_string(), - "out/example.cir".to_string(), - "--cir-tal-uri".to_string(), - "https://example.test/apnic.tal".to_string(), - "--cir-tal-uri".to_string(), - "https://example.test/arin.tal".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!( - args.cir_tal_uris, - vec![ - "https://example.test/apnic.tal".to_string(), - "https://example.test/arin.tal".to_string() - ] - ); - } - - #[test] - fn parse_rejects_incomplete_or_invalid_cir_flags() { - let argv_missing = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--cir-enable".to_string(), - ]; - let err = parse_args(&argv_missing).unwrap_err(); - assert!(err.contains("--cir-enable requires --cir-out"), "{err}"); - - let argv_needs_enable = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--cir-out".to_string(), - "out/example.cir".to_string(), - ]; - let err = parse_args(&argv_needs_enable).unwrap_err(); - assert!(err.contains("require --cir-enable"), "{err}"); - - let argv_offline_missing_uri = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - "--rsync-local-dir".to_string(), - "repo".to_string(), - "--cir-enable".to_string(), - "--cir-out".to_string(), - "out/example.cir".to_string(), - ]; - let err = parse_args(&argv_offline_missing_uri).unwrap_err(); - assert!(err.contains("requires --cir-tal-uri"), "{err}"); - } - - #[test] - fn parse_rejects_invalid_validation_time() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--validation-time".to_string(), - "not-a-time".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("invalid --validation-time"), "{err}"); - } - - #[test] - fn parse_rejects_invalid_max_instances() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--max-instances".to_string(), - "nope".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("invalid --max-instances"), "{err}"); - } - - #[test] - fn parse_rejects_missing_value_for_db() { - let argv = vec!["rpki".to_string(), "--db".to_string()]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("--db requires a value"), "{err}"); - } - - #[test] - fn parse_rejects_missing_value_for_tal_url() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("--tal-url requires a value"), "{err}"); - } - - #[test] - fn parse_rejects_missing_db() { - let argv = vec!["rpki".to_string(), "--tal-url".to_string(), "x".to_string()]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("--db is required"), "{err}"); - } - - #[test] - fn parse_rejects_missing_tal_mode() { - let argv = vec!["rpki".to_string(), "--db".to_string(), "db".to_string()]; - let err = parse_args(&argv).unwrap_err(); - assert!( - err.contains("--tal-url") || err.contains("--tal-path"), - "{err}" - ); - } - - #[test] - fn parse_accepts_tal_url_mode() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!(args.tal_url.as_deref(), Some("https://example.test/x.tal")); - assert_eq!( - args.tal_urls, - vec!["https://example.test/x.tal".to_string()] - ); - assert!(args.tal_path.is_none()); - assert!(args.ta_path.is_none()); - assert_eq!(args.tal_inputs.len(), 1); - assert_eq!(args.tal_inputs[0].tal_id, "x"); - assert_eq!(args.parallel_phase1_config, ParallelPhase1Config::default()); - assert_eq!(args.parallel_phase2_config, ParallelPhase2Config::default()); - } - - #[test] - fn parse_accepts_multi_tal_without_parallel_flags() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/arin.tal".to_string(), - "--tal-url".to_string(), - "https://example.test/apnic.tal".to_string(), - "--tal-url".to_string(), - "https://example.test/ripe.tal".to_string(), - "--parallel-max-repo-sync-workers-global".to_string(), - "8".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!(args.tal_urls.len(), 3); - assert_eq!(args.tal_inputs.len(), 3); - assert_eq!(args.tal_inputs[0].tal_id, "arin"); - assert_eq!(args.tal_inputs[1].tal_id, "apnic"); - assert_eq!(args.tal_inputs[2].tal_id, "ripe"); - assert_eq!(args.parallel_phase1_config.max_repo_sync_workers_global, 8); - } - - #[test] - fn parse_accepts_multi_tal_urls_by_default() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/arin.tal".to_string(), - "--tal-url".to_string(), - "https://example.test/apnic.tal".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!(args.tal_urls.len(), 2); - assert_eq!(args.tal_inputs.len(), 2); - } - - #[test] - fn parse_accepts_offline_mode_requires_ta() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--ta-path".to_string(), - "ta.cer".to_string(), - "--max-depth".to_string(), - "0".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!(args.tal_paths, vec![PathBuf::from("a.tal")]); - assert_eq!(args.ta_paths, vec![PathBuf::from("ta.cer")]); - assert_eq!(args.tal_path.as_deref(), Some(Path::new("a.tal"))); - assert_eq!(args.ta_path.as_deref(), Some(Path::new("ta.cer"))); - assert_eq!(args.max_depth, Some(0)); - } - - #[test] - fn parse_accepts_multiple_tal_path_pairs_by_default() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "apnic.tal".to_string(), - "--ta-path".to_string(), - "apnic-ta.cer".to_string(), - "--tal-path".to_string(), - "arin.tal".to_string(), - "--ta-path".to_string(), - "arin-ta.cer".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!(args.tal_paths.len(), 2); - assert_eq!(args.ta_paths.len(), 2); - assert_eq!(args.tal_inputs.len(), 2); - } - - #[test] - fn parse_rejects_mixed_tal_url_and_tal_path_modes() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/arin.tal".to_string(), - "--tal-path".to_string(), - "apnic.tal".to_string(), - "--ta-path".to_string(), - "apnic-ta.cer".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("must specify either one-or-more --tal-url or one-or-more --tal-path/--ta-path pairs"), "{err}"); - } - - #[test] - fn parse_rejects_mismatched_tal_path_and_ta_path_counts() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "apnic.tal".to_string(), - "--tal-path".to_string(), - "arin.tal".to_string(), - "--ta-path".to_string(), - "apnic-ta.cer".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!( - err.contains("--tal-path and --ta-path counts must match"), - "{err}" - ); - } - - #[test] - fn parse_accepts_tal_path_without_ta_when_disable_rrdp_is_set() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--disable-rrdp".to_string(), - "--rsync-command".to_string(), - "/tmp/fake-rsync".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!(args.tal_path.as_deref(), Some(Path::new("a.tal"))); - assert!(args.ta_path.is_none()); - assert!(args.disable_rrdp); - assert_eq!( - args.rsync_command.as_deref(), - Some(Path::new("/tmp/fake-rsync")) - ); - } - - #[test] - fn parse_accepts_multiple_tal_paths_without_ta_when_disable_rrdp() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--tal-path".to_string(), - "b.tal".to_string(), - "--disable-rrdp".to_string(), - "--rsync-command".to_string(), - "/tmp/fake-rsync".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!( - args.tal_paths, - vec![PathBuf::from("a.tal"), PathBuf::from("b.tal")] - ); - assert!(args.ta_paths.is_empty()); - assert_eq!(args.tal_inputs.len(), 2); - assert!(args.disable_rrdp); - } - - #[test] - fn parse_accepts_payload_delta_replay_mode_with_offline_tal_and_ta() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--ta-path".to_string(), - "ta.cer".to_string(), - "--payload-base-archive".to_string(), - "base-archive".to_string(), - "--payload-base-locks".to_string(), - "base-locks.json".to_string(), - "--payload-delta-archive".to_string(), - "delta-archive".to_string(), - "--payload-delta-locks".to_string(), - "delta-locks.json".to_string(), - ]; - let args = parse_args(&argv).expect("parse delta replay mode"); - assert_eq!( - args.payload_base_archive.as_deref(), - Some(Path::new("base-archive")) - ); - assert_eq!( - args.payload_base_locks.as_deref(), - Some(Path::new("base-locks.json")) - ); - assert_eq!( - args.payload_delta_archive.as_deref(), - Some(Path::new("delta-archive")) - ); - assert_eq!( - args.payload_delta_locks.as_deref(), - Some(Path::new("delta-locks.json")) - ); - } - - #[test] - fn parse_rejects_partial_payload_delta_arguments_and_mutual_exclusion() { - let argv_partial = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--ta-path".to_string(), - "ta.cer".to_string(), - "--payload-base-archive".to_string(), - "base-archive".to_string(), - ]; - let err = parse_args(&argv_partial).unwrap_err(); - assert!(err.contains("must be provided together"), "{err}"); - - let argv_both = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--ta-path".to_string(), - "ta.cer".to_string(), - "--payload-replay-archive".to_string(), - "archive".to_string(), - "--payload-replay-locks".to_string(), - "locks.json".to_string(), - "--payload-base-archive".to_string(), - "base-archive".to_string(), - "--payload-base-locks".to_string(), - "base-locks.json".to_string(), - "--payload-delta-archive".to_string(), - "delta-archive".to_string(), - "--payload-delta-locks".to_string(), - "delta-locks.json".to_string(), - ]; - let err = parse_args(&argv_both).unwrap_err(); - assert!(err.contains("mutually exclusive"), "{err}"); - } - - #[test] - fn parse_rejects_payload_delta_with_tal_url_or_rsync_local_dir() { - let argv_url = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--payload-base-archive".to_string(), - "base-archive".to_string(), - "--payload-base-locks".to_string(), - "base-locks.json".to_string(), - "--payload-delta-archive".to_string(), - "delta-archive".to_string(), - "--payload-delta-locks".to_string(), - "delta-locks.json".to_string(), - ]; - let err = parse_args(&argv_url).unwrap_err(); - assert!(err.contains("--tal-url is not supported"), "{err}"); - - let argv_rsync = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--ta-path".to_string(), - "ta.cer".to_string(), - "--payload-base-archive".to_string(), - "base-archive".to_string(), - "--payload-base-locks".to_string(), - "base-locks.json".to_string(), - "--payload-delta-archive".to_string(), - "delta-archive".to_string(), - "--payload-delta-locks".to_string(), - "delta-locks.json".to_string(), - "--rsync-local-dir".to_string(), - "repo".to_string(), - ]; - let err = parse_args(&argv_rsync).unwrap_err(); - assert!( - err.contains("payload delta replay mode cannot be combined with --rsync-local-dir"), - "{err}" - ); - } - - #[test] - fn parse_accepts_payload_replay_mode_with_offline_tal_and_ta() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--ta-path".to_string(), - "ta.cer".to_string(), - "--payload-replay-archive".to_string(), - "archive".to_string(), - "--payload-replay-locks".to_string(), - "locks.json".to_string(), - ]; - let args = parse_args(&argv).expect("parse replay mode"); - assert_eq!( - args.payload_replay_archive.as_deref(), - Some(Path::new("archive")) - ); - assert_eq!( - args.payload_replay_locks.as_deref(), - Some(Path::new("locks.json")) - ); - } - - #[test] - fn parse_rejects_partial_payload_replay_arguments() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--ta-path".to_string(), - "ta.cer".to_string(), - "--payload-replay-archive".to_string(), - "archive".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("must be provided together"), "{err}"); - } - - #[test] - fn parse_rejects_payload_replay_with_tal_url_or_rsync_local_dir() { - let argv_url = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--payload-replay-archive".to_string(), - "archive".to_string(), - "--payload-replay-locks".to_string(), - "locks.json".to_string(), - ]; - let err = parse_args(&argv_url).unwrap_err(); - assert!(err.contains("--tal-url is not supported"), "{err}"); - - let argv_rsync = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--ta-path".to_string(), - "ta.cer".to_string(), - "--payload-replay-archive".to_string(), - "archive".to_string(), - "--payload-replay-locks".to_string(), - "locks.json".to_string(), - "--rsync-local-dir".to_string(), - "repo".to_string(), - ]; - let err = parse_args(&argv_rsync).unwrap_err(); - assert!( - err.contains("cannot be combined with --rsync-local-dir"), - "{err}" - ); - } - - #[test] - fn parse_accepts_validation_time_rfc3339() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--validation-time".to_string(), - "2026-01-01T00:00:00Z".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert!(args.validation_time.is_some()); - } - - #[test] - fn parse_rejects_removed_revalidate_only_flag() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--revalidate-only".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("unknown argument: --revalidate-only"), "{err}"); - } - - #[test] - fn read_policy_accepts_valid_toml() { - let dir = tempfile::tempdir().expect("tmpdir"); - let p = dir.path().join("policy.toml"); - std::fs::write( - &p, - "signed_object_failure_policy = \"drop_publication_point\"\n", - ) - .expect("write policy"); - - let policy = read_policy(Some(&p)).expect("parse policy"); - assert_eq!( - policy.signed_object_failure_policy, - crate::policy::SignedObjectFailurePolicy::DropPublicationPoint - ); - assert_eq!(policy.strict, StrictPolicy::default()); - } - - #[test] - fn read_policy_accepts_strict_table() { - let dir = tempfile::tempdir().expect("tmpdir"); - let p = dir.path().join("policy.toml"); - std::fs::write( - &p, - r#" - [strict] - name = true - cms_der = true - "#, - ) - .expect("write policy"); - - let policy = read_policy(Some(&p)).expect("parse policy"); - assert_eq!( - policy.strict, - StrictPolicy { - name: true, - cms_der: true, - signed_attrs: false, - } - ); - } - - #[test] - fn read_policy_reports_missing_file() { - let dir = tempfile::tempdir().expect("tmpdir"); - let p = dir.path().join("missing.toml"); - let err = read_policy(Some(&p)).unwrap_err(); - assert!(err.contains("read policy file failed"), "{err}"); - } - - fn synthetic_post_validation_shared() -> PostValidationShared { - let tal_bytes = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/tal/apnic-rfc7730-https.tal"), - ) - .expect("read tal fixture"); - let ta_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/ta/apnic-ta.cer"), - ) - .expect("read ta fixture"); - - let discovery = crate::validation::from_tal::discover_root_ca_instance_from_tal_and_ta_der( - &tal_bytes, &ta_der, None, - ) - .expect("discover root"); - - let tree = crate::validation::tree::TreeRunOutput { - instances_processed: 1, - instances_failed: 0, - warnings: vec![ - crate::report::Warning::new("synthetic warning") - .with_rfc_refs(&[crate::report::RfcRef("RFC 6487 §4.8.8.1")]) - .with_context("rsync://example.test/repo/pp/"), - ], - vrps: vec![ - crate::validation::objects::Vrp { - asn: 64496, - prefix: crate::data_model::roa::IpPrefix { - afi: crate::data_model::roa::RoaAfi::Ipv4, - prefix_len: 24, - addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - }, - max_length: 24, - }, - crate::validation::objects::Vrp { - asn: 64497, - prefix: crate::data_model::roa::IpPrefix { - afi: crate::data_model::roa::RoaAfi::Ipv6, - prefix_len: 48, - addr: [0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - }, - max_length: 64, - }, - ], - aspas: vec![crate::validation::objects::AspaAttestation { - customer_as_id: 64496, - provider_as_ids: vec![64497, 64498], - }], - router_keys: Vec::new(), - }; - - let mut pp1 = crate::audit::PublicationPointAudit::default(); - pp1.source = "fresh".to_string(); - pp1.rrdp_notification_uri = Some("https://example.test/n1.xml".to_string()); - let mut pp2 = crate::audit::PublicationPointAudit::default(); - pp2.source = "fresh".to_string(); - pp2.rrdp_notification_uri = Some("https://example.test/n1.xml".to_string()); - let mut pp3 = crate::audit::PublicationPointAudit::default(); - pp3.source = "fresh".to_string(); - pp3.rrdp_notification_uri = Some("https://example.test/n2.xml".to_string()); - - let out = crate::validation::run_tree_from_tal::RunTreeFromTalAuditOutput { - discovery: discovery.clone(), - discoveries: vec![discovery], - successful_tal_inputs: Vec::new(), - tree, - publication_points: vec![pp1, pp2, pp3], - downloads: Vec::new(), - download_stats: crate::audit::AuditDownloadStats::default(), - current_repo_objects: Vec::new(), - ccr_accumulator: None, - }; - PostValidationShared::from_run_output(out) - } - - fn sample_cli_ccr_accumulator() -> CcrAccumulator { - let tal_bytes = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/tal/apnic-rfc7730-https.tal"), - ) - .expect("read tal fixture"); - let ta_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/ta/apnic-ta.cer"), - ) - .expect("read ta fixture"); - let discovery = crate::validation::from_tal::discover_root_ca_instance_from_tal_and_ta_der( - &tal_bytes, &ta_der, None, - ) - .expect("discover root"); - let mut accumulator = CcrAccumulator::new(vec![discovery.trust_anchor.clone()]); - let projection = crate::storage::VcirCcrManifestProjection { - manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), - manifest_sha256: vec![0x44; 32], - manifest_size: 2048, - manifest_ee_aki: vec![0x55; 20], - manifest_number_be: vec![1], - manifest_this_update: crate::storage::PackTime::from_utc_offset_datetime( - time::OffsetDateTime::now_utc(), - ), - manifest_sia_locations_der: vec![vec![ - 0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05, - b'r', b's', b'y', b'n', b'c', - ]], - subordinate_skis: vec![vec![0x33; 20]], - }; - accumulator - .append_manifest_projection(&projection) - .expect("append manifest projection"); - accumulator - } - - #[test] - fn build_report_and_helpers_work_on_synthetic_output() { - let shared = synthetic_post_validation_shared(); - let policy = Policy::default(); - let validation_time = time::OffsetDateTime::now_utc(); - let report = build_report(&policy, validation_time, &shared); - - assert_eq!(unique_rrdp_repos(&report), 2); - assert_eq!(report.vrps.len(), 2); - assert_eq!(report.aspas.len(), 1); - - print_summary(&report); - } - - #[test] - fn run_report_task_and_stage_timing_work() { - let shared = synthetic_post_validation_shared(); - let policy = Policy::default(); - let validation_time = time::OffsetDateTime::now_utc(); - let dir = tempfile::tempdir().expect("tmpdir"); - let report_path = dir.path().join("report.json"); - let report_output = run_report_task( - &policy, - validation_time, - &shared, - Some(&report_path), - ReportJsonFormat::Compact, - ) - .expect("run report task"); - - let report = report_output.report.as_ref().expect("report built"); - assert_eq!(report.vrps.len(), 2); - assert_eq!(report.aspas.len(), 1); - assert!(report_output.report_write_ms.is_some()); - - let report_json = std::fs::read_to_string(&report_path).expect("read report json"); - assert!(!report_json.contains('\n'), "{report_json}"); - - let stage_timing = RunStageTiming { - validation_ms: 1, - report_build_ms: report_output.report_build_ms, - report_write_ms: report_output.report_write_ms, - ccr_build_ms: Some(2), - ccr_build_breakdown: None, - ccr_write_ms: Some(3), - compare_view_build_ms: Some(4), - compare_view_write_ms: Some(5), - cir_build_cir_ms: Some(6), - cir_write_cir_ms: Some(7), - cir_total_ms: Some(8), - total_ms: 9, - publication_points: shared.publication_points.len(), - repo_sync_ms_total: 10, - publication_point_repo_sync_ms_total: 11, - download_event_count: 12, - rrdp_download_ms_total: 13, - rsync_download_ms_total: 14, - download_bytes_total: 15, - }; - write_stage_timing(Some(&report_path), &stage_timing).expect("write stage timing"); - let stage_timing_json = - std::fs::read_to_string(dir.path().join("stage-timing.json")).expect("read timing"); - assert!(stage_timing_json.contains("\"validation_ms\"")); - assert!(stage_timing_json.contains("\"ccr_build_ms\"")); - - let ccr_path = dir.path().join("result.ccr"); - write_stage_timing(Some(&ccr_path), &stage_timing) - .expect("write stage timing via ccr path"); - assert!( - dir.path().join("stage-timing.json").exists(), - "stage timing should use parent directory of the anchor path" - ); - - let skipped = ReportTaskOutput::skipped(); - assert!(skipped.report.is_none()); - assert_eq!(skipped.report_build_ms, 0); - assert!(skipped.report_write_ms.is_none()); - } - - #[test] - fn run_compare_view_task_writes_csv_from_shared_output() { - let shared = synthetic_post_validation_shared(); - let dir = tempfile::tempdir().expect("tmpdir"); - let vrps_path = dir.path().join("vrps.csv"); - let vaps_path = dir.path().join("vaps.csv"); - - let output = run_compare_view_task(&shared, Some(&vrps_path), Some(&vaps_path), "unknown") - .expect("write direct compare views"); - - assert!(output.build_ms.is_some()); - assert!(output.write_ms.is_some()); - let vrps_csv = std::fs::read_to_string(vrps_path).expect("read vrps csv"); - let vaps_csv = std::fs::read_to_string(vaps_path).expect("read vaps csv"); - assert!(vrps_csv.contains("ASN,IP Prefix,Max Length,Trust Anchor")); - assert!(vrps_csv.contains("AS64496,192.0.2.0/24,24,unknown")); - assert!(vrps_csv.contains("AS64497,2001:db8::/48,64,unknown")); - assert!(vaps_csv.contains("Customer ASN,Providers,Trust Anchor")); - assert!(vaps_csv.contains("AS64496,AS64497;AS64498,unknown")); - } - - #[test] - fn run_ccr_task_uses_accumulator_when_phase2_output_contains_reuse_sources() { - let mut shared = synthetic_post_validation_shared(); - shared.ccr_accumulator = Some(sample_cli_ccr_accumulator()); - let mut publication_points = shared - .publication_points - .iter() - .cloned() - .collect::>(); - publication_points[1].source = "vcir_current_instance".to_string(); - publication_points[2].source = "failed_no_cache".to_string(); - shared.publication_points = publication_points.into(); - let dir = tempfile::tempdir().expect("tmpdir"); - let ccr_path = dir.path().join("result.ccr"); - let store = RocksStore::open(&dir.path().join("db")).expect("open empty store"); - - let output = run_ccr_task( - &store, - &shared, - Some(&ccr_path), - time::OffsetDateTime::now_utc(), - ) - .expect("run ccr task"); - - assert!(output.ccr_build_ms.is_some()); - assert!(output.ccr_build_breakdown.is_none()); - let der = std::fs::read(&ccr_path).expect("read ccr"); - let ci = crate::ccr::decode_content_info(&der).expect("decode ccr"); - assert_eq!( - ci.content - .mfts - .as_ref() - .map(|manifest_state| manifest_state.mis.len()), - Some(1) - ); - } - - #[test] - fn write_json_writes_report() { - let report = AuditReportV2 { - format_version: 2, - meta: AuditRunMeta { - validation_time_rfc3339_utc: "2026-01-01T00:00:00Z".to_string(), - }, - policy: Policy::default(), - tree: TreeSummary { - instances_processed: 0, - instances_failed: 0, - warnings: Vec::new(), - }, - publication_points: Vec::new(), - vrps: Vec::new(), - aspas: Vec::new(), - downloads: Vec::new(), - download_stats: crate::audit::AuditDownloadStats::default(), - repo_sync_stats: crate::audit::AuditRepoSyncStats::default(), - }; - - let dir = tempfile::tempdir().expect("tmpdir"); - let pretty_path = dir.path().join("report-pretty.json"); - write_json(&pretty_path, &report, ReportJsonFormat::Pretty).expect("write pretty json"); - let pretty = std::fs::read_to_string(&pretty_path).expect("read pretty report"); - assert!(pretty.contains("\"format_version\"")); - assert!(pretty.contains("\"policy\"")); - assert!(pretty.contains("\n \"format_version\""), "{pretty}"); - - let compact_path = dir.path().join("report-compact.json"); - write_json(&compact_path, &report, ReportJsonFormat::Compact).expect("write compact json"); - let compact = std::fs::read_to_string(&compact_path).expect("read compact report"); - assert!(compact.contains("\"format_version\"")); - assert!(compact.contains("\"policy\"")); - assert!(!compact.contains('\n'), "{compact}"); - } - - #[test] - fn build_repo_sync_stats_aggregates_phase_and_terminal_state() { - let mut pp1 = crate::audit::PublicationPointAudit::default(); - pp1.repo_sync_phase = Some("rrdp_ok".to_string()); - pp1.repo_sync_duration_ms = Some(10); - pp1.repo_terminal_state = "fresh".to_string(); - - let mut pp2 = crate::audit::PublicationPointAudit::default(); - pp2.repo_sync_phase = Some("rrdp_failed_rsync_failed".to_string()); - pp2.repo_sync_duration_ms = Some(20); - pp2.repo_terminal_state = "failed_no_cache".to_string(); - - let mut pp3 = crate::audit::PublicationPointAudit::default(); - pp3.repo_sync_phase = Some("rrdp_failed_rsync_failed".to_string()); - pp3.repo_sync_duration_ms = Some(30); - pp3.repo_terminal_state = "failed_no_cache".to_string(); - - let stats = build_repo_sync_stats(&[pp1, pp2, pp3]); - assert_eq!(stats.publication_points_total, 3); - assert_eq!(stats.by_phase["rrdp_ok"].count, 1); - assert_eq!(stats.by_phase["rrdp_ok"].duration_ms_total, 10); - assert_eq!(stats.by_phase["rrdp_failed_rsync_failed"].count, 2); - assert_eq!( - stats.by_phase["rrdp_failed_rsync_failed"].duration_ms_total, - 50 - ); - assert_eq!(stats.by_terminal_state["fresh"].count, 1); - assert_eq!(stats.by_terminal_state["failed_no_cache"].count, 2); - assert_eq!( - stats.by_terminal_state["failed_no_cache"].duration_ms_total, - 50 - ); - } -} +#[path = "cli/tests.rs"] +mod tests; diff --git a/src/cli/output.rs b/src/cli/output.rs new file mode 100644 index 0000000..8ebd6b2 --- /dev/null +++ b/src/cli/output.rs @@ -0,0 +1,144 @@ +use std::io::BufWriter; +use std::path::Path; + +use crate::audit::AuditReportV2; +use crate::ccr::canonical_vrp_prefix; + +use super::{PostValidationShared, RunStageTiming}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ReportJsonFormat { + Pretty, + Compact, +} + +pub(super) fn write_json( + path: &Path, + report: &AuditReportV2, + format: ReportJsonFormat, +) -> Result<(), String> { + let f = std::fs::File::create(path) + .map_err(|e| format!("create report file failed: {}: {e}", path.display()))?; + let writer = BufWriter::new(f); + match format { + ReportJsonFormat::Pretty => serde_json::to_writer_pretty(writer, report), + ReportJsonFormat::Compact => serde_json::to_writer(writer, report), + } + .map_err(|e| format!("write report json failed: {e}"))?; + Ok(()) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct CompareViewTaskOutput { + pub(super) build_ms: Option, + pub(super) write_ms: Option, +} + +pub(super) fn run_compare_view_task( + shared: &PostValidationShared, + vrps_csv_out_path: Option<&Path>, + vaps_csv_out_path: Option<&Path>, + trust_anchor: &str, +) -> Result { + let mut build_ms = None; + let mut write_ms = None; + if let (Some(vrps_path), Some(vaps_path)) = (vrps_csv_out_path, vaps_csv_out_path) { + let started = std::time::Instant::now(); + build_ms = Some(0); + write_direct_vrp_csv(vrps_path, shared.vrps.as_ref(), trust_anchor)?; + write_direct_vap_csv(vaps_path, shared.aspas.as_ref(), trust_anchor)?; + write_ms = Some(started.elapsed().as_millis() as u64); + eprintln!( + "wrote compare views: vrps={} vaps={}", + vrps_path.display(), + vaps_path.display() + ); + } + Ok(CompareViewTaskOutput { build_ms, write_ms }) +} + +fn write_direct_vrp_csv( + path: &Path, + vrps: &[crate::validation::objects::Vrp], + trust_anchor: &str, +) -> Result<(), String> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| format!("create parent dirs failed: {}: {e}", parent.display()))?; + } + let file = std::fs::File::create(path) + .map_err(|e| format!("create file failed: {}: {e}", path.display()))?; + let mut writer = BufWriter::new(file); + use std::io::Write; + let trust_anchor = trust_anchor.to_ascii_lowercase(); + writeln!(writer, "ASN,IP Prefix,Max Length,Trust Anchor").map_err(|e| e.to_string())?; + for vrp in vrps { + writeln!( + writer, + "AS{},{},{},{}", + vrp.asn, + canonical_vrp_prefix(&vrp.prefix), + vrp.max_length, + trust_anchor + ) + .map_err(|e| e.to_string())?; + } + Ok(()) +} + +fn write_direct_vap_csv( + path: &Path, + aspas: &[crate::validation::objects::AspaAttestation], + trust_anchor: &str, +) -> Result<(), String> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| format!("create parent dirs failed: {}: {e}", parent.display()))?; + } + let file = std::fs::File::create(path) + .map_err(|e| format!("create file failed: {}: {e}", path.display()))?; + let mut writer = BufWriter::new(file); + use std::io::Write; + let trust_anchor = trust_anchor.to_ascii_lowercase(); + writeln!(writer, "Customer ASN,Providers,Trust Anchor").map_err(|e| e.to_string())?; + for aspa in aspas { + let mut providers = aspa.provider_as_ids.clone(); + providers.sort_unstable(); + providers.dedup(); + let providers = providers + .into_iter() + .map(|asn| format!("AS{asn}")) + .collect::>() + .join(";"); + writeln!( + writer, + "AS{},{},{}", + aspa.customer_as_id, providers, trust_anchor + ) + .map_err(|e| e.to_string())?; + } + Ok(()) +} + +pub(super) fn write_stage_timing( + report_json_path: Option<&Path>, + stage_timing: &RunStageTiming, +) -> Result<(), String> { + if let Some(path) = report_json_path + && let Some(parent) = path.parent() + { + let stage_timing_path = parent.join("stage-timing.json"); + std::fs::write( + &stage_timing_path, + serde_json::to_vec_pretty(stage_timing).map_err(|e| e.to_string())?, + ) + .map_err(|e| { + format!( + "write stage timing failed: {}: {e}", + stage_timing_path.display() + ) + })?; + eprintln!("analysis: wrote {}", stage_timing_path.display()); + } + Ok(()) +} diff --git a/src/cli/tests.rs b/src/cli/tests.rs new file mode 100644 index 0000000..0969261 --- /dev/null +++ b/src/cli/tests.rs @@ -0,0 +1,1592 @@ +use super::*; + +#[test] +fn parse_help_returns_usage() { + let argv = vec!["rpki".to_string(), "--help".to_string()]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("Usage:"), "{err}"); + assert!(err.contains("--db"), "{err}"); + assert!(err.contains("--rsync-mirror-root"), "{err}"); + assert!(err.contains("--rsync-scope"), "{err}"); + assert!(err.contains("--parallel-phase2-object-workers"), "{err}"); + assert!(!err.contains("--parallel-phase1"), "{err}"); + assert!(!err.contains("--parallel-phase2 "), "{err}"); +} + +#[test] +fn parse_rejects_unknown_argument() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--nope".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("unknown argument"), "{err}"); +} + +#[test] +fn parse_rejects_both_tal_url_and_tal_path() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!( + err.contains("one-or-more --tal-url or one-or-more --tal-path/--ta-path pairs"), + "{err}" + ); +} + +#[test] +fn parse_rejects_invalid_max_depth() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--max-depth".to_string(), + "nope".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("invalid --max-depth"), "{err}"); +} + +#[test] +fn parse_accepts_ccr_out_path() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + "--ccr-out".to_string(), + "out/example.ccr".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.ccr_out_path.as_deref(), + Some(std::path::Path::new("out/example.ccr")) + ); +} + +#[test] +fn parse_accepts_report_json_compact_when_report_json_is_set() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--report-json".to_string(), + "out/report.json".to_string(), + "--report-json-compact".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.report_json_path.as_deref(), + Some(std::path::Path::new("out/report.json")) + ); + assert!(args.report_json_compact); +} + +#[test] +fn parse_accepts_rsync_scope_policy() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--rsync-scope".to_string(), + "module-root".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!(args.rsync_scope_policy, RsyncScopePolicy::ModuleRoot); +} + +#[test] +fn parse_rejects_invalid_rsync_scope_policy() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--rsync-scope".to_string(), + "wide".to_string(), + ]; + let err = parse_args(&argv).expect_err("invalid rsync scope should fail"); + assert!(err.contains("invalid --rsync-scope"), "{err}"); +} + +#[test] +fn parse_accepts_strict_policy_list() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--strict".to_string(), + "name,cms-der".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.strict_policy, + Some(StrictPolicy { + name: true, + cms_der: true, + signed_attrs: false, + }) + ); +} + +#[test] +fn parse_accepts_strict_without_value_as_all() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--strict".to_string(), + "--report-json".to_string(), + "out/report.json".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!(args.strict_policy, Some(StrictPolicy::all())); +} + +#[test] +fn parse_rejects_unknown_strict_policy() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--strict=unknown".to_string(), + ]; + let err = parse_args(&argv).expect_err("unknown strict policy should fail"); + assert!(err.contains("unknown strict policy"), "{err}"); +} + +#[test] +fn effective_cir_tal_uris_filters_skipped_multi_tal_inputs() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "afrinic.tal".to_string(), + "--ta-path".to_string(), + "afrinic.cer".to_string(), + "--tal-path".to_string(), + "apnic.tal".to_string(), + "--ta-path".to_string(), + "apnic.cer".to_string(), + "--tal-path".to_string(), + "arin.tal".to_string(), + "--ta-path".to_string(), + "arin.cer".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out.cir".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/afrinic.cer".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/apnic.cer".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/arin.cer".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + let mut shared = synthetic_post_validation_shared(); + shared.discoveries = vec![shared.discovery.clone(), shared.discovery.clone()].into(); + shared.successful_tal_inputs = + vec![args.tal_inputs[0].clone(), args.tal_inputs[2].clone()].into(); + + let effective = effective_cir_tal_uris_for_discoveries( + &args, + &shared, + resolve_cir_export_tal_uris(&args).expect("resolve cir tal uris"), + ) + .expect("map effective cir tal uris"); + + assert_eq!( + effective, + vec![ + "https://example.test/afrinic.cer".to_string(), + "https://example.test/arin.cer".to_string(), + ] + ); +} + +#[test] +fn parse_rejects_report_json_compact_without_report_json() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--report-json-compact".to_string(), + ]; + let err = parse_args(&argv).expect_err("compact flag without report path should fail"); + assert!( + err.contains("--report-json-compact requires --report-json"), + "{err}" + ); +} + +#[test] +fn parse_accepts_skip_report_build_without_report_json() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--ccr-out".to_string(), + "out/result.ccr".to_string(), + "--skip-report-build".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.skip_report_build); + assert_eq!( + args.ccr_out_path.as_deref(), + Some(std::path::Path::new("out/result.ccr")) + ); +} + +#[test] +fn parse_accepts_skip_vcir_persist() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--skip-vcir-persist".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.skip_vcir_persist); +} + +#[test] +fn parse_rejects_skip_report_build_with_report_json() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--report-json".to_string(), + "out/report.json".to_string(), + "--skip-report-build".to_string(), + ]; + let err = parse_args(&argv).expect_err("skip report build with report path should fail"); + assert!( + err.contains("--skip-report-build cannot be combined with --report-json"), + "{err}" + ); +} + +#[test] +fn parse_accepts_direct_compare_view_csv_outputs() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--vrps-csv-out".to_string(), + "out/vrps.csv".to_string(), + "--vaps-csv-out".to_string(), + "out/vaps.csv".to_string(), + "--compare-view-trust-anchor".to_string(), + "unknown".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.vrps_csv_out_path.as_deref(), + Some(std::path::Path::new("out/vrps.csv")) + ); + assert_eq!( + args.vaps_csv_out_path.as_deref(), + Some(std::path::Path::new("out/vaps.csv")) + ); + assert_eq!(args.compare_view_trust_anchor.as_deref(), Some("unknown")); +} + +#[test] +fn parse_rejects_partial_direct_compare_view_csv_outputs() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--vrps-csv-out".to_string(), + "out/vrps.csv".to_string(), + ]; + let err = parse_args(&argv).expect_err("partial direct compare view output should fail"); + assert!( + err.contains("--vrps-csv-out and --vaps-csv-out must be provided together"), + "{err}" + ); +} + +#[test] +fn parse_accepts_external_raw_store_db() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--raw-store-db".to_string(), + "raw-store.db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.raw_store_db.as_deref(), + Some(std::path::Path::new("raw-store.db")) + ); +} + +#[test] +fn parse_accepts_external_repo_bytes_db() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--repo-bytes-db".to_string(), + "repo-bytes.db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.repo_bytes_db.as_deref(), + Some(std::path::Path::new("repo-bytes.db")) + ); +} + +#[test] +fn parse_accepts_cir_enable_with_raw_store_backend() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--raw-store-db".to_string(), + "raw-store.db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/root.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.cir_enabled); + assert_eq!( + args.raw_store_db.as_deref(), + Some(std::path::Path::new("raw-store.db")) + ); + assert_eq!(args.cir_static_root, None); +} + +#[test] +fn parse_accepts_cir_enable_with_required_paths_and_tal_override() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/root.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.cir_enabled); + assert_eq!( + args.cir_out_path.as_deref(), + Some(std::path::Path::new("out/example.cir")) + ); + assert_eq!( + args.cir_tal_uri.as_deref(), + Some("https://example.test/root.tal") + ); + assert_eq!( + args.cir_tal_uris, + vec!["https://example.test/root.tal".to_string()] + ); +} + +#[test] +fn parse_rejects_deprecated_cir_static_root() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + "--cir-static-root".to_string(), + "out/static".to_string(), + ]; + let err = parse_args(&argv).expect_err("cir-static-root should be rejected"); + assert!(err.contains("no longer supported"), "{err}"); +} + +#[test] +fn parse_accepts_default_parallel_config_and_phase2_overrides() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-object-workers".to_string(), + "3".to_string(), + "--parallel-phase2-worker-queue-capacity".to_string(), + "17".to_string(), + "--parallel-phase2-ready-batch-size".to_string(), + "31".to_string(), + "--parallel-phase2-ready-batch-wall-time-budget-ms".to_string(), + "43".to_string(), + "--parallel-phase2-result-drain-batch-size".to_string(), + "37".to_string(), + "--parallel-phase2-finalize-batch-size".to_string(), + "41".to_string(), + "--parallel-phase2-finalize-batch-wall-time-budget-ms".to_string(), + "47".to_string(), + "--parallel-phase2-finalize-queue-capacity".to_string(), + "8192".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!(args.parallel_phase2_config.object_workers, 3); + assert_eq!(args.parallel_phase2_config.worker_queue_capacity, 17); + assert_eq!(args.parallel_phase2_config.ready_batch_size, 31); + assert_eq!( + args.parallel_phase2_config.ready_batch_wall_time_budget_ms, + 43 + ); + assert_eq!( + args.parallel_phase2_config.object_result_drain_batch_size, + 37 + ); + assert_eq!( + args.parallel_phase2_config + .publication_point_finalize_batch_size, + 41 + ); + assert_eq!( + args.parallel_phase2_config + .publication_point_finalize_wall_time_budget_ms, + 47 + ); + assert_eq!( + args.parallel_phase2_config + .publication_point_finalize_queue_capacity, + 8192 + ); + assert_eq!(args.parallel_phase1_config, ParallelPhase1Config::default()); +} + +#[test] +fn parse_rejects_zero_phase2_ready_batch_size() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-ready-batch-size".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero ready batch must fail"); + assert!(err.contains("--parallel-phase2-ready-batch-size"), "{err}"); +} + +#[test] +fn parse_rejects_zero_phase2_result_drain_batch_size() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-result-drain-batch-size".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero result drain batch must fail"); + assert!( + err.contains("--parallel-phase2-result-drain-batch-size"), + "{err}" + ); +} + +#[test] +fn parse_rejects_zero_phase2_ready_batch_wall_time_budget_ms() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-ready-batch-wall-time-budget-ms".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero ready time budget must fail"); + assert!( + err.contains("--parallel-phase2-ready-batch-wall-time-budget-ms"), + "{err}" + ); +} + +#[test] +fn parse_rejects_zero_phase2_finalize_batch_size() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-finalize-batch-size".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero finalize batch must fail"); + assert!( + err.contains("--parallel-phase2-finalize-batch-size"), + "{err}" + ); +} + +#[test] +fn parse_rejects_zero_phase2_finalize_batch_wall_time_budget_ms() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-finalize-batch-wall-time-budget-ms".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero finalize time budget must fail"); + assert!( + err.contains("--parallel-phase2-finalize-batch-wall-time-budget-ms"), + "{err}" + ); +} + +#[test] +fn parse_rejects_zero_phase2_finalize_queue_capacity() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-finalize-queue-capacity".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero finalize queue capacity must fail"); + assert!( + err.contains("--parallel-phase2-finalize-queue-capacity"), + "{err}" + ); +} + +#[test] +fn parse_rejects_removed_parallel_enable_flags() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase1".to_string(), + ]; + let err = parse_args(&argv).expect_err("removed phase flag should fail"); + assert!(err.contains("unknown argument: --parallel-phase1"), "{err}"); + + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2".to_string(), + ]; + let err = parse_args(&argv).expect_err("removed phase flag should fail"); + assert!(err.contains("unknown argument: --parallel-phase2"), "{err}"); +} + +#[test] +fn parse_accepts_multi_tal_cir_overrides_in_file_mode() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "apnic.tal".to_string(), + "--ta-path".to_string(), + "apnic.cer".to_string(), + "--tal-path".to_string(), + "arin.tal".to_string(), + "--ta-path".to_string(), + "arin.cer".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/apnic.tal".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/arin.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.cir_tal_uris, + vec![ + "https://example.test/apnic.tal".to_string(), + "https://example.test/arin.tal".to_string() + ] + ); +} + +#[test] +fn parse_rejects_incomplete_or_invalid_cir_flags() { + let argv_missing = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--cir-enable".to_string(), + ]; + let err = parse_args(&argv_missing).unwrap_err(); + assert!(err.contains("--cir-enable requires --cir-out"), "{err}"); + + let argv_needs_enable = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + ]; + let err = parse_args(&argv_needs_enable).unwrap_err(); + assert!(err.contains("require --cir-enable"), "{err}"); + + let argv_offline_missing_uri = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + ]; + let err = parse_args(&argv_offline_missing_uri).unwrap_err(); + assert!(err.contains("requires --cir-tal-uri"), "{err}"); +} + +#[test] +fn parse_rejects_invalid_validation_time() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--validation-time".to_string(), + "not-a-time".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("invalid --validation-time"), "{err}"); +} + +#[test] +fn parse_rejects_invalid_max_instances() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--max-instances".to_string(), + "nope".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("invalid --max-instances"), "{err}"); +} + +#[test] +fn parse_rejects_missing_value_for_db() { + let argv = vec!["rpki".to_string(), "--db".to_string()]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("--db requires a value"), "{err}"); +} + +#[test] +fn parse_rejects_missing_value_for_tal_url() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("--tal-url requires a value"), "{err}"); +} + +#[test] +fn parse_rejects_missing_db() { + let argv = vec!["rpki".to_string(), "--tal-url".to_string(), "x".to_string()]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("--db is required"), "{err}"); +} + +#[test] +fn parse_rejects_missing_tal_mode() { + let argv = vec!["rpki".to_string(), "--db".to_string(), "db".to_string()]; + let err = parse_args(&argv).unwrap_err(); + assert!( + err.contains("--tal-url") || err.contains("--tal-path"), + "{err}" + ); +} + +#[test] +fn parse_accepts_tal_url_mode() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_url.as_deref(), Some("https://example.test/x.tal")); + assert_eq!( + args.tal_urls, + vec!["https://example.test/x.tal".to_string()] + ); + assert!(args.tal_path.is_none()); + assert!(args.ta_path.is_none()); + assert_eq!(args.tal_inputs.len(), 1); + assert_eq!(args.tal_inputs[0].tal_id, "x"); + assert_eq!(args.parallel_phase1_config, ParallelPhase1Config::default()); + assert_eq!(args.parallel_phase2_config, ParallelPhase2Config::default()); +} + +#[test] +fn parse_accepts_multi_tal_without_parallel_flags() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/arin.tal".to_string(), + "--tal-url".to_string(), + "https://example.test/apnic.tal".to_string(), + "--tal-url".to_string(), + "https://example.test/ripe.tal".to_string(), + "--parallel-max-repo-sync-workers-global".to_string(), + "8".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_urls.len(), 3); + assert_eq!(args.tal_inputs.len(), 3); + assert_eq!(args.tal_inputs[0].tal_id, "arin"); + assert_eq!(args.tal_inputs[1].tal_id, "apnic"); + assert_eq!(args.tal_inputs[2].tal_id, "ripe"); + assert_eq!(args.parallel_phase1_config.max_repo_sync_workers_global, 8); +} + +#[test] +fn parse_accepts_multi_tal_urls_by_default() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/arin.tal".to_string(), + "--tal-url".to_string(), + "https://example.test/apnic.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_urls.len(), 2); + assert_eq!(args.tal_inputs.len(), 2); +} + +#[test] +fn parse_accepts_offline_mode_requires_ta() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--max-depth".to_string(), + "0".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_paths, vec![PathBuf::from("a.tal")]); + assert_eq!(args.ta_paths, vec![PathBuf::from("ta.cer")]); + assert_eq!(args.tal_path.as_deref(), Some(Path::new("a.tal"))); + assert_eq!(args.ta_path.as_deref(), Some(Path::new("ta.cer"))); + assert_eq!(args.max_depth, Some(0)); +} + +#[test] +fn parse_accepts_multiple_tal_path_pairs_by_default() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "apnic.tal".to_string(), + "--ta-path".to_string(), + "apnic-ta.cer".to_string(), + "--tal-path".to_string(), + "arin.tal".to_string(), + "--ta-path".to_string(), + "arin-ta.cer".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_paths.len(), 2); + assert_eq!(args.ta_paths.len(), 2); + assert_eq!(args.tal_inputs.len(), 2); +} + +#[test] +fn parse_rejects_mixed_tal_url_and_tal_path_modes() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/arin.tal".to_string(), + "--tal-path".to_string(), + "apnic.tal".to_string(), + "--ta-path".to_string(), + "apnic-ta.cer".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!( + err.contains( + "must specify either one-or-more --tal-url or one-or-more --tal-path/--ta-path pairs" + ), + "{err}" + ); +} + +#[test] +fn parse_rejects_mismatched_tal_path_and_ta_path_counts() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "apnic.tal".to_string(), + "--tal-path".to_string(), + "arin.tal".to_string(), + "--ta-path".to_string(), + "apnic-ta.cer".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!( + err.contains("--tal-path and --ta-path counts must match"), + "{err}" + ); +} + +#[test] +fn parse_accepts_tal_path_without_ta_when_disable_rrdp_is_set() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--disable-rrdp".to_string(), + "--rsync-command".to_string(), + "/tmp/fake-rsync".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_path.as_deref(), Some(Path::new("a.tal"))); + assert!(args.ta_path.is_none()); + assert!(args.disable_rrdp); + assert_eq!( + args.rsync_command.as_deref(), + Some(Path::new("/tmp/fake-rsync")) + ); +} + +#[test] +fn parse_accepts_multiple_tal_paths_without_ta_when_disable_rrdp() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--tal-path".to_string(), + "b.tal".to_string(), + "--disable-rrdp".to_string(), + "--rsync-command".to_string(), + "/tmp/fake-rsync".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!( + args.tal_paths, + vec![PathBuf::from("a.tal"), PathBuf::from("b.tal")] + ); + assert!(args.ta_paths.is_empty()); + assert_eq!(args.tal_inputs.len(), 2); + assert!(args.disable_rrdp); +} + +#[test] +fn parse_accepts_payload_delta_replay_mode_with_offline_tal_and_ta() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-base-archive".to_string(), + "base-archive".to_string(), + "--payload-base-locks".to_string(), + "base-locks.json".to_string(), + "--payload-delta-archive".to_string(), + "delta-archive".to_string(), + "--payload-delta-locks".to_string(), + "delta-locks.json".to_string(), + ]; + let args = parse_args(&argv).expect("parse delta replay mode"); + assert_eq!( + args.payload_base_archive.as_deref(), + Some(Path::new("base-archive")) + ); + assert_eq!( + args.payload_base_locks.as_deref(), + Some(Path::new("base-locks.json")) + ); + assert_eq!( + args.payload_delta_archive.as_deref(), + Some(Path::new("delta-archive")) + ); + assert_eq!( + args.payload_delta_locks.as_deref(), + Some(Path::new("delta-locks.json")) + ); +} + +#[test] +fn parse_rejects_partial_payload_delta_arguments_and_mutual_exclusion() { + let argv_partial = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-base-archive".to_string(), + "base-archive".to_string(), + ]; + let err = parse_args(&argv_partial).unwrap_err(); + assert!(err.contains("must be provided together"), "{err}"); + + let argv_both = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-replay-archive".to_string(), + "archive".to_string(), + "--payload-replay-locks".to_string(), + "locks.json".to_string(), + "--payload-base-archive".to_string(), + "base-archive".to_string(), + "--payload-base-locks".to_string(), + "base-locks.json".to_string(), + "--payload-delta-archive".to_string(), + "delta-archive".to_string(), + "--payload-delta-locks".to_string(), + "delta-locks.json".to_string(), + ]; + let err = parse_args(&argv_both).unwrap_err(); + assert!(err.contains("mutually exclusive"), "{err}"); +} + +#[test] +fn parse_rejects_payload_delta_with_tal_url_or_rsync_local_dir() { + let argv_url = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--payload-base-archive".to_string(), + "base-archive".to_string(), + "--payload-base-locks".to_string(), + "base-locks.json".to_string(), + "--payload-delta-archive".to_string(), + "delta-archive".to_string(), + "--payload-delta-locks".to_string(), + "delta-locks.json".to_string(), + ]; + let err = parse_args(&argv_url).unwrap_err(); + assert!(err.contains("--tal-url is not supported"), "{err}"); + + let argv_rsync = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-base-archive".to_string(), + "base-archive".to_string(), + "--payload-base-locks".to_string(), + "base-locks.json".to_string(), + "--payload-delta-archive".to_string(), + "delta-archive".to_string(), + "--payload-delta-locks".to_string(), + "delta-locks.json".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + ]; + let err = parse_args(&argv_rsync).unwrap_err(); + assert!( + err.contains("payload delta replay mode cannot be combined with --rsync-local-dir"), + "{err}" + ); +} + +#[test] +fn parse_accepts_payload_replay_mode_with_offline_tal_and_ta() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-replay-archive".to_string(), + "archive".to_string(), + "--payload-replay-locks".to_string(), + "locks.json".to_string(), + ]; + let args = parse_args(&argv).expect("parse replay mode"); + assert_eq!( + args.payload_replay_archive.as_deref(), + Some(Path::new("archive")) + ); + assert_eq!( + args.payload_replay_locks.as_deref(), + Some(Path::new("locks.json")) + ); +} + +#[test] +fn parse_rejects_partial_payload_replay_arguments() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-replay-archive".to_string(), + "archive".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("must be provided together"), "{err}"); +} + +#[test] +fn parse_rejects_payload_replay_with_tal_url_or_rsync_local_dir() { + let argv_url = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--payload-replay-archive".to_string(), + "archive".to_string(), + "--payload-replay-locks".to_string(), + "locks.json".to_string(), + ]; + let err = parse_args(&argv_url).unwrap_err(); + assert!(err.contains("--tal-url is not supported"), "{err}"); + + let argv_rsync = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-replay-archive".to_string(), + "archive".to_string(), + "--payload-replay-locks".to_string(), + "locks.json".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + ]; + let err = parse_args(&argv_rsync).unwrap_err(); + assert!( + err.contains("cannot be combined with --rsync-local-dir"), + "{err}" + ); +} + +#[test] +fn parse_accepts_validation_time_rfc3339() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--validation-time".to_string(), + "2026-01-01T00:00:00Z".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert!(args.validation_time.is_some()); +} + +#[test] +fn parse_rejects_removed_revalidate_only_flag() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--revalidate-only".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("unknown argument: --revalidate-only"), "{err}"); +} + +#[test] +fn read_policy_accepts_valid_toml() { + let dir = tempfile::tempdir().expect("tmpdir"); + let p = dir.path().join("policy.toml"); + std::fs::write( + &p, + "signed_object_failure_policy = \"drop_publication_point\"\n", + ) + .expect("write policy"); + + let policy = read_policy(Some(&p)).expect("parse policy"); + assert_eq!( + policy.signed_object_failure_policy, + crate::policy::SignedObjectFailurePolicy::DropPublicationPoint + ); + assert_eq!(policy.strict, StrictPolicy::default()); +} + +#[test] +fn read_policy_accepts_strict_table() { + let dir = tempfile::tempdir().expect("tmpdir"); + let p = dir.path().join("policy.toml"); + std::fs::write( + &p, + r#" + [strict] + name = true + cms_der = true + "#, + ) + .expect("write policy"); + + let policy = read_policy(Some(&p)).expect("parse policy"); + assert_eq!( + policy.strict, + StrictPolicy { + name: true, + cms_der: true, + signed_attrs: false, + } + ); +} + +#[test] +fn read_policy_reports_missing_file() { + let dir = tempfile::tempdir().expect("tmpdir"); + let p = dir.path().join("missing.toml"); + let err = read_policy(Some(&p)).unwrap_err(); + assert!(err.contains("read policy file failed"), "{err}"); +} + +fn synthetic_post_validation_shared() -> PostValidationShared { + let tal_bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/tal/apnic-rfc7730-https.tal"), + ) + .expect("read tal fixture"); + let ta_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ta/apnic-ta.cer"), + ) + .expect("read ta fixture"); + + let discovery = crate::validation::from_tal::discover_root_ca_instance_from_tal_and_ta_der( + &tal_bytes, &ta_der, None, + ) + .expect("discover root"); + + let tree = crate::validation::tree::TreeRunOutput { + instances_processed: 1, + instances_failed: 0, + warnings: vec![ + crate::report::Warning::new("synthetic warning") + .with_rfc_refs(&[crate::report::RfcRef("RFC 6487 §4.8.8.1")]) + .with_context("rsync://example.test/repo/pp/"), + ], + vrps: vec![ + crate::validation::objects::Vrp { + asn: 64496, + prefix: crate::data_model::roa::IpPrefix { + afi: crate::data_model::roa::RoaAfi::Ipv4, + prefix_len: 24, + addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + max_length: 24, + }, + crate::validation::objects::Vrp { + asn: 64497, + prefix: crate::data_model::roa::IpPrefix { + afi: crate::data_model::roa::RoaAfi::Ipv6, + prefix_len: 48, + addr: [0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + max_length: 64, + }, + ], + aspas: vec![crate::validation::objects::AspaAttestation { + customer_as_id: 64496, + provider_as_ids: vec![64497, 64498], + }], + router_keys: Vec::new(), + }; + + let mut pp1 = crate::audit::PublicationPointAudit::default(); + pp1.source = "fresh".to_string(); + pp1.rrdp_notification_uri = Some("https://example.test/n1.xml".to_string()); + let mut pp2 = crate::audit::PublicationPointAudit::default(); + pp2.source = "fresh".to_string(); + pp2.rrdp_notification_uri = Some("https://example.test/n1.xml".to_string()); + let mut pp3 = crate::audit::PublicationPointAudit::default(); + pp3.source = "fresh".to_string(); + pp3.rrdp_notification_uri = Some("https://example.test/n2.xml".to_string()); + + let out = crate::validation::run_tree_from_tal::RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points: vec![pp1, pp2, pp3], + downloads: Vec::new(), + download_stats: crate::audit::AuditDownloadStats::default(), + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }; + PostValidationShared::from_run_output(out) +} + +fn sample_cli_ccr_accumulator() -> CcrAccumulator { + let tal_bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/tal/apnic-rfc7730-https.tal"), + ) + .expect("read tal fixture"); + let ta_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ta/apnic-ta.cer"), + ) + .expect("read ta fixture"); + let discovery = crate::validation::from_tal::discover_root_ca_instance_from_tal_and_ta_der( + &tal_bytes, &ta_der, None, + ) + .expect("discover root"); + let mut accumulator = CcrAccumulator::new(vec![discovery.trust_anchor.clone()]); + let projection = crate::storage::VcirCcrManifestProjection { + manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), + manifest_sha256: vec![0x44; 32], + manifest_size: 2048, + manifest_ee_aki: vec![0x55; 20], + manifest_number_be: vec![1], + manifest_this_update: crate::storage::PackTime::from_utc_offset_datetime( + time::OffsetDateTime::now_utc(), + ), + manifest_sia_locations_der: vec![vec![ + 0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05, + b'r', b's', b'y', b'n', b'c', + ]], + subordinate_skis: vec![vec![0x33; 20]], + }; + accumulator + .append_manifest_projection(&projection) + .expect("append manifest projection"); + accumulator +} + +#[test] +fn build_report_and_helpers_work_on_synthetic_output() { + let shared = synthetic_post_validation_shared(); + let policy = Policy::default(); + let validation_time = time::OffsetDateTime::now_utc(); + let report = build_report(&policy, validation_time, &shared); + + assert_eq!(unique_rrdp_repos(&report), 2); + assert_eq!(report.vrps.len(), 2); + assert_eq!(report.aspas.len(), 1); + + print_summary(&report); +} + +#[test] +fn run_report_task_and_stage_timing_work() { + let shared = synthetic_post_validation_shared(); + let policy = Policy::default(); + let validation_time = time::OffsetDateTime::now_utc(); + let dir = tempfile::tempdir().expect("tmpdir"); + let report_path = dir.path().join("report.json"); + let report_output = run_report_task( + &policy, + validation_time, + &shared, + Some(&report_path), + ReportJsonFormat::Compact, + ) + .expect("run report task"); + + let report = report_output.report.as_ref().expect("report built"); + assert_eq!(report.vrps.len(), 2); + assert_eq!(report.aspas.len(), 1); + assert!(report_output.report_write_ms.is_some()); + + let report_json = std::fs::read_to_string(&report_path).expect("read report json"); + assert!(!report_json.contains('\n'), "{report_json}"); + + let stage_timing = RunStageTiming { + validation_ms: 1, + report_build_ms: report_output.report_build_ms, + report_write_ms: report_output.report_write_ms, + ccr_build_ms: Some(2), + ccr_build_breakdown: None, + ccr_write_ms: Some(3), + compare_view_build_ms: Some(4), + compare_view_write_ms: Some(5), + cir_build_cir_ms: Some(6), + cir_write_cir_ms: Some(7), + cir_total_ms: Some(8), + total_ms: 9, + publication_points: shared.publication_points.len(), + repo_sync_ms_total: 10, + publication_point_repo_sync_ms_total: 11, + download_event_count: 12, + rrdp_download_ms_total: 13, + rsync_download_ms_total: 14, + download_bytes_total: 15, + }; + write_stage_timing(Some(&report_path), &stage_timing).expect("write stage timing"); + let stage_timing_json = + std::fs::read_to_string(dir.path().join("stage-timing.json")).expect("read timing"); + assert!(stage_timing_json.contains("\"validation_ms\"")); + assert!(stage_timing_json.contains("\"ccr_build_ms\"")); + + let ccr_path = dir.path().join("result.ccr"); + write_stage_timing(Some(&ccr_path), &stage_timing).expect("write stage timing via ccr path"); + assert!( + dir.path().join("stage-timing.json").exists(), + "stage timing should use parent directory of the anchor path" + ); + + let skipped = ReportTaskOutput::skipped(); + assert!(skipped.report.is_none()); + assert_eq!(skipped.report_build_ms, 0); + assert!(skipped.report_write_ms.is_none()); +} + +#[test] +fn run_compare_view_task_writes_csv_from_shared_output() { + let shared = synthetic_post_validation_shared(); + let dir = tempfile::tempdir().expect("tmpdir"); + let vrps_path = dir.path().join("vrps.csv"); + let vaps_path = dir.path().join("vaps.csv"); + + let output = run_compare_view_task(&shared, Some(&vrps_path), Some(&vaps_path), "unknown") + .expect("write direct compare views"); + + assert!(output.build_ms.is_some()); + assert!(output.write_ms.is_some()); + let vrps_csv = std::fs::read_to_string(vrps_path).expect("read vrps csv"); + let vaps_csv = std::fs::read_to_string(vaps_path).expect("read vaps csv"); + assert!(vrps_csv.contains("ASN,IP Prefix,Max Length,Trust Anchor")); + assert!(vrps_csv.contains("AS64496,192.0.2.0/24,24,unknown")); + assert!(vrps_csv.contains("AS64497,2001:db8::/48,64,unknown")); + assert!(vaps_csv.contains("Customer ASN,Providers,Trust Anchor")); + assert!(vaps_csv.contains("AS64496,AS64497;AS64498,unknown")); +} + +#[test] +fn run_ccr_task_uses_accumulator_when_phase2_output_contains_reuse_sources() { + let mut shared = synthetic_post_validation_shared(); + shared.ccr_accumulator = Some(sample_cli_ccr_accumulator()); + let mut publication_points = shared + .publication_points + .iter() + .cloned() + .collect::>(); + publication_points[1].source = "vcir_current_instance".to_string(); + publication_points[2].source = "failed_no_cache".to_string(); + shared.publication_points = publication_points.into(); + let dir = tempfile::tempdir().expect("tmpdir"); + let ccr_path = dir.path().join("result.ccr"); + let store = RocksStore::open(&dir.path().join("db")).expect("open empty store"); + + let output = run_ccr_task( + &store, + &shared, + Some(&ccr_path), + time::OffsetDateTime::now_utc(), + ) + .expect("run ccr task"); + + assert!(output.ccr_build_ms.is_some()); + assert!(output.ccr_build_breakdown.is_none()); + let der = std::fs::read(&ccr_path).expect("read ccr"); + let ci = crate::ccr::decode_content_info(&der).expect("decode ccr"); + assert_eq!( + ci.content + .mfts + .as_ref() + .map(|manifest_state| manifest_state.mis.len()), + Some(1) + ); +} + +#[test] +fn write_json_writes_report() { + let report = AuditReportV2 { + format_version: 2, + meta: AuditRunMeta { + validation_time_rfc3339_utc: "2026-01-01T00:00:00Z".to_string(), + }, + policy: Policy::default(), + tree: TreeSummary { + instances_processed: 0, + instances_failed: 0, + warnings: Vec::new(), + }, + publication_points: Vec::new(), + vrps: Vec::new(), + aspas: Vec::new(), + downloads: Vec::new(), + download_stats: crate::audit::AuditDownloadStats::default(), + repo_sync_stats: crate::audit::AuditRepoSyncStats::default(), + }; + + let dir = tempfile::tempdir().expect("tmpdir"); + let pretty_path = dir.path().join("report-pretty.json"); + write_json(&pretty_path, &report, ReportJsonFormat::Pretty).expect("write pretty json"); + let pretty = std::fs::read_to_string(&pretty_path).expect("read pretty report"); + assert!(pretty.contains("\"format_version\"")); + assert!(pretty.contains("\"policy\"")); + assert!(pretty.contains("\n \"format_version\""), "{pretty}"); + + let compact_path = dir.path().join("report-compact.json"); + write_json(&compact_path, &report, ReportJsonFormat::Compact).expect("write compact json"); + let compact = std::fs::read_to_string(&compact_path).expect("read compact report"); + assert!(compact.contains("\"format_version\"")); + assert!(compact.contains("\"policy\"")); + assert!(!compact.contains('\n'), "{compact}"); +} + +#[test] +fn build_repo_sync_stats_aggregates_phase_and_terminal_state() { + let mut pp1 = crate::audit::PublicationPointAudit::default(); + pp1.repo_sync_phase = Some("rrdp_ok".to_string()); + pp1.repo_sync_duration_ms = Some(10); + pp1.repo_terminal_state = "fresh".to_string(); + + let mut pp2 = crate::audit::PublicationPointAudit::default(); + pp2.repo_sync_phase = Some("rrdp_failed_rsync_failed".to_string()); + pp2.repo_sync_duration_ms = Some(20); + pp2.repo_terminal_state = "failed_no_cache".to_string(); + + let mut pp3 = crate::audit::PublicationPointAudit::default(); + pp3.repo_sync_phase = Some("rrdp_failed_rsync_failed".to_string()); + pp3.repo_sync_duration_ms = Some(30); + pp3.repo_terminal_state = "failed_no_cache".to_string(); + + let stats = build_repo_sync_stats(&[pp1, pp2, pp3]); + assert_eq!(stats.publication_points_total, 3); + assert_eq!(stats.by_phase["rrdp_ok"].count, 1); + assert_eq!(stats.by_phase["rrdp_ok"].duration_ms_total, 10); + assert_eq!(stats.by_phase["rrdp_failed_rsync_failed"].count, 2); + assert_eq!( + stats.by_phase["rrdp_failed_rsync_failed"].duration_ms_total, + 50 + ); + assert_eq!(stats.by_terminal_state["fresh"].count, 1); + assert_eq!(stats.by_terminal_state["failed_no_cache"].count, 2); + assert_eq!( + stats.by_terminal_state["failed_no_cache"].duration_ms_total, + 50 + ); +} diff --git a/src/lib.rs b/src/lib.rs index 16e8caf..427b4f9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,4 +33,6 @@ pub mod storage; #[cfg(feature = "full")] pub mod sync; #[cfg(feature = "full")] +pub mod tools; +#[cfg(feature = "full")] pub mod validation; diff --git a/src/storage.rs b/src/storage.rs index 4fd3349..b4b228b 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -1,126 +1,25 @@ +mod config; +mod keys; +mod pack; + use std::collections::HashSet; use std::path::Path; -use rocksdb::{ - ColumnFamily, ColumnFamilyDescriptor, DB, DBCompressionType, Direction, IteratorMode, Options, - WriteBatch, -}; -use serde::{Deserialize, Serialize, de::DeserializeOwned}; -use sha2::Digest; +use rocksdb::{ColumnFamily, DB, Direction, IteratorMode, Options, WriteBatch}; +use serde::{Deserialize, Serialize}; use crate::blob_store::{ExternalRawStoreDb, ExternalRepoBytesDb, RawObjectStore}; -use crate::data_model::common::der_take_tlv; use crate::data_model::rc::{AsResourceSet, IpResourceSet}; -pub const CF_REPOSITORY_VIEW: &str = "repository_view"; -pub const CF_RAW_BY_HASH: &str = "raw_by_hash"; -pub const CF_RAW_BLOB: &str = "raw_blob"; -pub const CF_VCIR: &str = "vcir"; -pub const CF_MANIFEST_REPLAY_META: &str = "manifest_replay_meta"; -pub const CF_AUDIT_RULE_INDEX: &str = "audit_rule_index"; -pub const CF_RRDP_SOURCE: &str = "rrdp_source"; -pub const CF_RRDP_SOURCE_MEMBER: &str = "rrdp_source_member"; -pub const CF_RRDP_URI_OWNER: &str = "rrdp_uri_owner"; - -pub const ALL_COLUMN_FAMILY_NAMES: &[&str] = &[ - CF_REPOSITORY_VIEW, - CF_RAW_BY_HASH, - CF_RAW_BLOB, - CF_VCIR, - CF_MANIFEST_REPLAY_META, - CF_AUDIT_RULE_INDEX, - CF_RRDP_SOURCE, - CF_RRDP_SOURCE_MEMBER, - CF_RRDP_URI_OWNER, -]; - -const REPOSITORY_VIEW_KEY_PREFIX: &str = "repo_view:"; -const RAW_BY_HASH_KEY_PREFIX: &str = "rawbyhash:"; -const RAW_BLOB_KEY_PREFIX: &str = "rawblob:"; -const VCIR_KEY_PREFIX: &str = "vcir:"; -const MANIFEST_REPLAY_META_KEY_PREFIX: &str = "manifest_replay_meta:"; -const AUDIT_ROA_RULE_KEY_PREFIX: &str = "audit:roa_rule:"; -const AUDIT_ASPA_RULE_KEY_PREFIX: &str = "audit:aspa_rule:"; -const AUDIT_ROUTER_KEY_RULE_KEY_PREFIX: &str = "audit:router_key_rule:"; -const RRDP_SOURCE_KEY_PREFIX: &str = "rrdp_source:"; -const RRDP_SOURCE_MEMBER_KEY_PREFIX: &str = "rrdp_source_member:"; -const RRDP_URI_OWNER_KEY_PREFIX: &str = "rrdp_uri_owner:"; - -const WORK_DB_BLOB_MODE_ENV: &str = "RPKI_WORK_DB_BLOB_MODE"; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum WorkDbBlobMode { - Current, - Disabled, - Lz4, -} - -fn parse_work_db_blob_mode(raw: &str) -> Option { - match raw.trim().to_ascii_lowercase().as_str() { - "" | "default" => Some(default_work_db_blob_mode()), - "current" | "legacy" => Some(WorkDbBlobMode::Current), - "disabled" | "disable" | "off" | "none" | "no_blob" | "no-blob" => { - Some(WorkDbBlobMode::Disabled) - } - "lz4" | "blob_lz4" | "blob-lz4" => Some(WorkDbBlobMode::Lz4), - _ => None, - } -} - -fn default_work_db_blob_mode() -> WorkDbBlobMode { - WorkDbBlobMode::Disabled -} - -fn work_db_blob_mode_from_env() -> WorkDbBlobMode { - let Ok(raw) = std::env::var(WORK_DB_BLOB_MODE_ENV) else { - return default_work_db_blob_mode(); - }; - match parse_work_db_blob_mode(&raw) { - Some(mode) => mode, - None => { - eprintln!( - "warning: unsupported {WORK_DB_BLOB_MODE_ENV}={raw:?}; using default work-db blobdb mode" - ); - default_work_db_blob_mode() - } - } -} - -fn configure_work_db_options(opts: &mut Options, blob_mode: WorkDbBlobMode) { - opts.set_compression_type(DBCompressionType::Lz4); - match blob_mode { - WorkDbBlobMode::Current => enable_blobdb_current(opts), - WorkDbBlobMode::Disabled => {} - WorkDbBlobMode::Lz4 => { - enable_blobdb_current(opts); - opts.set_blob_compression_type(DBCompressionType::Lz4); - } - } -} - -fn cf_opts(blob_mode: WorkDbBlobMode) -> Options { - let mut opts = Options::default(); - configure_work_db_options(&mut opts, blob_mode); - opts -} - -fn raw_blob_key(sha256_hex: &str) -> String { - format!("{RAW_BLOB_KEY_PREFIX}{sha256_hex}") -} - -pub fn column_family_descriptors() -> Vec { - column_family_descriptors_for_blob_mode(work_db_blob_mode_from_env()) -} - -fn column_family_descriptors_for_blob_mode( - blob_mode: WorkDbBlobMode, -) -> Vec { - ALL_COLUMN_FAMILY_NAMES - .iter() - .map(|name| ColumnFamilyDescriptor::new(*name, cf_opts(blob_mode))) - .collect() -} - +use config::*; +pub use config::{ + ALL_COLUMN_FAMILY_NAMES, CF_AUDIT_RULE_INDEX, CF_MANIFEST_REPLAY_META, CF_RAW_BY_HASH, + CF_REPOSITORY_VIEW, CF_RRDP_SOURCE, CF_RRDP_SOURCE_MEMBER, CF_RRDP_URI_OWNER, CF_VCIR, + column_family_descriptors, +}; +use keys::*; +use pack::compute_sha256_32; +pub use pack::{PackBytes, PackFile, PackTime}; #[derive(Debug, thiserror::Error)] pub enum StorageError { #[error("rocksdb error: {0}")] @@ -150,10 +49,6 @@ pub struct RocksStore { external_repo_bytes: Option, } -pub mod pack { - pub use super::{PackFile, PackTime}; -} - #[derive(Clone, Debug, PartialEq, Eq)] pub enum RrdpDeltaOp { Upsert { rsync_uri: String, bytes: Vec }, @@ -1691,1854 +1586,6 @@ impl RocksStore { } } -fn repository_view_key(rsync_uri: &str) -> String { - format!("{REPOSITORY_VIEW_KEY_PREFIX}{rsync_uri}") -} - -fn repository_view_prefix(rsync_uri_prefix: &str) -> String { - format!("{REPOSITORY_VIEW_KEY_PREFIX}{rsync_uri_prefix}") -} - -fn raw_by_hash_key(sha256_hex: &str) -> String { - format!("{RAW_BY_HASH_KEY_PREFIX}{sha256_hex}") -} - -fn vcir_key(manifest_rsync_uri: &str) -> String { - format!("{VCIR_KEY_PREFIX}{manifest_rsync_uri}") -} - -fn manifest_replay_meta_key(manifest_rsync_uri: &str) -> String { - format!("{MANIFEST_REPLAY_META_KEY_PREFIX}{manifest_rsync_uri}") -} - -fn audit_rule_kind_for_output_type(output_type: VcirOutputType) -> Option { - match output_type { - VcirOutputType::Vrp => Some(AuditRuleKind::Roa), - VcirOutputType::Aspa => Some(AuditRuleKind::Aspa), - VcirOutputType::RouterKey => Some(AuditRuleKind::RouterKey), - } -} - -fn audit_rule_key(kind: AuditRuleKind, rule_hash: &str) -> String { - format!("{}{rule_hash}", kind.key_prefix()) -} - -fn rrdp_source_key(notify_uri: &str) -> String { - format!("{RRDP_SOURCE_KEY_PREFIX}{notify_uri}") -} - -fn rrdp_source_member_key(notify_uri: &str, rsync_uri: &str) -> String { - format!("{RRDP_SOURCE_MEMBER_KEY_PREFIX}{notify_uri}:{rsync_uri}") -} - -fn rrdp_source_member_prefix(notify_uri: &str) -> String { - format!("{RRDP_SOURCE_MEMBER_KEY_PREFIX}{notify_uri}:") -} - -fn rrdp_uri_owner_key(rsync_uri: &str) -> String { - format!("{RRDP_URI_OWNER_KEY_PREFIX}{rsync_uri}") -} - -fn encode_cbor(value: &T, entity: &'static str) -> StorageResult> { - serde_cbor::to_vec(value).map_err(|e| StorageError::Codec { - entity, - detail: e.to_string(), - }) -} - -fn decode_cbor(bytes: &[u8], entity: &'static str) -> StorageResult { - serde_cbor::from_slice(bytes).map_err(|e| StorageError::Codec { - entity, - detail: e.to_string(), - }) -} - -fn validate_non_empty(field: &'static str, value: &str) -> StorageResult<()> { - if value.is_empty() { - return Err(StorageError::InvalidData { - entity: field, - detail: "must not be empty".to_string(), - }); - } - Ok(()) -} - -fn validate_sha256_hex(field: &'static str, value: &str) -> StorageResult<()> { - if value.len() != 64 || !value.as_bytes().iter().all(u8::is_ascii_hexdigit) { - return Err(StorageError::InvalidData { - entity: field, - detail: "must be a 64-character lowercase or uppercase SHA-256 hex string".to_string(), - }); - } - Ok(()) -} - -fn decode_sha256_hex_32(field: &'static str, value: &str) -> StorageResult<[u8; 32]> { - validate_sha256_hex(field, value)?; - let mut out = [0u8; 32]; - hex::decode_to_slice(value, &mut out).map_err(|e| StorageError::InvalidData { - entity: field, - detail: format!("hex decode failed: {e}"), - })?; - Ok(out) -} - -fn validate_manifest_number_be(field: &'static str, value: &[u8]) -> StorageResult<()> { - if value.is_empty() { - return Err(StorageError::InvalidData { - entity: field, - detail: "must not be empty".to_string(), - }); - } - if value.len() > 20 { - return Err(StorageError::InvalidData { - entity: field, - detail: "must be at most 20 octets".to_string(), - }); - } - if value.len() > 1 && value[0] == 0 { - return Err(StorageError::InvalidData { - entity: field, - detail: "must be minimal big-endian without leading zeros".to_string(), - }); - } - Ok(()) -} - -fn validate_sha256_digest_bytes(field: &'static str, value: &[u8]) -> StorageResult<()> { - if value.len() != 32 { - return Err(StorageError::InvalidData { - entity: field, - detail: format!("must be 32 bytes, got {}", value.len()), - }); - } - Ok(()) -} - -fn validate_fixed_len_bytes( - field: &'static str, - value: &[u8], - expected_len: usize, -) -> StorageResult<()> { - if value.len() != expected_len { - return Err(StorageError::InvalidData { - entity: field, - detail: format!("must be {expected_len} bytes, got {}", value.len()), - }); - } - Ok(()) -} - -fn validate_sorted_unique_fixed_len_bytes( - field: &'static str, - values: &[Vec], - expected_len: usize, -) -> StorageResult<()> { - for value in values { - validate_fixed_len_bytes(field, value, expected_len)?; - } - for window in values.windows(2) { - if window[0] >= window[1] { - return Err(StorageError::InvalidData { - entity: field, - detail: "must be strictly sorted and unique".to_string(), - }); - } - } - Ok(()) -} - -fn validate_full_der_with_tag( - field: &'static str, - der: &[u8], - expected_tag: Option, -) -> StorageResult<()> { - let (tag, _value, rem) = der_take_tlv(der).map_err(|detail| StorageError::InvalidData { - entity: field, - detail, - })?; - if !rem.is_empty() { - return Err(StorageError::InvalidData { - entity: field, - detail: "trailing bytes after DER object".to_string(), - }); - } - if let Some(expected_tag) = expected_tag { - if tag != expected_tag { - return Err(StorageError::InvalidData { - entity: field, - detail: format!("unexpected tag 0x{tag:02X}, expected 0x{expected_tag:02X}"), - }); - } - } - Ok(()) -} - -fn parse_time(field: &'static str, value: &PackTime) -> StorageResult { - value.parse().map_err(|detail| StorageError::InvalidData { - entity: field, - detail, - }) -} - -fn enable_blobdb_current(opts: &mut Options) { - #[allow(unused_mut)] - let mut _enabled = false; - - #[allow(dead_code)] - fn _set(opts: &mut Options) { - opts.set_enable_blob_files(true); - opts.set_min_blob_size(1024); - } - - _set(opts); -} - -#[derive(Clone, Debug)] -pub enum PackBytes { - Eager(std::sync::Arc<[u8]>), - LazyExternal { - sha256_hex: String, - store: std::sync::Arc, - cache: std::sync::Arc>>, - }, - LazyRepoBytes { - sha256_hex: String, - store: std::sync::Arc, - cache: std::sync::Arc>>, - }, -} - -impl PackBytes { - pub fn eager(bytes: Vec) -> Self { - Self::Eager(std::sync::Arc::from(bytes)) - } - - pub fn lazy_external(sha256_hex: String, store: std::sync::Arc) -> Self { - Self::LazyExternal { - sha256_hex, - store, - cache: std::sync::Arc::new(std::sync::OnceLock::new()), - } - } - - pub fn lazy_repo_bytes(sha256_hex: String, store: std::sync::Arc) -> Self { - Self::LazyRepoBytes { - sha256_hex, - store, - cache: std::sync::Arc::new(std::sync::OnceLock::new()), - } - } - - pub fn as_slice(&self) -> Result<&[u8], String> { - match self { - Self::Eager(bytes) => Ok(bytes.as_ref()), - Self::LazyExternal { - sha256_hex, - store, - cache, - } => { - if cache.get().is_none() { - let bytes = store - .get_blob_bytes(sha256_hex) - .map_err(|e| e.to_string())? - .ok_or_else(|| format!("missing raw blob for sha256={sha256_hex}"))?; - let _ = cache.set(std::sync::Arc::from(bytes)); - } - let bytes = cache - .get() - .ok_or_else(|| format!("missing raw blob cache for sha256={sha256_hex}"))?; - Ok(bytes.as_ref()) - } - Self::LazyRepoBytes { - sha256_hex, - store, - cache, - } => { - if cache.get().is_none() { - let bytes = store - .get_blob_bytes(sha256_hex) - .map_err(|e| e.to_string())? - .ok_or_else(|| format!("missing repo bytes for sha256={sha256_hex}"))?; - let _ = cache.set(std::sync::Arc::from(bytes)); - } - let bytes = cache - .get() - .ok_or_else(|| format!("missing repo bytes cache for sha256={sha256_hex}"))?; - Ok(bytes.as_ref()) - } - } - } - - pub fn to_vec(&self) -> Result, String> { - Ok(self.as_slice()?.to_vec()) - } -} - -impl PartialEq for PackBytes { - fn eq(&self, other: &Self) -> bool { - match (self.as_slice(), other.as_slice()) { - (Ok(a), Ok(b)) => a == b, - _ => false, - } - } -} - -impl Eq for PackBytes {} - -#[derive(Clone, Debug)] -pub struct PackFile { - pub rsync_uri: String, - pub bytes: PackBytes, - pub sha256: [u8; 32], -} - -impl PackFile { - pub fn new(rsync_uri: impl Into, bytes: PackBytes, sha256: [u8; 32]) -> Self { - Self { - rsync_uri: rsync_uri.into(), - bytes, - sha256, - } - } - - pub fn from_bytes_with_sha256( - rsync_uri: impl Into, - bytes: Vec, - sha256: [u8; 32], - ) -> Self { - Self::new(rsync_uri, PackBytes::eager(bytes), sha256) - } - - pub fn from_lazy_external_raw_store( - rsync_uri: impl Into, - sha256_hex: String, - sha256: [u8; 32], - store: std::sync::Arc, - ) -> Self { - Self::new( - rsync_uri, - PackBytes::lazy_external(sha256_hex, store), - sha256, - ) - } - - pub fn from_lazy_repo_bytes( - rsync_uri: impl Into, - sha256_hex: String, - sha256: [u8; 32], - store: std::sync::Arc, - ) -> Self { - Self::new( - rsync_uri, - PackBytes::lazy_repo_bytes(sha256_hex, store), - sha256, - ) - } - - pub fn from_bytes_compute_sha256(rsync_uri: impl Into, bytes: Vec) -> Self { - let sha256 = compute_sha256_32(&bytes); - Self::new(rsync_uri, PackBytes::eager(bytes), sha256) - } - - pub fn bytes(&self) -> Result<&[u8], String> { - self.bytes.as_slice() - } - - pub fn bytes_cloned(&self) -> Result, String> { - self.bytes.to_vec() - } - - pub fn compute_sha256(&self) -> Result<[u8; 32], String> { - Ok(compute_sha256_32(self.bytes()?)) - } -} - -impl PartialEq for PackFile { - fn eq(&self, other: &Self) -> bool { - self.rsync_uri == other.rsync_uri - && self.sha256 == other.sha256 - && self.bytes == other.bytes - } -} - -impl Eq for PackFile {} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct PackTime { - pub rfc3339_utc: String, -} - -impl PackTime { - pub fn from_utc_offset_datetime(t: time::OffsetDateTime) -> Self { - use time::format_description::well_known::Rfc3339; - let utc = t.to_offset(time::UtcOffset::UTC); - let s = utc.format(&Rfc3339).expect("format RFC 3339 UTC time"); - Self { rfc3339_utc: s } - } - - pub fn parse(&self) -> Result { - use time::format_description::well_known::Rfc3339; - time::OffsetDateTime::parse(&self.rfc3339_utc, &Rfc3339).map_err(|e| e.to_string()) - } -} - -fn compute_sha256_32(bytes: &[u8]) -> [u8; 32] { - let digest = sha2::Sha256::digest(bytes); - let mut out = [0u8; 32]; - out.copy_from_slice(&digest); - out -} - #[cfg(test)] -mod tests { - use super::*; - - fn pack_time(hour: i64) -> PackTime { - PackTime::from_utc_offset_datetime( - time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(hour), - ) - } - - fn sha256_hex(input: &[u8]) -> String { - hex::encode(compute_sha256_32(input)) - } - - #[test] - fn parse_work_db_blob_mode_accepts_supported_values() { - assert_eq!(default_work_db_blob_mode(), WorkDbBlobMode::Disabled); - assert_eq!( - parse_work_db_blob_mode("default"), - Some(WorkDbBlobMode::Disabled) - ); - assert_eq!( - parse_work_db_blob_mode("current"), - Some(WorkDbBlobMode::Current) - ); - assert_eq!( - parse_work_db_blob_mode("legacy"), - Some(WorkDbBlobMode::Current) - ); - assert_eq!( - parse_work_db_blob_mode("disabled"), - Some(WorkDbBlobMode::Disabled) - ); - assert_eq!( - parse_work_db_blob_mode("no-blob"), - Some(WorkDbBlobMode::Disabled) - ); - assert_eq!(parse_work_db_blob_mode("lz4"), Some(WorkDbBlobMode::Lz4)); - assert_eq!( - parse_work_db_blob_mode("blob-lz4"), - Some(WorkDbBlobMode::Lz4) - ); - assert_eq!(parse_work_db_blob_mode("unexpected"), None); - } - - fn sample_repository_view_entry(rsync_uri: &str, bytes: &[u8]) -> RepositoryViewEntry { - RepositoryViewEntry { - rsync_uri: rsync_uri.to_string(), - current_hash: Some(sha256_hex(bytes)), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("cer".to_string()), - state: RepositoryViewState::Present, - } - } - - fn sample_raw_by_hash_entry(bytes: Vec) -> RawByHashEntry { - RawByHashEntry { - sha256_hex: sha256_hex(&bytes), - bytes, - origin_uris: vec!["rsync://example.test/repo/object.cer".to_string()], - object_type: Some("cer".to_string()), - encoding: Some("der".to_string()), - } - } - - fn sample_ccr_manifest_projection( - manifest_rsync_uri: &str, - manifest_this_update: PackTime, - subordinate_skis: Vec>, - ) -> VcirCcrManifestProjection { - VcirCcrManifestProjection { - manifest_rsync_uri: manifest_rsync_uri.to_string(), - manifest_sha256: vec![0x11; 32], - manifest_size: 4096, - manifest_ee_aki: vec![0x22; 20], - manifest_number_be: vec![3], - manifest_this_update, - manifest_sia_locations_der: vec![vec![ - 0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05, - b'r', b's', b'y', b'n', b'c', - ]], - subordinate_skis, - } - } - - fn sample_vcir(manifest_rsync_uri: &str) -> ValidatedCaInstanceResult { - let roa_bytes = b"roa-object".to_vec(); - let ee_bytes = b"ee-cert".to_vec(); - let child_bytes = b"child-cert".to_vec(); - let child_ski = "1234567890abcdef1234567890abcdef12345678".to_string(); - ValidatedCaInstanceResult { - manifest_rsync_uri: manifest_rsync_uri.to_string(), - parent_manifest_rsync_uri: Some( - "rsync://example.test/repo/parent/parent.mft".to_string(), - ), - tal_id: "apnic".to_string(), - ca_subject_name: "CN=Example CA".to_string(), - ca_ski: "00112233445566778899aabbccddeeff00112233".to_string(), - issuer_ski: "ffeeddccbbaa99887766554433221100ffeeddcc".to_string(), - last_successful_validation_time: pack_time(0), - current_manifest_rsync_uri: manifest_rsync_uri.to_string(), - current_crl_rsync_uri: "rsync://example.test/repo/current.crl".to_string(), - validated_manifest_meta: ValidatedManifestMeta { - validated_manifest_number: vec![3], - validated_manifest_this_update: pack_time(0), - validated_manifest_next_update: pack_time(24), - }, - ccr_manifest_projection: sample_ccr_manifest_projection( - manifest_rsync_uri, - pack_time(0), - vec![hex::decode(&child_ski).expect("decode child ski")], - ), - instance_gate: VcirInstanceGate { - manifest_next_update: pack_time(24), - current_crl_next_update: pack_time(12), - self_ca_not_after: pack_time(48), - instance_effective_until: pack_time(12), - }, - child_entries: vec![VcirChildEntry { - child_manifest_rsync_uri: "rsync://example.test/repo/child/child.mft".to_string(), - child_cert_rsync_uri: "rsync://example.test/repo/child/child.cer".to_string(), - child_cert_hash: sha256_hex(&child_bytes), - child_ski, - child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(), - child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), - child_rrdp_notification_uri: Some( - "https://example.test/child-notify.xml".to_string(), - ), - child_effective_ip_resources: None, - child_effective_as_resources: None, - accepted_at_validation_time: pack_time(0), - }], - local_outputs: vec![ - VcirLocalOutput { - output_id: "vrp-1".to_string(), - output_type: VcirOutputType::Vrp, - item_effective_until: pack_time(12), - source_object_uri: "rsync://example.test/repo/object.roa".to_string(), - source_object_type: "roa".to_string(), - source_object_hash: sha256_hex(&roa_bytes), - source_ee_cert_hash: sha256_hex(&ee_bytes), - payload_json: r#"{"asn":64496,"prefix":"203.0.113.0/24"}"#.to_string(), - rule_hash: sha256_hex(b"vrp-rule-1"), - validation_path_hint: vec![manifest_rsync_uri.to_string()], - }, - VcirLocalOutput { - output_id: "aspa-1".to_string(), - output_type: VcirOutputType::Aspa, - item_effective_until: pack_time(10), - source_object_uri: "rsync://example.test/repo/object.asa".to_string(), - source_object_type: "aspa".to_string(), - source_object_hash: sha256_hex(b"aspa-object"), - source_ee_cert_hash: sha256_hex(b"aspa-ee-cert"), - payload_json: r#"{"customer_as":64496,"providers":[64497]}"#.to_string(), - rule_hash: sha256_hex(b"aspa-rule-1"), - validation_path_hint: vec![manifest_rsync_uri.to_string()], - }, - ], - related_artifacts: vec![ - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::Manifest, - artifact_kind: VcirArtifactKind::Mft, - uri: Some(manifest_rsync_uri.to_string()), - sha256: sha256_hex(b"manifest-object"), - object_type: Some("mft".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::CurrentCrl, - artifact_kind: VcirArtifactKind::Crl, - uri: Some("rsync://example.test/repo/current.crl".to_string()), - sha256: sha256_hex(b"current-crl"), - object_type: Some("crl".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }, - ], - summary: VcirSummary { - local_vrp_count: 1, - local_aspa_count: 1, - local_router_key_count: 0, - child_count: 1, - accepted_object_count: 4, - rejected_object_count: 0, - }, - audit_summary: VcirAuditSummary { - failed_fetch_eligible: true, - last_failed_fetch_reason: None, - warning_count: 0, - audit_flags: vec!["validated-fresh".to_string()], - }, - } - } - - #[test] - fn vcir_ccr_manifest_projection_validate_accepts_valid_projection() { - let projection = sample_ccr_manifest_projection( - "rsync://example.test/repo/current.mft", - pack_time(0), - vec![vec![0x33; 20], vec![0x44; 20]], - ); - projection.validate_internal().expect("valid projection"); - } - - #[test] - fn vcir_ccr_manifest_projection_validate_rejects_invalid_fields() { - let mut bad_hash = sample_ccr_manifest_projection( - "rsync://example.test/repo/current.mft", - pack_time(0), - vec![vec![0x33; 20]], - ); - bad_hash.manifest_sha256 = vec![0x11; 31]; - assert!(matches!( - bad_hash.validate_internal(), - Err(StorageError::InvalidData { .. }) - )); - - let mut bad_locations = sample_ccr_manifest_projection( - "rsync://example.test/repo/current.mft", - pack_time(0), - vec![vec![0x33; 20]], - ); - bad_locations.manifest_sia_locations_der = vec![vec![0x04, 0x00]]; - assert!(matches!( - bad_locations.validate_internal(), - Err(StorageError::InvalidData { .. }) - )); - - let bad_subordinates = sample_ccr_manifest_projection( - "rsync://example.test/repo/current.mft", - pack_time(0), - vec![vec![0x44; 20], vec![0x33; 20]], - ); - assert!(matches!( - bad_subordinates.validate_internal(), - Err(StorageError::InvalidData { .. }) - )); - } - - fn sample_audit_rule_entry(kind: AuditRuleKind) -> AuditRuleIndexEntry { - AuditRuleIndexEntry { - kind, - rule_hash: sha256_hex(match kind { - AuditRuleKind::Roa => b"roa-index-rule", - AuditRuleKind::Aspa => b"aspa-index-rule", - AuditRuleKind::RouterKey => b"router-key-index-rule", - }), - manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), - source_object_uri: match kind { - AuditRuleKind::Roa => "rsync://example.test/repo/object.roa".to_string(), - AuditRuleKind::Aspa => "rsync://example.test/repo/object.asa".to_string(), - AuditRuleKind::RouterKey => "rsync://example.test/repo/router.cer".to_string(), - }, - source_object_hash: sha256_hex(match kind { - AuditRuleKind::Roa => b"roa-object", - AuditRuleKind::Aspa => b"aspa-object", - AuditRuleKind::RouterKey => b"router-key-object", - }), - output_id: match kind { - AuditRuleKind::Roa => "vrp-1".to_string(), - AuditRuleKind::Aspa => "aspa-1".to_string(), - AuditRuleKind::RouterKey => "router-key-1".to_string(), - }, - item_effective_until: pack_time(12), - } - } - - fn sample_rrdp_source_record(notify_uri: &str) -> RrdpSourceRecord { - RrdpSourceRecord { - notify_uri: notify_uri.to_string(), - last_session_id: Some("session-1".to_string()), - last_serial: Some(42), - first_seen_at: pack_time(0), - last_seen_at: pack_time(1), - last_sync_at: Some(pack_time(1)), - sync_state: RrdpSourceSyncState::DeltaReady, - last_snapshot_uri: Some("https://rrdp.example.test/snapshot.xml".to_string()), - last_snapshot_hash: Some(sha256_hex(b"snapshot-bytes")), - last_error: None, - } - } - - fn sample_rrdp_source_member_record( - notify_uri: &str, - rsync_uri: &str, - serial: u64, - ) -> RrdpSourceMemberRecord { - RrdpSourceMemberRecord { - notify_uri: notify_uri.to_string(), - rsync_uri: rsync_uri.to_string(), - current_hash: Some(sha256_hex(rsync_uri.as_bytes())), - object_type: Some("cer".to_string()), - present: true, - last_confirmed_session_id: "session-1".to_string(), - last_confirmed_serial: serial, - last_changed_at: pack_time(serial as i64), - } - } - - fn sample_rrdp_uri_owner_record(notify_uri: &str, rsync_uri: &str) -> RrdpUriOwnerRecord { - RrdpUriOwnerRecord { - rsync_uri: rsync_uri.to_string(), - notify_uri: notify_uri.to_string(), - current_hash: Some(sha256_hex(rsync_uri.as_bytes())), - last_confirmed_session_id: "session-1".to_string(), - last_confirmed_serial: 7, - last_changed_at: pack_time(7), - owner_state: RrdpUriOwnerState::Active, - } - } - - #[test] - fn repository_view_and_raw_by_hash_roundtrip() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let entry1 = sample_repository_view_entry("rsync://example.test/repo/a.cer", b"object-a"); - let entry2 = - sample_repository_view_entry("rsync://example.test/repo/sub/b.roa", b"object-b"); - store - .put_repository_view_entry(&entry1) - .expect("put repository view entry1"); - store - .put_repository_view_entry(&entry2) - .expect("put repository view entry2"); - - let got1 = store - .get_repository_view_entry(&entry1.rsync_uri) - .expect("get repository view entry1") - .expect("entry1 exists"); - assert_eq!(got1, entry1); - - let got_prefix = store - .list_repository_view_entries_with_prefix("rsync://example.test/repo/sub/") - .expect("list repository view prefix"); - assert_eq!(got_prefix, vec![entry2.clone()]); - - store - .delete_repository_view_entry(&entry1.rsync_uri) - .expect("delete repository view entry1"); - assert!( - store - .get_repository_view_entry(&entry1.rsync_uri) - .expect("get deleted repository view entry1") - .is_none() - ); - - let raw = sample_raw_by_hash_entry(b"raw-der-object".to_vec()); - store - .put_raw_by_hash_entry(&raw) - .expect("put raw_by_hash entry"); - let got_raw = store - .get_raw_by_hash_entry(&raw.sha256_hex) - .expect("get raw_by_hash entry") - .expect("raw entry exists"); - assert_eq!(got_raw, raw); - } - - #[test] - fn raw_by_hash_routes_to_external_raw_store_when_configured() { - let td = tempfile::tempdir().expect("tempdir"); - let main_db = td.path().join("main-db"); - let raw_db = td.path().join("raw-store.db"); - - let raw = sample_raw_by_hash_entry(b"external-raw".to_vec()); - { - let store = - RocksStore::open_with_external_raw_store(&main_db, &raw_db).expect("open store"); - store.put_raw_by_hash_entry(&raw).expect("put external raw"); - - let got = store - .get_raw_by_hash_entry(&raw.sha256_hex) - .expect("get external raw") - .expect("raw exists"); - assert_eq!(got, raw); - } - - let main_store = RocksStore::open(&main_db).expect("open main only"); - assert!( - main_store - .get_raw_by_hash_entry(&raw.sha256_hex) - .expect("read main store") - .is_none(), - "main db should not contain raw entry when external raw store is configured" - ); - } - - #[test] - fn put_blob_bytes_batch_uses_internal_blob_cf_without_raw_entry() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let bytes = b"internal-blob-only".to_vec(); - let hash = sha256_hex(&bytes); - - store - .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) - .expect("put blob bytes"); - - assert_eq!( - store.get_blob_bytes(&hash).expect("get blob bytes"), - Some(bytes.clone()) - ); - assert!( - store - .get_raw_by_hash_entry(&hash) - .expect("get raw entry") - .is_none() - ); - } - - #[test] - fn put_blob_bytes_batch_routes_to_external_raw_store_without_raw_entry() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open_with_external_raw_store( - &td.path().join("main-db"), - &td.path().join("raw-store.db"), - ) - .expect("open store"); - let bytes = b"external-blob-only".to_vec(); - let hash = sha256_hex(&bytes); - - store - .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) - .expect("put external blob bytes"); - - assert_eq!(store.get_blob_bytes(&hash).unwrap(), Some(bytes)); - assert!(store.get_raw_by_hash_entry(&hash).unwrap().is_none()); - } - - #[test] - fn repo_bytes_db_is_physically_separate_from_external_raw_store() { - let td = tempfile::tempdir().expect("tempdir"); - let main_db = td.path().join("main-db"); - let raw_db = td.path().join("raw-store.db"); - let repo_bytes_db = td.path().join("repo-bytes.db"); - let store = - RocksStore::open_with_external_stores(&main_db, Some(&raw_db), Some(&repo_bytes_db)) - .expect("open store"); - let repo_bytes = b"repo-object".to_vec(); - let repo_hash = sha256_hex(&repo_bytes); - let raw = sample_raw_by_hash_entry(b"raw-evidence".to_vec()); - - store - .put_blob_bytes_batch(&[(repo_hash.clone(), repo_bytes.clone())]) - .expect("put repo bytes"); - store.put_raw_by_hash_entry(&raw).expect("put raw evidence"); - - assert_eq!(store.get_blob_bytes(&repo_hash).unwrap(), Some(repo_bytes)); - assert_eq!( - store.get_raw_by_hash_entry(&raw.sha256_hex).unwrap(), - Some(raw.clone()) - ); - drop(store); - - let raw_only = - RocksStore::open_with_external_raw_store(&td.path().join("raw-reader"), &raw_db) - .expect("open raw only"); - assert!( - raw_only.get_blob_bytes(&repo_hash).unwrap().is_none(), - "repo object bytes must not be written into raw-store.db" - ); - - let repo_only = RocksStore::open_with_external_repo_bytes( - &td.path().join("repo-reader"), - &repo_bytes_db, - ) - .expect("open repo bytes only"); - assert_eq!( - repo_only.get_blob_bytes(&repo_hash).unwrap(), - Some(b"repo-object".to_vec()) - ); - assert!( - repo_only.get_blob_bytes(&raw.sha256_hex).unwrap().is_none(), - "raw evidence bytes must not be written into repo-bytes.db" - ); - } - - #[test] - fn put_blob_bytes_batch_accepts_empty_batch_with_external_raw_store() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open_with_external_raw_store( - &td.path().join("main-db"), - &td.path().join("raw-store.db"), - ) - .expect("open store"); - - store - .put_blob_bytes_batch(&[]) - .expect("empty external blob batch should be a no-op"); - } - - #[test] - fn get_blob_bytes_internal_falls_back_to_raw_entry_when_blob_missing() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let raw = sample_raw_by_hash_entry(b"raw-fallback".to_vec()); - - store.put_raw_by_hash_entry(&raw).expect("put raw entry"); - - assert_eq!( - store - .get_blob_bytes(&raw.sha256_hex) - .expect("get blob bytes via raw fallback"), - Some(raw.bytes.clone()) - ); - } - - #[test] - fn get_blob_bytes_batch_internal_prefers_blob_cf_and_falls_back_to_raw_entry() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let blob_bytes = b"blob-cf-object".to_vec(); - let blob_hash = sha256_hex(&blob_bytes); - store - .put_blob_bytes_batch(&[(blob_hash.clone(), blob_bytes.clone())]) - .expect("put blob bytes"); - - let raw = sample_raw_by_hash_entry(b"raw-fallback-batch".to_vec()); - store.put_raw_by_hash_entry(&raw).expect("put raw fallback"); - - let batch = store - .get_blob_bytes_batch(&[blob_hash.clone(), raw.sha256_hex.clone(), "00".repeat(32)]) - .expect("get blob bytes batch"); - assert_eq!(batch, vec![Some(blob_bytes), Some(raw.bytes.clone()), None]); - } - - #[test] - fn get_blob_bytes_batch_routes_to_external_raw_store_without_raw_entry() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open_with_external_raw_store( - &td.path().join("main-db"), - &td.path().join("raw-store.db"), - ) - .expect("open store"); - let bytes = b"external-batch-blob".to_vec(); - let hash = sha256_hex(&bytes); - - store - .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) - .expect("put external blob bytes"); - - assert_eq!( - store - .get_blob_bytes_batch(&[hash, "00".repeat(32)]) - .expect("get external blob batch"), - vec![Some(bytes), None] - ); - } - - #[test] - fn get_blob_bytes_rejects_invalid_hash_for_internal_store() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let err = store - .get_blob_bytes("not-a-valid-hash") - .expect_err("invalid hash must fail"); - assert!(matches!(err, StorageError::InvalidData { .. })); - } - - #[test] - fn get_blob_bytes_batch_rejects_invalid_hash_for_internal_store() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let err = store - .get_blob_bytes_batch(&["not-a-valid-hash".to_string()]) - .expect_err("invalid hash must fail"); - assert!(matches!(err, StorageError::InvalidData { .. })); - } - - #[test] - fn get_blob_bytes_batch_returns_empty_for_empty_request_internal() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - assert!( - store - .get_blob_bytes_batch(&[]) - .expect("empty blob batch request") - .is_empty() - ); - } - - #[test] - fn put_blob_bytes_batch_accepts_empty_batch() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - store - .put_blob_bytes_batch(&[]) - .expect("empty blob batch should be a no-op"); - } - - #[test] - fn put_blob_bytes_batch_rejects_empty_bytes() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let err = store - .put_blob_bytes_batch(&[(sha256_hex(b"valid"), Vec::new())]) - .expect_err("empty bytes must fail"); - assert!(matches!(err, StorageError::InvalidData { .. })); - } - - #[test] - fn delete_raw_by_hash_entry_internal_preserves_blob_bytes() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let bytes = b"blob-persists-after-raw-delete".to_vec(); - let hash = sha256_hex(&bytes); - let raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone()); - - store - .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) - .expect("put blob bytes"); - store.put_raw_by_hash_entry(&raw).expect("put raw entry"); - - store - .delete_raw_by_hash_entry(&hash) - .expect("delete raw entry only"); - - assert!(store.get_raw_by_hash_entry(&hash).unwrap().is_none()); - assert_eq!(store.get_blob_bytes(&hash).unwrap(), Some(bytes)); - } - - #[test] - fn delete_raw_by_hash_entry_rejects_invalid_hash() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let err = store - .delete_raw_by_hash_entry("not-a-valid-hash") - .expect_err("invalid hash must fail"); - assert!(matches!(err, StorageError::InvalidData { .. })); - } - - #[test] - fn delete_raw_by_hash_entry_routes_to_external_raw_store() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open_with_external_raw_store( - &td.path().join("main-db"), - &td.path().join("raw-store.db"), - ) - .expect("open store"); - let raw = sample_raw_by_hash_entry(b"external-delete".to_vec()); - - store.put_raw_by_hash_entry(&raw).expect("put raw entry"); - store - .delete_raw_by_hash_entry(&raw.sha256_hex) - .expect("delete external raw entry"); - - assert!( - store - .get_raw_by_hash_entry(&raw.sha256_hex) - .unwrap() - .is_none() - ); - assert!(store.get_blob_bytes(&raw.sha256_hex).unwrap().is_none()); - } - - #[test] - fn repository_view_and_raw_by_hash_validation_errors_are_reported() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let invalid_view = RepositoryViewEntry { - rsync_uri: "rsync://example.test/repo/withdrawn.cer".to_string(), - current_hash: None, - repository_source: None, - object_type: None, - state: RepositoryViewState::Present, - }; - let err = store - .put_repository_view_entry(&invalid_view) - .expect_err("missing current_hash must fail"); - assert!(err.to_string().contains("current_hash is required")); - - let invalid_raw = RawByHashEntry { - sha256_hex: sha256_hex(b"expected"), - bytes: b"actual".to_vec(), - origin_uris: vec!["rsync://example.test/repo/object.cer".to_string()], - object_type: None, - encoding: None, - }; - let err = store - .put_raw_by_hash_entry(&invalid_raw) - .expect_err("mismatched raw_by_hash entry must fail"); - assert!(err.to_string().contains("does not match bytes")); - } - - #[test] - fn vcir_roundtrip_and_validation_failures_are_reported() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let vcir = sample_vcir("rsync://example.test/repo/current.mft"); - store.put_vcir(&vcir).expect("put vcir"); - let got = store - .get_vcir(&vcir.manifest_rsync_uri) - .expect("get vcir") - .expect("vcir exists"); - assert_eq!(got, vcir); - let replay_meta = store - .get_manifest_replay_meta(&vcir.manifest_rsync_uri) - .expect("get manifest replay meta") - .expect("manifest replay meta exists"); - assert_eq!( - replay_meta, - ManifestReplayMeta { - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - manifest_number_be: vcir - .validated_manifest_meta - .validated_manifest_number - .clone(), - manifest_this_update: vcir - .validated_manifest_meta - .validated_manifest_this_update - .clone(), - manifest_sha256: vcir.ccr_manifest_projection.manifest_sha256.clone(), - updated_at_validation_time: vcir.last_successful_validation_time.clone(), - } - ); - - let mut invalid = sample_vcir("rsync://example.test/repo/invalid.mft"); - invalid.summary.local_vrp_count = 9; - let err = store - .put_vcir(&invalid) - .expect_err("invalid vcir must fail"); - assert!(err.to_string().contains("local_vrp_count=9")); - - let mut invalid = sample_vcir("rsync://example.test/repo/invalid-2.mft"); - invalid.instance_gate.instance_effective_until = pack_time(11); - let err = store - .put_vcir(&invalid) - .expect_err("invalid instance gate must fail"); - assert!(err.to_string().contains("instance_effective_until")); - - store - .delete_vcir(&vcir.manifest_rsync_uri) - .expect("delete vcir"); - assert!( - store - .get_vcir(&vcir.manifest_rsync_uri) - .expect("get deleted vcir") - .is_none() - ); - assert!( - store - .get_manifest_replay_meta(&vcir.manifest_rsync_uri) - .expect("get deleted manifest replay meta") - .is_none() - ); - } - - #[test] - fn manifest_replay_meta_validation_reports_invalid_fields() { - let mut meta = ManifestReplayMeta { - manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), - manifest_number_be: vec![3], - manifest_this_update: pack_time(0), - manifest_sha256: vec![0x11; 32], - updated_at_validation_time: pack_time(1), - }; - meta.validate_internal().expect("valid replay meta"); - - meta.manifest_sha256 = vec![0x11; 31]; - let err = meta - .validate_internal() - .expect_err("short manifest sha must fail"); - assert!(err.to_string().contains("must be 32 bytes")); - - meta.manifest_sha256 = vec![0x11; 32]; - meta.manifest_number_be = vec![0, 3]; - let err = meta - .validate_internal() - .expect_err("non-minimal manifest number must fail"); - assert!(err.to_string().contains("minimal big-endian")); - } - - #[test] - fn list_vcirs_returns_all_entries() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let vcir1 = sample_vcir("rsync://example.test/repo/a.mft"); - let vcir2 = sample_vcir("rsync://example.test/repo/b.mft"); - store.put_vcir(&vcir1).expect("put vcir1"); - store.put_vcir(&vcir2).expect("put vcir2"); - - let mut got = store.list_vcirs().expect("list vcirs"); - got.sort_by(|a, b| a.manifest_rsync_uri.cmp(&b.manifest_rsync_uri)); - assert_eq!(got, vec![vcir1, vcir2]); - } - - #[test] - fn audit_rule_index_roundtrip_for_roa_aspa_and_router_key() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let roa = sample_audit_rule_entry(AuditRuleKind::Roa); - let aspa = sample_audit_rule_entry(AuditRuleKind::Aspa); - let router_key = sample_audit_rule_entry(AuditRuleKind::RouterKey); - store - .put_audit_rule_index_entry(&roa) - .expect("put roa audit rule entry"); - store - .put_audit_rule_index_entry(&aspa) - .expect("put aspa audit rule entry"); - store - .put_audit_rule_index_entry(&router_key) - .expect("put router key audit rule entry"); - - let got_roa = store - .get_audit_rule_index_entry(AuditRuleKind::Roa, &roa.rule_hash) - .expect("get roa audit rule entry") - .expect("roa entry exists"); - let got_aspa = store - .get_audit_rule_index_entry(AuditRuleKind::Aspa, &aspa.rule_hash) - .expect("get aspa audit rule entry") - .expect("aspa entry exists"); - let got_router_key = store - .get_audit_rule_index_entry(AuditRuleKind::RouterKey, &router_key.rule_hash) - .expect("get router key audit rule entry") - .expect("router key entry exists"); - assert_eq!(got_roa, roa); - assert_eq!(got_aspa, aspa); - assert_eq!(got_router_key, router_key); - - store - .delete_audit_rule_index_entry(AuditRuleKind::Roa, &roa.rule_hash) - .expect("delete roa audit rule entry"); - assert!( - store - .get_audit_rule_index_entry(AuditRuleKind::Roa, &roa.rule_hash) - .expect("get deleted roa audit rule entry") - .is_none() - ); - - let mut invalid = sample_audit_rule_entry(AuditRuleKind::Roa); - invalid.rule_hash = "bad".to_string(); - let err = store - .put_audit_rule_index_entry(&invalid) - .expect_err("invalid audit rule hash must fail"); - assert!(err.to_string().contains("64-character")); - } - - #[test] - fn replace_vcir_and_audit_rule_indexes_replaces_previous_entries_in_one_step() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let mut previous = sample_vcir("rsync://example.test/repo/current.mft"); - previous.local_outputs = vec![VcirLocalOutput { - output_id: "old-output".to_string(), - output_type: VcirOutputType::Vrp, - item_effective_until: pack_time(10), - source_object_uri: "rsync://example.test/repo/old.roa".to_string(), - source_object_type: "roa".to_string(), - source_object_hash: sha256_hex(b"old-roa"), - source_ee_cert_hash: sha256_hex(b"old-ee"), - payload_json: "{}".to_string(), - rule_hash: sha256_hex(b"old-rule"), - validation_path_hint: vec![previous.manifest_rsync_uri.clone()], - }]; - previous.summary.local_vrp_count = 1; - previous.summary.local_aspa_count = 0; - previous.summary.local_router_key_count = 0; - store - .replace_vcir_and_audit_rule_indexes(None, &previous) - .expect("store previous vcir"); - assert!( - store - .get_audit_rule_index_entry( - AuditRuleKind::Roa, - &previous.local_outputs[0].rule_hash - ) - .expect("get old audit entry") - .is_some() - ); - - let mut current = sample_vcir("rsync://example.test/repo/current.mft"); - current.local_outputs = vec![VcirLocalOutput { - output_id: "new-output".to_string(), - output_type: VcirOutputType::Aspa, - item_effective_until: pack_time(11), - source_object_uri: "rsync://example.test/repo/new.asa".to_string(), - source_object_type: "aspa".to_string(), - source_object_hash: sha256_hex(b"new-aspa"), - source_ee_cert_hash: sha256_hex(b"new-ee"), - payload_json: "{}".to_string(), - rule_hash: sha256_hex(b"new-rule"), - validation_path_hint: vec![current.manifest_rsync_uri.clone()], - }]; - current.summary.local_vrp_count = 0; - current.summary.local_aspa_count = 1; - store - .replace_vcir_and_audit_rule_indexes(Some(&previous), ¤t) - .expect("replace vcir and audit indexes"); - - let got = store - .get_vcir(¤t.manifest_rsync_uri) - .expect("get replaced vcir") - .expect("vcir exists"); - assert_eq!(got, current); - let replay_meta = store - .get_manifest_replay_meta(¤t.manifest_rsync_uri) - .expect("get replaced replay meta") - .expect("replay meta exists"); - assert_eq!( - replay_meta.manifest_number_be, - current.validated_manifest_meta.validated_manifest_number - ); - assert_eq!( - replay_meta.manifest_sha256, - current.ccr_manifest_projection.manifest_sha256 - ); - assert!( - store - .get_audit_rule_index_entry( - AuditRuleKind::Roa, - &previous.local_outputs[0].rule_hash - ) - .expect("get deleted old audit entry") - .is_none() - ); - assert!( - store - .get_audit_rule_index_entry( - AuditRuleKind::Aspa, - ¤t.local_outputs[0].rule_hash - ) - .expect("get new audit entry") - .is_some() - ); - } - - #[test] - fn storage_helpers_cover_optional_validation_paths() { - let withdrawn = RepositoryViewEntry { - rsync_uri: "rsync://example.test/repo/withdrawn.cer".to_string(), - current_hash: Some(sha256_hex(b"withdrawn")), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("cer".to_string()), - state: RepositoryViewState::Withdrawn, - }; - withdrawn - .validate_internal() - .expect("withdrawn repository view validates"); - - let raw = RawByHashEntry::from_bytes(sha256_hex(b"helper-bytes"), b"helper-bytes".to_vec()); - raw.validate_internal() - .expect("raw_by_hash helper validates"); - - let empty_raw = RawByHashEntry { - sha256_hex: sha256_hex(b"x"), - bytes: Vec::new(), - origin_uris: Vec::new(), - object_type: None, - encoding: None, - }; - let err = empty_raw - .validate_internal() - .expect_err("empty raw bytes must fail"); - assert!(err.to_string().contains("bytes must not be empty")); - - let duplicate_origin_raw = RawByHashEntry { - sha256_hex: sha256_hex(b"dup-origin"), - bytes: b"dup-origin".to_vec(), - origin_uris: vec![ - "rsync://example.test/repo/object.cer".to_string(), - "rsync://example.test/repo/object.cer".to_string(), - ], - object_type: Some("cer".to_string()), - encoding: Some("der".to_string()), - }; - let err = duplicate_origin_raw - .validate_internal() - .expect_err("duplicate origin URI must fail"); - assert!(err.to_string().contains("duplicate origin URI")); - } - - #[test] - fn rrdp_source_optional_fields_and_owner_without_hash_validate() { - let source = RrdpSourceRecord { - notify_uri: "https://rrdp.example.test/notification.xml".to_string(), - last_session_id: None, - last_serial: None, - first_seen_at: pack_time(0), - last_seen_at: pack_time(1), - last_sync_at: None, - sync_state: RrdpSourceSyncState::Empty, - last_snapshot_uri: None, - last_snapshot_hash: None, - last_error: Some("network timeout".to_string()), - }; - source - .validate_internal() - .expect("source with optional fields validates"); - - let owner = RrdpUriOwnerRecord { - rsync_uri: "rsync://example.test/repo/object.cer".to_string(), - notify_uri: "https://rrdp.example.test/notification.xml".to_string(), - current_hash: None, - last_confirmed_session_id: "session-1".to_string(), - last_confirmed_serial: 5, - last_changed_at: pack_time(5), - owner_state: RrdpUriOwnerState::Withdrawn, - }; - owner - .validate_internal() - .expect("owner without hash validates when withdrawn"); - } - - #[test] - fn rrdp_source_binding_records_roundtrip_and_prefix_iteration() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let notify_uri = "https://rrdp.example.test/notification.xml"; - let source = sample_rrdp_source_record(notify_uri); - store - .put_rrdp_source_record(&source) - .expect("put rrdp source record"); - let got_source = store - .get_rrdp_source_record(notify_uri) - .expect("get rrdp source record") - .expect("rrdp source exists"); - assert_eq!(got_source, source); - - let member1 = - sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/a.cer", 1); - let member2 = - sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/b.roa", 2); - let other_member = sample_rrdp_source_member_record( - "https://other.example.test/notification.xml", - "rsync://other.example.test/repo/c.cer", - 3, - ); - store - .put_rrdp_source_member_record(&member1) - .expect("put member1"); - store - .put_rrdp_source_member_record(&member2) - .expect("put member2"); - store - .put_rrdp_source_member_record(&other_member) - .expect("put other member"); - - let mut members = store - .list_rrdp_source_member_records(notify_uri) - .expect("list rrdp source members"); - members.sort_by(|a, b| a.rsync_uri.cmp(&b.rsync_uri)); - assert_eq!(members, vec![member1.clone(), member2.clone()]); - - let got_member = store - .get_rrdp_source_member_record(notify_uri, &member1.rsync_uri) - .expect("get member1") - .expect("member1 exists"); - assert_eq!(got_member, member1); - - let owner = sample_rrdp_uri_owner_record(notify_uri, &member1.rsync_uri); - store - .put_rrdp_uri_owner_record(&owner) - .expect("put uri owner record"); - let got_owner = store - .get_rrdp_uri_owner_record(&member1.rsync_uri) - .expect("get uri owner record") - .expect("uri owner exists"); - assert_eq!(got_owner, owner); - store - .delete_rrdp_uri_owner_record(&member1.rsync_uri) - .expect("delete uri owner record"); - assert!( - store - .get_rrdp_uri_owner_record(&member1.rsync_uri) - .expect("get deleted uri owner") - .is_none() - ); - - let mut invalid_source = - sample_rrdp_source_record("https://invalid.example/notification.xml"); - invalid_source.last_snapshot_hash = Some("bad".to_string()); - let err = store - .put_rrdp_source_record(&invalid_source) - .expect_err("invalid source hash must fail"); - assert!(err.to_string().contains("last_snapshot_hash")); - - let invalid_member = RrdpSourceMemberRecord { - notify_uri: notify_uri.to_string(), - rsync_uri: "rsync://example.test/repo/deleted.cer".to_string(), - current_hash: None, - object_type: None, - present: true, - last_confirmed_session_id: "session-1".to_string(), - last_confirmed_serial: 10, - last_changed_at: pack_time(10), - }; - let err = store - .put_rrdp_source_member_record(&invalid_member) - .expect_err("present member without hash must fail"); - assert!(err.to_string().contains("current_hash is required")); - } - #[test] - fn projection_batch_roundtrip_writes_repository_view_member_and_owner_records() { - let dir = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(dir.path()).expect("open store"); - - let view = RepositoryViewEntry { - rsync_uri: "rsync://example.test/repo/a.roa".to_string(), - current_hash: Some(hex::encode([1u8; 32])), - repository_source: Some("https://example.test/notify.xml".to_string()), - object_type: Some("roa".to_string()), - state: RepositoryViewState::Present, - }; - let member = RrdpSourceMemberRecord { - notify_uri: "https://example.test/notify.xml".to_string(), - rsync_uri: "rsync://example.test/repo/a.roa".to_string(), - current_hash: Some(hex::encode([1u8; 32])), - object_type: Some("roa".to_string()), - present: true, - last_confirmed_session_id: "session-1".to_string(), - last_confirmed_serial: 7, - last_changed_at: pack_time(1), - }; - let owner = RrdpUriOwnerRecord { - rsync_uri: "rsync://example.test/repo/a.roa".to_string(), - notify_uri: "https://example.test/notify.xml".to_string(), - current_hash: Some(hex::encode([1u8; 32])), - last_confirmed_session_id: "session-1".to_string(), - last_confirmed_serial: 7, - last_changed_at: pack_time(1), - owner_state: RrdpUriOwnerState::Active, - }; - - store - .put_projection_batch(&[view.clone()], &[member.clone()], &[owner.clone()]) - .expect("write projection batch"); - - assert_eq!( - store - .get_repository_view_entry(&view.rsync_uri) - .expect("get view") - .expect("present view"), - view - ); - assert_eq!( - store - .get_rrdp_source_member_record(&member.notify_uri, &member.rsync_uri) - .expect("get member") - .expect("present member"), - member - ); - assert_eq!( - store - .get_rrdp_uri_owner_record(&owner.rsync_uri) - .expect("get owner") - .expect("present owner"), - owner - ); - } - - #[test] - fn current_rrdp_source_member_helpers_filter_present_records() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let notify_uri = "https://rrdp.example.test/notification.xml"; - let mut present_a = - sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/a.cer", 1); - let mut withdrawn_b = - sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/b.roa", 2); - withdrawn_b.present = false; - let present_c = - sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/c.crl", 3); - let other_source = sample_rrdp_source_member_record( - "https://other.example.test/notification.xml", - "rsync://other.example.test/repo/x.cer", - 4, - ); - present_a.last_confirmed_serial = 10; - - store - .put_rrdp_source_member_record(&present_a) - .expect("put present a"); - store - .put_rrdp_source_member_record(&withdrawn_b) - .expect("put withdrawn b"); - store - .put_rrdp_source_member_record(&present_c) - .expect("put present c"); - store - .put_rrdp_source_member_record(&other_source) - .expect("put other source"); - - let members = store - .list_current_rrdp_source_members(notify_uri) - .expect("list current members"); - assert_eq!( - members - .iter() - .map(|record| record.rsync_uri.as_str()) - .collect::>(), - vec![ - "rsync://example.test/repo/a.cer", - "rsync://example.test/repo/c.crl", - ] - ); - - assert!( - store - .is_current_rrdp_source_member(notify_uri, &present_a.rsync_uri) - .expect("current a") - ); - assert!( - !store - .is_current_rrdp_source_member(notify_uri, &withdrawn_b.rsync_uri) - .expect("withdrawn b") - ); - assert!( - !store - .is_current_rrdp_source_member(notify_uri, &other_source.rsync_uri) - .expect("other source") - ); - } - - #[test] - fn load_current_object_bytes_by_uri_uses_repository_view_and_raw_by_hash() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let present_bytes = b"present-object".to_vec(); - let present_hash = sha256_hex(&present_bytes); - let mut present_raw = - RawByHashEntry::from_bytes(present_hash.clone(), present_bytes.clone()); - present_raw - .origin_uris - .push("rsync://example.test/repo/present.roa".to_string()); - present_raw.object_type = Some("roa".to_string()); - store - .put_raw_by_hash_entry(&present_raw) - .expect("put present raw"); - store - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: "rsync://example.test/repo/present.roa".to_string(), - current_hash: Some(present_hash), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("roa".to_string()), - state: RepositoryViewState::Present, - }) - .expect("put present view"); - - let replaced_bytes = b"replaced-object".to_vec(); - let replaced_hash = sha256_hex(&replaced_bytes); - let mut replaced_raw = - RawByHashEntry::from_bytes(replaced_hash.clone(), replaced_bytes.clone()); - replaced_raw - .origin_uris - .push("rsync://example.test/repo/replaced.cer".to_string()); - replaced_raw.object_type = Some("cer".to_string()); - store - .put_raw_by_hash_entry(&replaced_raw) - .expect("put replaced raw"); - store - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: "rsync://example.test/repo/replaced.cer".to_string(), - current_hash: Some(replaced_hash), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("cer".to_string()), - state: RepositoryViewState::Replaced, - }) - .expect("put replaced view"); - - store - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: "rsync://example.test/repo/withdrawn.crl".to_string(), - current_hash: Some(sha256_hex(b"withdrawn")), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("crl".to_string()), - state: RepositoryViewState::Withdrawn, - }) - .expect("put withdrawn view"); - - assert_eq!( - store - .load_current_object_bytes_by_uri("rsync://example.test/repo/present.roa") - .expect("load present"), - Some(present_bytes) - ); - assert_eq!( - store - .load_current_object_bytes_by_uri("rsync://example.test/repo/replaced.cer") - .expect("load replaced"), - Some(replaced_bytes) - ); - assert_eq!( - store - .load_current_object_bytes_by_uri("rsync://example.test/repo/withdrawn.crl") - .expect("load withdrawn"), - None - ); - assert_eq!( - store - .load_current_object_bytes_by_uri("rsync://example.test/repo/missing.roa") - .expect("load missing"), - None - ); - } - - #[test] - fn load_current_object_bytes_by_uri_errors_when_raw_by_hash_is_missing() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let rsync_uri = "rsync://example.test/repo/missing.cer"; - - store - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: rsync_uri.to_string(), - current_hash: Some(hex::encode([0x11; 32])), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("cer".to_string()), - state: RepositoryViewState::Present, - }) - .expect("put view"); - let err = store - .load_current_object_bytes_by_uri(rsync_uri) - .expect_err("missing raw_by_hash should error"); - assert!(matches!(err, StorageError::InvalidData { .. })); - } - - #[test] - fn load_current_object_with_hash_by_uri_returns_hash_and_bytes() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let rsync_uri = "rsync://example.test/repo/present.roa"; - let bytes = b"present-object".to_vec(); - let hash = sha256_hex(&bytes); - - let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone()); - raw.origin_uris.push(rsync_uri.to_string()); - raw.object_type = Some("roa".to_string()); - store.put_raw_by_hash_entry(&raw).expect("put raw"); - store - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: rsync_uri.to_string(), - current_hash: Some(hash.clone()), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("roa".to_string()), - state: RepositoryViewState::Present, - }) - .expect("put view"); - - let got = store - .load_current_object_with_hash_by_uri(rsync_uri) - .expect("load current object") - .expect("current object exists"); - assert_eq!(got.current_hash_hex, hash); - assert_eq!(got.current_hash, compute_sha256_32(&bytes)); - assert_eq!(got.bytes, bytes); - } - - #[test] - fn load_current_object_with_hash_by_uri_uses_internal_blob_cf_without_raw_entry() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let rsync_uri = "rsync://example.test/repo/blob-only.roa"; - let bytes = b"blob-only-current-object".to_vec(); - let hash = sha256_hex(&bytes); - - store - .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) - .expect("put blob bytes"); - store - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: rsync_uri.to_string(), - current_hash: Some(hash.clone()), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("roa".to_string()), - state: RepositoryViewState::Present, - }) - .expect("put view"); - - let got = store - .load_current_object_with_hash_by_uri(rsync_uri) - .expect("load current object") - .expect("current object exists"); - assert_eq!(got.current_hash_hex, hash); - assert_eq!(got.current_hash, compute_sha256_32(&bytes)); - assert_eq!(got.bytes, bytes); - assert!( - store - .get_raw_by_hash_entry(&got.current_hash_hex) - .expect("get raw entry") - .is_none() - ); - } - - #[test] - fn load_current_object_bytes_by_uri_uses_internal_blob_cf_without_raw_entry() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let rsync_uri = "rsync://example.test/repo/blob-only-bytes.roa"; - let bytes = b"blob-only-current-object-bytes".to_vec(); - let hash = sha256_hex(&bytes); - - store - .put_blob_bytes_batch(&[(hash, bytes.clone())]) - .expect("put blob bytes"); - store - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: rsync_uri.to_string(), - current_hash: Some(sha256_hex(&bytes)), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("roa".to_string()), - state: RepositoryViewState::Present, - }) - .expect("put view"); - - assert_eq!( - store - .load_current_object_bytes_by_uri(rsync_uri) - .expect("load current object bytes"), - Some(bytes) - ); - } - - #[test] - fn pack_file_can_lazy_load_bytes_from_external_raw_store() { - let td = tempfile::tempdir().expect("tempdir"); - let raw_store = std::sync::Arc::new( - ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store"), - ); - let bytes = b"lazy-pack-file".to_vec(); - let sha256_hex = sha256_hex(&bytes); - raw_store - .put_raw_entry(&RawByHashEntry::from_bytes( - sha256_hex.clone(), - bytes.clone(), - )) - .expect("put raw entry"); - - let file = PackFile::from_lazy_external_raw_store( - "rsync://example.test/repo/a.roa", - sha256_hex, - compute_sha256_32(&bytes), - raw_store, - ); - - assert_eq!(file.bytes().expect("lazy bytes"), bytes.as_slice()); - assert_eq!(file.bytes_cloned().expect("cloned bytes"), bytes); - } -} +#[path = "storage/tests.rs"] +mod tests; diff --git a/src/storage/config.rs b/src/storage/config.rs new file mode 100644 index 0000000..883e59a --- /dev/null +++ b/src/storage/config.rs @@ -0,0 +1,119 @@ +use rocksdb::{ColumnFamilyDescriptor, DBCompressionType, Options}; + +pub const CF_REPOSITORY_VIEW: &str = "repository_view"; +pub const CF_RAW_BY_HASH: &str = "raw_by_hash"; +pub const CF_RAW_BLOB: &str = "raw_blob"; +pub const CF_VCIR: &str = "vcir"; +pub const CF_MANIFEST_REPLAY_META: &str = "manifest_replay_meta"; +pub const CF_AUDIT_RULE_INDEX: &str = "audit_rule_index"; +pub const CF_RRDP_SOURCE: &str = "rrdp_source"; +pub const CF_RRDP_SOURCE_MEMBER: &str = "rrdp_source_member"; +pub const CF_RRDP_URI_OWNER: &str = "rrdp_uri_owner"; + +pub const ALL_COLUMN_FAMILY_NAMES: &[&str] = &[ + CF_REPOSITORY_VIEW, + CF_RAW_BY_HASH, + CF_RAW_BLOB, + CF_VCIR, + CF_MANIFEST_REPLAY_META, + CF_AUDIT_RULE_INDEX, + CF_RRDP_SOURCE, + CF_RRDP_SOURCE_MEMBER, + CF_RRDP_URI_OWNER, +]; + +pub(super) const REPOSITORY_VIEW_KEY_PREFIX: &str = "repo_view:"; +pub(super) const RAW_BY_HASH_KEY_PREFIX: &str = "rawbyhash:"; +pub(super) const RAW_BLOB_KEY_PREFIX: &str = "rawblob:"; +pub(super) const VCIR_KEY_PREFIX: &str = "vcir:"; +pub(super) const MANIFEST_REPLAY_META_KEY_PREFIX: &str = "manifest_replay_meta:"; +pub(super) const AUDIT_ROA_RULE_KEY_PREFIX: &str = "audit:roa_rule:"; +pub(super) const AUDIT_ASPA_RULE_KEY_PREFIX: &str = "audit:aspa_rule:"; +pub(super) const AUDIT_ROUTER_KEY_RULE_KEY_PREFIX: &str = "audit:router_key_rule:"; +pub(super) const RRDP_SOURCE_KEY_PREFIX: &str = "rrdp_source:"; +pub(super) const RRDP_SOURCE_MEMBER_KEY_PREFIX: &str = "rrdp_source_member:"; +pub(super) const RRDP_URI_OWNER_KEY_PREFIX: &str = "rrdp_uri_owner:"; + +const WORK_DB_BLOB_MODE_ENV: &str = "RPKI_WORK_DB_BLOB_MODE"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum WorkDbBlobMode { + Current, + Disabled, + Lz4, +} + +pub(super) fn parse_work_db_blob_mode(raw: &str) -> Option { + match raw.trim().to_ascii_lowercase().as_str() { + "" | "default" => Some(default_work_db_blob_mode()), + "current" | "legacy" => Some(WorkDbBlobMode::Current), + "disabled" | "disable" | "off" | "none" | "no_blob" | "no-blob" => { + Some(WorkDbBlobMode::Disabled) + } + "lz4" | "blob_lz4" | "blob-lz4" => Some(WorkDbBlobMode::Lz4), + _ => None, + } +} + +pub(super) fn default_work_db_blob_mode() -> WorkDbBlobMode { + WorkDbBlobMode::Disabled +} + +pub(super) fn work_db_blob_mode_from_env() -> WorkDbBlobMode { + let Ok(raw) = std::env::var(WORK_DB_BLOB_MODE_ENV) else { + return default_work_db_blob_mode(); + }; + match parse_work_db_blob_mode(&raw) { + Some(mode) => mode, + None => { + eprintln!( + "warning: unsupported {WORK_DB_BLOB_MODE_ENV}={raw:?}; using default work-db blobdb mode" + ); + default_work_db_blob_mode() + } + } +} + +pub(super) fn configure_work_db_options(opts: &mut Options, blob_mode: WorkDbBlobMode) { + opts.set_compression_type(DBCompressionType::Lz4); + match blob_mode { + WorkDbBlobMode::Current => enable_blobdb_current(opts), + WorkDbBlobMode::Disabled => {} + WorkDbBlobMode::Lz4 => { + enable_blobdb_current(opts); + opts.set_blob_compression_type(DBCompressionType::Lz4); + } + } +} + +pub(super) fn cf_opts(blob_mode: WorkDbBlobMode) -> Options { + let mut opts = Options::default(); + configure_work_db_options(&mut opts, blob_mode); + opts +} + +pub fn column_family_descriptors() -> Vec { + column_family_descriptors_for_blob_mode(work_db_blob_mode_from_env()) +} + +pub(super) fn column_family_descriptors_for_blob_mode( + blob_mode: WorkDbBlobMode, +) -> Vec { + ALL_COLUMN_FAMILY_NAMES + .iter() + .map(|name| ColumnFamilyDescriptor::new(*name, cf_opts(blob_mode))) + .collect() +} + +pub(super) fn enable_blobdb_current(opts: &mut Options) { + #[allow(unused_mut)] + let mut _enabled = false; + + #[allow(dead_code)] + fn _set(opts: &mut Options) { + opts.set_enable_blob_files(true); + opts.set_min_blob_size(1024); + } + + _set(opts); +} diff --git a/src/storage/keys.rs b/src/storage/keys.rs new file mode 100644 index 0000000..b985a32 --- /dev/null +++ b/src/storage/keys.rs @@ -0,0 +1,209 @@ +use serde::{Serialize, de::DeserializeOwned}; + +use crate::data_model::common::der_take_tlv; + +use super::config::*; +use super::pack::PackTime; +use super::{AuditRuleKind, StorageError, StorageResult, VcirOutputType}; + +pub(super) fn repository_view_key(rsync_uri: &str) -> String { + format!("{REPOSITORY_VIEW_KEY_PREFIX}{rsync_uri}") +} + +pub(super) fn repository_view_prefix(rsync_uri_prefix: &str) -> String { + format!("{REPOSITORY_VIEW_KEY_PREFIX}{rsync_uri_prefix}") +} + +pub(super) fn raw_by_hash_key(sha256_hex: &str) -> String { + format!("{RAW_BY_HASH_KEY_PREFIX}{sha256_hex}") +} + +pub(super) fn raw_blob_key(sha256_hex: &str) -> String { + format!("{RAW_BLOB_KEY_PREFIX}{sha256_hex}") +} + +pub(super) fn vcir_key(manifest_rsync_uri: &str) -> String { + format!("{VCIR_KEY_PREFIX}{manifest_rsync_uri}") +} + +pub(super) fn manifest_replay_meta_key(manifest_rsync_uri: &str) -> String { + format!("{MANIFEST_REPLAY_META_KEY_PREFIX}{manifest_rsync_uri}") +} + +pub(super) fn audit_rule_kind_for_output_type( + output_type: VcirOutputType, +) -> Option { + match output_type { + VcirOutputType::Vrp => Some(AuditRuleKind::Roa), + VcirOutputType::Aspa => Some(AuditRuleKind::Aspa), + VcirOutputType::RouterKey => Some(AuditRuleKind::RouterKey), + } +} + +pub(super) fn audit_rule_key(kind: AuditRuleKind, rule_hash: &str) -> String { + format!("{}{rule_hash}", kind.key_prefix()) +} + +pub(super) fn rrdp_source_key(notify_uri: &str) -> String { + format!("{RRDP_SOURCE_KEY_PREFIX}{notify_uri}") +} + +pub(super) fn rrdp_source_member_key(notify_uri: &str, rsync_uri: &str) -> String { + format!("{RRDP_SOURCE_MEMBER_KEY_PREFIX}{notify_uri}:{rsync_uri}") +} + +pub(super) fn rrdp_source_member_prefix(notify_uri: &str) -> String { + format!("{RRDP_SOURCE_MEMBER_KEY_PREFIX}{notify_uri}:") +} + +pub(super) fn rrdp_uri_owner_key(rsync_uri: &str) -> String { + format!("{RRDP_URI_OWNER_KEY_PREFIX}{rsync_uri}") +} + +pub(super) fn encode_cbor(value: &T, entity: &'static str) -> StorageResult> { + serde_cbor::to_vec(value).map_err(|e| StorageError::Codec { + entity, + detail: e.to_string(), + }) +} + +pub(super) fn decode_cbor( + bytes: &[u8], + entity: &'static str, +) -> StorageResult { + serde_cbor::from_slice(bytes).map_err(|e| StorageError::Codec { + entity, + detail: e.to_string(), + }) +} + +pub(super) fn validate_non_empty(field: &'static str, value: &str) -> StorageResult<()> { + if value.is_empty() { + return Err(StorageError::InvalidData { + entity: field, + detail: "must not be empty".to_string(), + }); + } + Ok(()) +} + +pub(super) fn validate_sha256_hex(field: &'static str, value: &str) -> StorageResult<()> { + if value.len() != 64 || !value.as_bytes().iter().all(u8::is_ascii_hexdigit) { + return Err(StorageError::InvalidData { + entity: field, + detail: "must be a 64-character lowercase or uppercase SHA-256 hex string".to_string(), + }); + } + Ok(()) +} + +pub(super) fn decode_sha256_hex_32(field: &'static str, value: &str) -> StorageResult<[u8; 32]> { + validate_sha256_hex(field, value)?; + let mut out = [0u8; 32]; + hex::decode_to_slice(value, &mut out).map_err(|e| StorageError::InvalidData { + entity: field, + detail: format!("hex decode failed: {e}"), + })?; + Ok(out) +} + +pub(super) fn validate_manifest_number_be(field: &'static str, value: &[u8]) -> StorageResult<()> { + if value.is_empty() { + return Err(StorageError::InvalidData { + entity: field, + detail: "must not be empty".to_string(), + }); + } + if value.len() > 20 { + return Err(StorageError::InvalidData { + entity: field, + detail: "must be at most 20 octets".to_string(), + }); + } + if value.len() > 1 && value[0] == 0 { + return Err(StorageError::InvalidData { + entity: field, + detail: "must be minimal big-endian without leading zeros".to_string(), + }); + } + Ok(()) +} + +pub(super) fn validate_sha256_digest_bytes(field: &'static str, value: &[u8]) -> StorageResult<()> { + if value.len() != 32 { + return Err(StorageError::InvalidData { + entity: field, + detail: format!("must be 32 bytes, got {}", value.len()), + }); + } + Ok(()) +} + +pub(super) fn validate_fixed_len_bytes( + field: &'static str, + value: &[u8], + expected_len: usize, +) -> StorageResult<()> { + if value.len() != expected_len { + return Err(StorageError::InvalidData { + entity: field, + detail: format!("must be {expected_len} bytes, got {}", value.len()), + }); + } + Ok(()) +} + +pub(super) fn validate_sorted_unique_fixed_len_bytes( + field: &'static str, + values: &[Vec], + expected_len: usize, +) -> StorageResult<()> { + for value in values { + validate_fixed_len_bytes(field, value, expected_len)?; + } + for window in values.windows(2) { + if window[0] >= window[1] { + return Err(StorageError::InvalidData { + entity: field, + detail: "must be strictly sorted and unique".to_string(), + }); + } + } + Ok(()) +} + +pub(super) fn validate_full_der_with_tag( + field: &'static str, + der: &[u8], + expected_tag: Option, +) -> StorageResult<()> { + let (tag, _value, rem) = der_take_tlv(der).map_err(|detail| StorageError::InvalidData { + entity: field, + detail, + })?; + if !rem.is_empty() { + return Err(StorageError::InvalidData { + entity: field, + detail: "trailing bytes after DER object".to_string(), + }); + } + if let Some(expected_tag) = expected_tag { + if tag != expected_tag { + return Err(StorageError::InvalidData { + entity: field, + detail: format!("unexpected tag 0x{tag:02X}, expected 0x{expected_tag:02X}"), + }); + } + } + Ok(()) +} + +pub(super) fn parse_time( + field: &'static str, + value: &PackTime, +) -> StorageResult { + value.parse().map_err(|detail| StorageError::InvalidData { + entity: field, + detail, + }) +} diff --git a/src/storage/pack.rs b/src/storage/pack.rs new file mode 100644 index 0000000..deda515 --- /dev/null +++ b/src/storage/pack.rs @@ -0,0 +1,200 @@ +use serde::{Deserialize, Serialize}; +use sha2::Digest; + +use crate::blob_store::{ExternalRawStoreDb, ExternalRepoBytesDb, RawObjectStore}; + +#[derive(Clone, Debug)] +pub enum PackBytes { + Eager(std::sync::Arc<[u8]>), + LazyExternal { + sha256_hex: String, + store: std::sync::Arc, + cache: std::sync::Arc>>, + }, + LazyRepoBytes { + sha256_hex: String, + store: std::sync::Arc, + cache: std::sync::Arc>>, + }, +} + +impl PackBytes { + pub fn eager(bytes: Vec) -> Self { + Self::Eager(std::sync::Arc::from(bytes)) + } + + pub fn lazy_external(sha256_hex: String, store: std::sync::Arc) -> Self { + Self::LazyExternal { + sha256_hex, + store, + cache: std::sync::Arc::new(std::sync::OnceLock::new()), + } + } + + pub fn lazy_repo_bytes(sha256_hex: String, store: std::sync::Arc) -> Self { + Self::LazyRepoBytes { + sha256_hex, + store, + cache: std::sync::Arc::new(std::sync::OnceLock::new()), + } + } + + pub fn as_slice(&self) -> Result<&[u8], String> { + match self { + Self::Eager(bytes) => Ok(bytes.as_ref()), + Self::LazyExternal { + sha256_hex, + store, + cache, + } => { + if cache.get().is_none() { + let bytes = store + .get_blob_bytes(sha256_hex) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("missing raw blob for sha256={sha256_hex}"))?; + let _ = cache.set(std::sync::Arc::from(bytes)); + } + let bytes = cache + .get() + .ok_or_else(|| format!("missing raw blob cache for sha256={sha256_hex}"))?; + Ok(bytes.as_ref()) + } + Self::LazyRepoBytes { + sha256_hex, + store, + cache, + } => { + if cache.get().is_none() { + let bytes = store + .get_blob_bytes(sha256_hex) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("missing repo bytes for sha256={sha256_hex}"))?; + let _ = cache.set(std::sync::Arc::from(bytes)); + } + let bytes = cache + .get() + .ok_or_else(|| format!("missing repo bytes cache for sha256={sha256_hex}"))?; + Ok(bytes.as_ref()) + } + } + } + + pub fn to_vec(&self) -> Result, String> { + Ok(self.as_slice()?.to_vec()) + } +} + +impl PartialEq for PackBytes { + fn eq(&self, other: &Self) -> bool { + match (self.as_slice(), other.as_slice()) { + (Ok(a), Ok(b)) => a == b, + _ => false, + } + } +} + +impl Eq for PackBytes {} + +#[derive(Clone, Debug)] +pub struct PackFile { + pub rsync_uri: String, + pub bytes: PackBytes, + pub sha256: [u8; 32], +} + +impl PackFile { + pub fn new(rsync_uri: impl Into, bytes: PackBytes, sha256: [u8; 32]) -> Self { + Self { + rsync_uri: rsync_uri.into(), + bytes, + sha256, + } + } + + pub fn from_bytes_with_sha256( + rsync_uri: impl Into, + bytes: Vec, + sha256: [u8; 32], + ) -> Self { + Self::new(rsync_uri, PackBytes::eager(bytes), sha256) + } + + pub fn from_lazy_external_raw_store( + rsync_uri: impl Into, + sha256_hex: String, + sha256: [u8; 32], + store: std::sync::Arc, + ) -> Self { + Self::new( + rsync_uri, + PackBytes::lazy_external(sha256_hex, store), + sha256, + ) + } + + pub fn from_lazy_repo_bytes( + rsync_uri: impl Into, + sha256_hex: String, + sha256: [u8; 32], + store: std::sync::Arc, + ) -> Self { + Self::new( + rsync_uri, + PackBytes::lazy_repo_bytes(sha256_hex, store), + sha256, + ) + } + + pub fn from_bytes_compute_sha256(rsync_uri: impl Into, bytes: Vec) -> Self { + let sha256 = compute_sha256_32(&bytes); + Self::new(rsync_uri, PackBytes::eager(bytes), sha256) + } + + pub fn bytes(&self) -> Result<&[u8], String> { + self.bytes.as_slice() + } + + pub fn bytes_cloned(&self) -> Result, String> { + self.bytes.to_vec() + } + + pub fn compute_sha256(&self) -> Result<[u8; 32], String> { + Ok(compute_sha256_32(self.bytes()?)) + } +} + +impl PartialEq for PackFile { + fn eq(&self, other: &Self) -> bool { + self.rsync_uri == other.rsync_uri + && self.sha256 == other.sha256 + && self.bytes == other.bytes + } +} + +impl Eq for PackFile {} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PackTime { + pub rfc3339_utc: String, +} + +impl PackTime { + pub fn from_utc_offset_datetime(t: time::OffsetDateTime) -> Self { + use time::format_description::well_known::Rfc3339; + let utc = t.to_offset(time::UtcOffset::UTC); + let s = utc.format(&Rfc3339).expect("format RFC 3339 UTC time"); + Self { rfc3339_utc: s } + } + + pub fn parse(&self) -> Result { + use time::format_description::well_known::Rfc3339; + time::OffsetDateTime::parse(&self.rfc3339_utc, &Rfc3339).map_err(|e| e.to_string()) + } +} + +pub(super) fn compute_sha256_32(bytes: &[u8]) -> [u8; 32] { + let digest = sha2::Sha256::digest(bytes); + let mut out = [0u8; 32]; + out.copy_from_slice(&digest); + out +} diff --git a/src/storage/tests.rs b/src/storage/tests.rs new file mode 100644 index 0000000..95abc3c --- /dev/null +++ b/src/storage/tests.rs @@ -0,0 +1,1454 @@ +use super::*; + +fn pack_time(hour: i64) -> PackTime { + PackTime::from_utc_offset_datetime( + time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(hour), + ) +} + +fn sha256_hex(input: &[u8]) -> String { + hex::encode(compute_sha256_32(input)) +} + +#[test] +fn parse_work_db_blob_mode_accepts_supported_values() { + assert_eq!(default_work_db_blob_mode(), WorkDbBlobMode::Disabled); + assert_eq!( + parse_work_db_blob_mode("default"), + Some(WorkDbBlobMode::Disabled) + ); + assert_eq!( + parse_work_db_blob_mode("current"), + Some(WorkDbBlobMode::Current) + ); + assert_eq!( + parse_work_db_blob_mode("legacy"), + Some(WorkDbBlobMode::Current) + ); + assert_eq!( + parse_work_db_blob_mode("disabled"), + Some(WorkDbBlobMode::Disabled) + ); + assert_eq!( + parse_work_db_blob_mode("no-blob"), + Some(WorkDbBlobMode::Disabled) + ); + assert_eq!(parse_work_db_blob_mode("lz4"), Some(WorkDbBlobMode::Lz4)); + assert_eq!( + parse_work_db_blob_mode("blob-lz4"), + Some(WorkDbBlobMode::Lz4) + ); + assert_eq!(parse_work_db_blob_mode("unexpected"), None); +} + +fn sample_repository_view_entry(rsync_uri: &str, bytes: &[u8]) -> RepositoryViewEntry { + RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(sha256_hex(bytes)), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("cer".to_string()), + state: RepositoryViewState::Present, + } +} + +fn sample_raw_by_hash_entry(bytes: Vec) -> RawByHashEntry { + RawByHashEntry { + sha256_hex: sha256_hex(&bytes), + bytes, + origin_uris: vec!["rsync://example.test/repo/object.cer".to_string()], + object_type: Some("cer".to_string()), + encoding: Some("der".to_string()), + } +} + +fn sample_ccr_manifest_projection( + manifest_rsync_uri: &str, + manifest_this_update: PackTime, + subordinate_skis: Vec>, +) -> VcirCcrManifestProjection { + VcirCcrManifestProjection { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + manifest_sha256: vec![0x11; 32], + manifest_size: 4096, + manifest_ee_aki: vec![0x22; 20], + manifest_number_be: vec![3], + manifest_this_update, + manifest_sia_locations_der: vec![vec![ + 0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05, + b'r', b's', b'y', b'n', b'c', + ]], + subordinate_skis, + } +} + +fn sample_vcir(manifest_rsync_uri: &str) -> ValidatedCaInstanceResult { + let roa_bytes = b"roa-object".to_vec(); + let ee_bytes = b"ee-cert".to_vec(); + let child_bytes = b"child-cert".to_vec(); + let child_ski = "1234567890abcdef1234567890abcdef12345678".to_string(); + ValidatedCaInstanceResult { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + parent_manifest_rsync_uri: Some("rsync://example.test/repo/parent/parent.mft".to_string()), + tal_id: "apnic".to_string(), + ca_subject_name: "CN=Example CA".to_string(), + ca_ski: "00112233445566778899aabbccddeeff00112233".to_string(), + issuer_ski: "ffeeddccbbaa99887766554433221100ffeeddcc".to_string(), + last_successful_validation_time: pack_time(0), + current_manifest_rsync_uri: manifest_rsync_uri.to_string(), + current_crl_rsync_uri: "rsync://example.test/repo/current.crl".to_string(), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: vec![3], + validated_manifest_this_update: pack_time(0), + validated_manifest_next_update: pack_time(24), + }, + ccr_manifest_projection: sample_ccr_manifest_projection( + manifest_rsync_uri, + pack_time(0), + vec![hex::decode(&child_ski).expect("decode child ski")], + ), + instance_gate: VcirInstanceGate { + manifest_next_update: pack_time(24), + current_crl_next_update: pack_time(12), + self_ca_not_after: pack_time(48), + instance_effective_until: pack_time(12), + }, + child_entries: vec![VcirChildEntry { + child_manifest_rsync_uri: "rsync://example.test/repo/child/child.mft".to_string(), + child_cert_rsync_uri: "rsync://example.test/repo/child/child.cer".to_string(), + child_cert_hash: sha256_hex(&child_bytes), + child_ski, + child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(), + child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), + child_rrdp_notification_uri: Some("https://example.test/child-notify.xml".to_string()), + child_effective_ip_resources: None, + child_effective_as_resources: None, + accepted_at_validation_time: pack_time(0), + }], + local_outputs: vec![ + VcirLocalOutput { + output_id: "vrp-1".to_string(), + output_type: VcirOutputType::Vrp, + item_effective_until: pack_time(12), + source_object_uri: "rsync://example.test/repo/object.roa".to_string(), + source_object_type: "roa".to_string(), + source_object_hash: sha256_hex(&roa_bytes), + source_ee_cert_hash: sha256_hex(&ee_bytes), + payload_json: r#"{"asn":64496,"prefix":"203.0.113.0/24"}"#.to_string(), + rule_hash: sha256_hex(b"vrp-rule-1"), + validation_path_hint: vec![manifest_rsync_uri.to_string()], + }, + VcirLocalOutput { + output_id: "aspa-1".to_string(), + output_type: VcirOutputType::Aspa, + item_effective_until: pack_time(10), + source_object_uri: "rsync://example.test/repo/object.asa".to_string(), + source_object_type: "aspa".to_string(), + source_object_hash: sha256_hex(b"aspa-object"), + source_ee_cert_hash: sha256_hex(b"aspa-ee-cert"), + payload_json: r#"{"customer_as":64496,"providers":[64497]}"#.to_string(), + rule_hash: sha256_hex(b"aspa-rule-1"), + validation_path_hint: vec![manifest_rsync_uri.to_string()], + }, + ], + related_artifacts: vec![ + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::Manifest, + artifact_kind: VcirArtifactKind::Mft, + uri: Some(manifest_rsync_uri.to_string()), + sha256: sha256_hex(b"manifest-object"), + object_type: Some("mft".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::CurrentCrl, + artifact_kind: VcirArtifactKind::Crl, + uri: Some("rsync://example.test/repo/current.crl".to_string()), + sha256: sha256_hex(b"current-crl"), + object_type: Some("crl".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + }, + ], + summary: VcirSummary { + local_vrp_count: 1, + local_aspa_count: 1, + local_router_key_count: 0, + child_count: 1, + accepted_object_count: 4, + rejected_object_count: 0, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible: true, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: vec!["validated-fresh".to_string()], + }, + } +} + +#[test] +fn vcir_ccr_manifest_projection_validate_accepts_valid_projection() { + let projection = sample_ccr_manifest_projection( + "rsync://example.test/repo/current.mft", + pack_time(0), + vec![vec![0x33; 20], vec![0x44; 20]], + ); + projection.validate_internal().expect("valid projection"); +} + +#[test] +fn vcir_ccr_manifest_projection_validate_rejects_invalid_fields() { + let mut bad_hash = sample_ccr_manifest_projection( + "rsync://example.test/repo/current.mft", + pack_time(0), + vec![vec![0x33; 20]], + ); + bad_hash.manifest_sha256 = vec![0x11; 31]; + assert!(matches!( + bad_hash.validate_internal(), + Err(StorageError::InvalidData { .. }) + )); + + let mut bad_locations = sample_ccr_manifest_projection( + "rsync://example.test/repo/current.mft", + pack_time(0), + vec![vec![0x33; 20]], + ); + bad_locations.manifest_sia_locations_der = vec![vec![0x04, 0x00]]; + assert!(matches!( + bad_locations.validate_internal(), + Err(StorageError::InvalidData { .. }) + )); + + let bad_subordinates = sample_ccr_manifest_projection( + "rsync://example.test/repo/current.mft", + pack_time(0), + vec![vec![0x44; 20], vec![0x33; 20]], + ); + assert!(matches!( + bad_subordinates.validate_internal(), + Err(StorageError::InvalidData { .. }) + )); +} + +fn sample_audit_rule_entry(kind: AuditRuleKind) -> AuditRuleIndexEntry { + AuditRuleIndexEntry { + kind, + rule_hash: sha256_hex(match kind { + AuditRuleKind::Roa => b"roa-index-rule", + AuditRuleKind::Aspa => b"aspa-index-rule", + AuditRuleKind::RouterKey => b"router-key-index-rule", + }), + manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), + source_object_uri: match kind { + AuditRuleKind::Roa => "rsync://example.test/repo/object.roa".to_string(), + AuditRuleKind::Aspa => "rsync://example.test/repo/object.asa".to_string(), + AuditRuleKind::RouterKey => "rsync://example.test/repo/router.cer".to_string(), + }, + source_object_hash: sha256_hex(match kind { + AuditRuleKind::Roa => b"roa-object", + AuditRuleKind::Aspa => b"aspa-object", + AuditRuleKind::RouterKey => b"router-key-object", + }), + output_id: match kind { + AuditRuleKind::Roa => "vrp-1".to_string(), + AuditRuleKind::Aspa => "aspa-1".to_string(), + AuditRuleKind::RouterKey => "router-key-1".to_string(), + }, + item_effective_until: pack_time(12), + } +} + +fn sample_rrdp_source_record(notify_uri: &str) -> RrdpSourceRecord { + RrdpSourceRecord { + notify_uri: notify_uri.to_string(), + last_session_id: Some("session-1".to_string()), + last_serial: Some(42), + first_seen_at: pack_time(0), + last_seen_at: pack_time(1), + last_sync_at: Some(pack_time(1)), + sync_state: RrdpSourceSyncState::DeltaReady, + last_snapshot_uri: Some("https://rrdp.example.test/snapshot.xml".to_string()), + last_snapshot_hash: Some(sha256_hex(b"snapshot-bytes")), + last_error: None, + } +} + +fn sample_rrdp_source_member_record( + notify_uri: &str, + rsync_uri: &str, + serial: u64, +) -> RrdpSourceMemberRecord { + RrdpSourceMemberRecord { + notify_uri: notify_uri.to_string(), + rsync_uri: rsync_uri.to_string(), + current_hash: Some(sha256_hex(rsync_uri.as_bytes())), + object_type: Some("cer".to_string()), + present: true, + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: serial, + last_changed_at: pack_time(serial as i64), + } +} + +fn sample_rrdp_uri_owner_record(notify_uri: &str, rsync_uri: &str) -> RrdpUriOwnerRecord { + RrdpUriOwnerRecord { + rsync_uri: rsync_uri.to_string(), + notify_uri: notify_uri.to_string(), + current_hash: Some(sha256_hex(rsync_uri.as_bytes())), + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: 7, + last_changed_at: pack_time(7), + owner_state: RrdpUriOwnerState::Active, + } +} + +#[test] +fn repository_view_and_raw_by_hash_roundtrip() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let entry1 = sample_repository_view_entry("rsync://example.test/repo/a.cer", b"object-a"); + let entry2 = sample_repository_view_entry("rsync://example.test/repo/sub/b.roa", b"object-b"); + store + .put_repository_view_entry(&entry1) + .expect("put repository view entry1"); + store + .put_repository_view_entry(&entry2) + .expect("put repository view entry2"); + + let got1 = store + .get_repository_view_entry(&entry1.rsync_uri) + .expect("get repository view entry1") + .expect("entry1 exists"); + assert_eq!(got1, entry1); + + let got_prefix = store + .list_repository_view_entries_with_prefix("rsync://example.test/repo/sub/") + .expect("list repository view prefix"); + assert_eq!(got_prefix, vec![entry2.clone()]); + + store + .delete_repository_view_entry(&entry1.rsync_uri) + .expect("delete repository view entry1"); + assert!( + store + .get_repository_view_entry(&entry1.rsync_uri) + .expect("get deleted repository view entry1") + .is_none() + ); + + let raw = sample_raw_by_hash_entry(b"raw-der-object".to_vec()); + store + .put_raw_by_hash_entry(&raw) + .expect("put raw_by_hash entry"); + let got_raw = store + .get_raw_by_hash_entry(&raw.sha256_hex) + .expect("get raw_by_hash entry") + .expect("raw entry exists"); + assert_eq!(got_raw, raw); +} + +#[test] +fn raw_by_hash_routes_to_external_raw_store_when_configured() { + let td = tempfile::tempdir().expect("tempdir"); + let main_db = td.path().join("main-db"); + let raw_db = td.path().join("raw-store.db"); + + let raw = sample_raw_by_hash_entry(b"external-raw".to_vec()); + { + let store = + RocksStore::open_with_external_raw_store(&main_db, &raw_db).expect("open store"); + store.put_raw_by_hash_entry(&raw).expect("put external raw"); + + let got = store + .get_raw_by_hash_entry(&raw.sha256_hex) + .expect("get external raw") + .expect("raw exists"); + assert_eq!(got, raw); + } + + let main_store = RocksStore::open(&main_db).expect("open main only"); + assert!( + main_store + .get_raw_by_hash_entry(&raw.sha256_hex) + .expect("read main store") + .is_none(), + "main db should not contain raw entry when external raw store is configured" + ); +} + +#[test] +fn put_blob_bytes_batch_uses_internal_blob_cf_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let bytes = b"internal-blob-only".to_vec(); + let hash = sha256_hex(&bytes); + + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("put blob bytes"); + + assert_eq!( + store.get_blob_bytes(&hash).expect("get blob bytes"), + Some(bytes.clone()) + ); + assert!( + store + .get_raw_by_hash_entry(&hash) + .expect("get raw entry") + .is_none() + ); +} + +#[test] +fn put_blob_bytes_batch_routes_to_external_raw_store_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open_with_external_raw_store( + &td.path().join("main-db"), + &td.path().join("raw-store.db"), + ) + .expect("open store"); + let bytes = b"external-blob-only".to_vec(); + let hash = sha256_hex(&bytes); + + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("put external blob bytes"); + + assert_eq!(store.get_blob_bytes(&hash).unwrap(), Some(bytes)); + assert!(store.get_raw_by_hash_entry(&hash).unwrap().is_none()); +} + +#[test] +fn repo_bytes_db_is_physically_separate_from_external_raw_store() { + let td = tempfile::tempdir().expect("tempdir"); + let main_db = td.path().join("main-db"); + let raw_db = td.path().join("raw-store.db"); + let repo_bytes_db = td.path().join("repo-bytes.db"); + let store = + RocksStore::open_with_external_stores(&main_db, Some(&raw_db), Some(&repo_bytes_db)) + .expect("open store"); + let repo_bytes = b"repo-object".to_vec(); + let repo_hash = sha256_hex(&repo_bytes); + let raw = sample_raw_by_hash_entry(b"raw-evidence".to_vec()); + + store + .put_blob_bytes_batch(&[(repo_hash.clone(), repo_bytes.clone())]) + .expect("put repo bytes"); + store.put_raw_by_hash_entry(&raw).expect("put raw evidence"); + + assert_eq!(store.get_blob_bytes(&repo_hash).unwrap(), Some(repo_bytes)); + assert_eq!( + store.get_raw_by_hash_entry(&raw.sha256_hex).unwrap(), + Some(raw.clone()) + ); + drop(store); + + let raw_only = RocksStore::open_with_external_raw_store(&td.path().join("raw-reader"), &raw_db) + .expect("open raw only"); + assert!( + raw_only.get_blob_bytes(&repo_hash).unwrap().is_none(), + "repo object bytes must not be written into raw-store.db" + ); + + let repo_only = + RocksStore::open_with_external_repo_bytes(&td.path().join("repo-reader"), &repo_bytes_db) + .expect("open repo bytes only"); + assert_eq!( + repo_only.get_blob_bytes(&repo_hash).unwrap(), + Some(b"repo-object".to_vec()) + ); + assert!( + repo_only.get_blob_bytes(&raw.sha256_hex).unwrap().is_none(), + "raw evidence bytes must not be written into repo-bytes.db" + ); +} + +#[test] +fn put_blob_bytes_batch_accepts_empty_batch_with_external_raw_store() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open_with_external_raw_store( + &td.path().join("main-db"), + &td.path().join("raw-store.db"), + ) + .expect("open store"); + + store + .put_blob_bytes_batch(&[]) + .expect("empty external blob batch should be a no-op"); +} + +#[test] +fn get_blob_bytes_internal_falls_back_to_raw_entry_when_blob_missing() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let raw = sample_raw_by_hash_entry(b"raw-fallback".to_vec()); + + store.put_raw_by_hash_entry(&raw).expect("put raw entry"); + + assert_eq!( + store + .get_blob_bytes(&raw.sha256_hex) + .expect("get blob bytes via raw fallback"), + Some(raw.bytes.clone()) + ); +} + +#[test] +fn get_blob_bytes_batch_internal_prefers_blob_cf_and_falls_back_to_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let blob_bytes = b"blob-cf-object".to_vec(); + let blob_hash = sha256_hex(&blob_bytes); + store + .put_blob_bytes_batch(&[(blob_hash.clone(), blob_bytes.clone())]) + .expect("put blob bytes"); + + let raw = sample_raw_by_hash_entry(b"raw-fallback-batch".to_vec()); + store.put_raw_by_hash_entry(&raw).expect("put raw fallback"); + + let batch = store + .get_blob_bytes_batch(&[blob_hash.clone(), raw.sha256_hex.clone(), "00".repeat(32)]) + .expect("get blob bytes batch"); + assert_eq!(batch, vec![Some(blob_bytes), Some(raw.bytes.clone()), None]); +} + +#[test] +fn get_blob_bytes_batch_routes_to_external_raw_store_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open_with_external_raw_store( + &td.path().join("main-db"), + &td.path().join("raw-store.db"), + ) + .expect("open store"); + let bytes = b"external-batch-blob".to_vec(); + let hash = sha256_hex(&bytes); + + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("put external blob bytes"); + + assert_eq!( + store + .get_blob_bytes_batch(&[hash, "00".repeat(32)]) + .expect("get external blob batch"), + vec![Some(bytes), None] + ); +} + +#[test] +fn get_blob_bytes_rejects_invalid_hash_for_internal_store() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let err = store + .get_blob_bytes("not-a-valid-hash") + .expect_err("invalid hash must fail"); + assert!(matches!(err, StorageError::InvalidData { .. })); +} + +#[test] +fn get_blob_bytes_batch_rejects_invalid_hash_for_internal_store() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let err = store + .get_blob_bytes_batch(&["not-a-valid-hash".to_string()]) + .expect_err("invalid hash must fail"); + assert!(matches!(err, StorageError::InvalidData { .. })); +} + +#[test] +fn get_blob_bytes_batch_returns_empty_for_empty_request_internal() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + assert!( + store + .get_blob_bytes_batch(&[]) + .expect("empty blob batch request") + .is_empty() + ); +} + +#[test] +fn put_blob_bytes_batch_accepts_empty_batch() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + store + .put_blob_bytes_batch(&[]) + .expect("empty blob batch should be a no-op"); +} + +#[test] +fn put_blob_bytes_batch_rejects_empty_bytes() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let err = store + .put_blob_bytes_batch(&[(sha256_hex(b"valid"), Vec::new())]) + .expect_err("empty bytes must fail"); + assert!(matches!(err, StorageError::InvalidData { .. })); +} + +#[test] +fn delete_raw_by_hash_entry_internal_preserves_blob_bytes() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let bytes = b"blob-persists-after-raw-delete".to_vec(); + let hash = sha256_hex(&bytes); + let raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone()); + + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("put blob bytes"); + store.put_raw_by_hash_entry(&raw).expect("put raw entry"); + + store + .delete_raw_by_hash_entry(&hash) + .expect("delete raw entry only"); + + assert!(store.get_raw_by_hash_entry(&hash).unwrap().is_none()); + assert_eq!(store.get_blob_bytes(&hash).unwrap(), Some(bytes)); +} + +#[test] +fn delete_raw_by_hash_entry_rejects_invalid_hash() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let err = store + .delete_raw_by_hash_entry("not-a-valid-hash") + .expect_err("invalid hash must fail"); + assert!(matches!(err, StorageError::InvalidData { .. })); +} + +#[test] +fn delete_raw_by_hash_entry_routes_to_external_raw_store() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open_with_external_raw_store( + &td.path().join("main-db"), + &td.path().join("raw-store.db"), + ) + .expect("open store"); + let raw = sample_raw_by_hash_entry(b"external-delete".to_vec()); + + store.put_raw_by_hash_entry(&raw).expect("put raw entry"); + store + .delete_raw_by_hash_entry(&raw.sha256_hex) + .expect("delete external raw entry"); + + assert!( + store + .get_raw_by_hash_entry(&raw.sha256_hex) + .unwrap() + .is_none() + ); + assert!(store.get_blob_bytes(&raw.sha256_hex).unwrap().is_none()); +} + +#[test] +fn repository_view_and_raw_by_hash_validation_errors_are_reported() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let invalid_view = RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/withdrawn.cer".to_string(), + current_hash: None, + repository_source: None, + object_type: None, + state: RepositoryViewState::Present, + }; + let err = store + .put_repository_view_entry(&invalid_view) + .expect_err("missing current_hash must fail"); + assert!(err.to_string().contains("current_hash is required")); + + let invalid_raw = RawByHashEntry { + sha256_hex: sha256_hex(b"expected"), + bytes: b"actual".to_vec(), + origin_uris: vec!["rsync://example.test/repo/object.cer".to_string()], + object_type: None, + encoding: None, + }; + let err = store + .put_raw_by_hash_entry(&invalid_raw) + .expect_err("mismatched raw_by_hash entry must fail"); + assert!(err.to_string().contains("does not match bytes")); +} + +#[test] +fn vcir_roundtrip_and_validation_failures_are_reported() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + store.put_vcir(&vcir).expect("put vcir"); + let got = store + .get_vcir(&vcir.manifest_rsync_uri) + .expect("get vcir") + .expect("vcir exists"); + assert_eq!(got, vcir); + let replay_meta = store + .get_manifest_replay_meta(&vcir.manifest_rsync_uri) + .expect("get manifest replay meta") + .expect("manifest replay meta exists"); + assert_eq!( + replay_meta, + ManifestReplayMeta { + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + manifest_number_be: vcir + .validated_manifest_meta + .validated_manifest_number + .clone(), + manifest_this_update: vcir + .validated_manifest_meta + .validated_manifest_this_update + .clone(), + manifest_sha256: vcir.ccr_manifest_projection.manifest_sha256.clone(), + updated_at_validation_time: vcir.last_successful_validation_time.clone(), + } + ); + + let mut invalid = sample_vcir("rsync://example.test/repo/invalid.mft"); + invalid.summary.local_vrp_count = 9; + let err = store + .put_vcir(&invalid) + .expect_err("invalid vcir must fail"); + assert!(err.to_string().contains("local_vrp_count=9")); + + let mut invalid = sample_vcir("rsync://example.test/repo/invalid-2.mft"); + invalid.instance_gate.instance_effective_until = pack_time(11); + let err = store + .put_vcir(&invalid) + .expect_err("invalid instance gate must fail"); + assert!(err.to_string().contains("instance_effective_until")); + + store + .delete_vcir(&vcir.manifest_rsync_uri) + .expect("delete vcir"); + assert!( + store + .get_vcir(&vcir.manifest_rsync_uri) + .expect("get deleted vcir") + .is_none() + ); + assert!( + store + .get_manifest_replay_meta(&vcir.manifest_rsync_uri) + .expect("get deleted manifest replay meta") + .is_none() + ); +} + +#[test] +fn manifest_replay_meta_validation_reports_invalid_fields() { + let mut meta = ManifestReplayMeta { + manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), + manifest_number_be: vec![3], + manifest_this_update: pack_time(0), + manifest_sha256: vec![0x11; 32], + updated_at_validation_time: pack_time(1), + }; + meta.validate_internal().expect("valid replay meta"); + + meta.manifest_sha256 = vec![0x11; 31]; + let err = meta + .validate_internal() + .expect_err("short manifest sha must fail"); + assert!(err.to_string().contains("must be 32 bytes")); + + meta.manifest_sha256 = vec![0x11; 32]; + meta.manifest_number_be = vec![0, 3]; + let err = meta + .validate_internal() + .expect_err("non-minimal manifest number must fail"); + assert!(err.to_string().contains("minimal big-endian")); +} + +#[test] +fn list_vcirs_returns_all_entries() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let vcir1 = sample_vcir("rsync://example.test/repo/a.mft"); + let vcir2 = sample_vcir("rsync://example.test/repo/b.mft"); + store.put_vcir(&vcir1).expect("put vcir1"); + store.put_vcir(&vcir2).expect("put vcir2"); + + let mut got = store.list_vcirs().expect("list vcirs"); + got.sort_by(|a, b| a.manifest_rsync_uri.cmp(&b.manifest_rsync_uri)); + assert_eq!(got, vec![vcir1, vcir2]); +} + +#[test] +fn audit_rule_index_roundtrip_for_roa_aspa_and_router_key() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let roa = sample_audit_rule_entry(AuditRuleKind::Roa); + let aspa = sample_audit_rule_entry(AuditRuleKind::Aspa); + let router_key = sample_audit_rule_entry(AuditRuleKind::RouterKey); + store + .put_audit_rule_index_entry(&roa) + .expect("put roa audit rule entry"); + store + .put_audit_rule_index_entry(&aspa) + .expect("put aspa audit rule entry"); + store + .put_audit_rule_index_entry(&router_key) + .expect("put router key audit rule entry"); + + let got_roa = store + .get_audit_rule_index_entry(AuditRuleKind::Roa, &roa.rule_hash) + .expect("get roa audit rule entry") + .expect("roa entry exists"); + let got_aspa = store + .get_audit_rule_index_entry(AuditRuleKind::Aspa, &aspa.rule_hash) + .expect("get aspa audit rule entry") + .expect("aspa entry exists"); + let got_router_key = store + .get_audit_rule_index_entry(AuditRuleKind::RouterKey, &router_key.rule_hash) + .expect("get router key audit rule entry") + .expect("router key entry exists"); + assert_eq!(got_roa, roa); + assert_eq!(got_aspa, aspa); + assert_eq!(got_router_key, router_key); + + store + .delete_audit_rule_index_entry(AuditRuleKind::Roa, &roa.rule_hash) + .expect("delete roa audit rule entry"); + assert!( + store + .get_audit_rule_index_entry(AuditRuleKind::Roa, &roa.rule_hash) + .expect("get deleted roa audit rule entry") + .is_none() + ); + + let mut invalid = sample_audit_rule_entry(AuditRuleKind::Roa); + invalid.rule_hash = "bad".to_string(); + let err = store + .put_audit_rule_index_entry(&invalid) + .expect_err("invalid audit rule hash must fail"); + assert!(err.to_string().contains("64-character")); +} + +#[test] +fn replace_vcir_and_audit_rule_indexes_replaces_previous_entries_in_one_step() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let mut previous = sample_vcir("rsync://example.test/repo/current.mft"); + previous.local_outputs = vec![VcirLocalOutput { + output_id: "old-output".to_string(), + output_type: VcirOutputType::Vrp, + item_effective_until: pack_time(10), + source_object_uri: "rsync://example.test/repo/old.roa".to_string(), + source_object_type: "roa".to_string(), + source_object_hash: sha256_hex(b"old-roa"), + source_ee_cert_hash: sha256_hex(b"old-ee"), + payload_json: "{}".to_string(), + rule_hash: sha256_hex(b"old-rule"), + validation_path_hint: vec![previous.manifest_rsync_uri.clone()], + }]; + previous.summary.local_vrp_count = 1; + previous.summary.local_aspa_count = 0; + previous.summary.local_router_key_count = 0; + store + .replace_vcir_and_audit_rule_indexes(None, &previous) + .expect("store previous vcir"); + assert!( + store + .get_audit_rule_index_entry(AuditRuleKind::Roa, &previous.local_outputs[0].rule_hash) + .expect("get old audit entry") + .is_some() + ); + + let mut current = sample_vcir("rsync://example.test/repo/current.mft"); + current.local_outputs = vec![VcirLocalOutput { + output_id: "new-output".to_string(), + output_type: VcirOutputType::Aspa, + item_effective_until: pack_time(11), + source_object_uri: "rsync://example.test/repo/new.asa".to_string(), + source_object_type: "aspa".to_string(), + source_object_hash: sha256_hex(b"new-aspa"), + source_ee_cert_hash: sha256_hex(b"new-ee"), + payload_json: "{}".to_string(), + rule_hash: sha256_hex(b"new-rule"), + validation_path_hint: vec![current.manifest_rsync_uri.clone()], + }]; + current.summary.local_vrp_count = 0; + current.summary.local_aspa_count = 1; + store + .replace_vcir_and_audit_rule_indexes(Some(&previous), ¤t) + .expect("replace vcir and audit indexes"); + + let got = store + .get_vcir(¤t.manifest_rsync_uri) + .expect("get replaced vcir") + .expect("vcir exists"); + assert_eq!(got, current); + let replay_meta = store + .get_manifest_replay_meta(¤t.manifest_rsync_uri) + .expect("get replaced replay meta") + .expect("replay meta exists"); + assert_eq!( + replay_meta.manifest_number_be, + current.validated_manifest_meta.validated_manifest_number + ); + assert_eq!( + replay_meta.manifest_sha256, + current.ccr_manifest_projection.manifest_sha256 + ); + assert!( + store + .get_audit_rule_index_entry(AuditRuleKind::Roa, &previous.local_outputs[0].rule_hash) + .expect("get deleted old audit entry") + .is_none() + ); + assert!( + store + .get_audit_rule_index_entry(AuditRuleKind::Aspa, ¤t.local_outputs[0].rule_hash) + .expect("get new audit entry") + .is_some() + ); +} + +#[test] +fn storage_helpers_cover_optional_validation_paths() { + let withdrawn = RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/withdrawn.cer".to_string(), + current_hash: Some(sha256_hex(b"withdrawn")), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("cer".to_string()), + state: RepositoryViewState::Withdrawn, + }; + withdrawn + .validate_internal() + .expect("withdrawn repository view validates"); + + let raw = RawByHashEntry::from_bytes(sha256_hex(b"helper-bytes"), b"helper-bytes".to_vec()); + raw.validate_internal() + .expect("raw_by_hash helper validates"); + + let empty_raw = RawByHashEntry { + sha256_hex: sha256_hex(b"x"), + bytes: Vec::new(), + origin_uris: Vec::new(), + object_type: None, + encoding: None, + }; + let err = empty_raw + .validate_internal() + .expect_err("empty raw bytes must fail"); + assert!(err.to_string().contains("bytes must not be empty")); + + let duplicate_origin_raw = RawByHashEntry { + sha256_hex: sha256_hex(b"dup-origin"), + bytes: b"dup-origin".to_vec(), + origin_uris: vec![ + "rsync://example.test/repo/object.cer".to_string(), + "rsync://example.test/repo/object.cer".to_string(), + ], + object_type: Some("cer".to_string()), + encoding: Some("der".to_string()), + }; + let err = duplicate_origin_raw + .validate_internal() + .expect_err("duplicate origin URI must fail"); + assert!(err.to_string().contains("duplicate origin URI")); +} + +#[test] +fn rrdp_source_optional_fields_and_owner_without_hash_validate() { + let source = RrdpSourceRecord { + notify_uri: "https://rrdp.example.test/notification.xml".to_string(), + last_session_id: None, + last_serial: None, + first_seen_at: pack_time(0), + last_seen_at: pack_time(1), + last_sync_at: None, + sync_state: RrdpSourceSyncState::Empty, + last_snapshot_uri: None, + last_snapshot_hash: None, + last_error: Some("network timeout".to_string()), + }; + source + .validate_internal() + .expect("source with optional fields validates"); + + let owner = RrdpUriOwnerRecord { + rsync_uri: "rsync://example.test/repo/object.cer".to_string(), + notify_uri: "https://rrdp.example.test/notification.xml".to_string(), + current_hash: None, + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: 5, + last_changed_at: pack_time(5), + owner_state: RrdpUriOwnerState::Withdrawn, + }; + owner + .validate_internal() + .expect("owner without hash validates when withdrawn"); +} + +#[test] +fn rrdp_source_binding_records_roundtrip_and_prefix_iteration() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let notify_uri = "https://rrdp.example.test/notification.xml"; + let source = sample_rrdp_source_record(notify_uri); + store + .put_rrdp_source_record(&source) + .expect("put rrdp source record"); + let got_source = store + .get_rrdp_source_record(notify_uri) + .expect("get rrdp source record") + .expect("rrdp source exists"); + assert_eq!(got_source, source); + + let member1 = + sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/a.cer", 1); + let member2 = + sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/b.roa", 2); + let other_member = sample_rrdp_source_member_record( + "https://other.example.test/notification.xml", + "rsync://other.example.test/repo/c.cer", + 3, + ); + store + .put_rrdp_source_member_record(&member1) + .expect("put member1"); + store + .put_rrdp_source_member_record(&member2) + .expect("put member2"); + store + .put_rrdp_source_member_record(&other_member) + .expect("put other member"); + + let mut members = store + .list_rrdp_source_member_records(notify_uri) + .expect("list rrdp source members"); + members.sort_by(|a, b| a.rsync_uri.cmp(&b.rsync_uri)); + assert_eq!(members, vec![member1.clone(), member2.clone()]); + + let got_member = store + .get_rrdp_source_member_record(notify_uri, &member1.rsync_uri) + .expect("get member1") + .expect("member1 exists"); + assert_eq!(got_member, member1); + + let owner = sample_rrdp_uri_owner_record(notify_uri, &member1.rsync_uri); + store + .put_rrdp_uri_owner_record(&owner) + .expect("put uri owner record"); + let got_owner = store + .get_rrdp_uri_owner_record(&member1.rsync_uri) + .expect("get uri owner record") + .expect("uri owner exists"); + assert_eq!(got_owner, owner); + store + .delete_rrdp_uri_owner_record(&member1.rsync_uri) + .expect("delete uri owner record"); + assert!( + store + .get_rrdp_uri_owner_record(&member1.rsync_uri) + .expect("get deleted uri owner") + .is_none() + ); + + let mut invalid_source = sample_rrdp_source_record("https://invalid.example/notification.xml"); + invalid_source.last_snapshot_hash = Some("bad".to_string()); + let err = store + .put_rrdp_source_record(&invalid_source) + .expect_err("invalid source hash must fail"); + assert!(err.to_string().contains("last_snapshot_hash")); + + let invalid_member = RrdpSourceMemberRecord { + notify_uri: notify_uri.to_string(), + rsync_uri: "rsync://example.test/repo/deleted.cer".to_string(), + current_hash: None, + object_type: None, + present: true, + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: 10, + last_changed_at: pack_time(10), + }; + let err = store + .put_rrdp_source_member_record(&invalid_member) + .expect_err("present member without hash must fail"); + assert!(err.to_string().contains("current_hash is required")); +} +#[test] +fn projection_batch_roundtrip_writes_repository_view_member_and_owner_records() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(dir.path()).expect("open store"); + + let view = RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + current_hash: Some(hex::encode([1u8; 32])), + repository_source: Some("https://example.test/notify.xml".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }; + let member = RrdpSourceMemberRecord { + notify_uri: "https://example.test/notify.xml".to_string(), + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + current_hash: Some(hex::encode([1u8; 32])), + object_type: Some("roa".to_string()), + present: true, + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: 7, + last_changed_at: pack_time(1), + }; + let owner = RrdpUriOwnerRecord { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + notify_uri: "https://example.test/notify.xml".to_string(), + current_hash: Some(hex::encode([1u8; 32])), + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: 7, + last_changed_at: pack_time(1), + owner_state: RrdpUriOwnerState::Active, + }; + + store + .put_projection_batch(&[view.clone()], &[member.clone()], &[owner.clone()]) + .expect("write projection batch"); + + assert_eq!( + store + .get_repository_view_entry(&view.rsync_uri) + .expect("get view") + .expect("present view"), + view + ); + assert_eq!( + store + .get_rrdp_source_member_record(&member.notify_uri, &member.rsync_uri) + .expect("get member") + .expect("present member"), + member + ); + assert_eq!( + store + .get_rrdp_uri_owner_record(&owner.rsync_uri) + .expect("get owner") + .expect("present owner"), + owner + ); +} + +#[test] +fn current_rrdp_source_member_helpers_filter_present_records() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let notify_uri = "https://rrdp.example.test/notification.xml"; + let mut present_a = + sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/a.cer", 1); + let mut withdrawn_b = + sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/b.roa", 2); + withdrawn_b.present = false; + let present_c = + sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/c.crl", 3); + let other_source = sample_rrdp_source_member_record( + "https://other.example.test/notification.xml", + "rsync://other.example.test/repo/x.cer", + 4, + ); + present_a.last_confirmed_serial = 10; + + store + .put_rrdp_source_member_record(&present_a) + .expect("put present a"); + store + .put_rrdp_source_member_record(&withdrawn_b) + .expect("put withdrawn b"); + store + .put_rrdp_source_member_record(&present_c) + .expect("put present c"); + store + .put_rrdp_source_member_record(&other_source) + .expect("put other source"); + + let members = store + .list_current_rrdp_source_members(notify_uri) + .expect("list current members"); + assert_eq!( + members + .iter() + .map(|record| record.rsync_uri.as_str()) + .collect::>(), + vec![ + "rsync://example.test/repo/a.cer", + "rsync://example.test/repo/c.crl", + ] + ); + + assert!( + store + .is_current_rrdp_source_member(notify_uri, &present_a.rsync_uri) + .expect("current a") + ); + assert!( + !store + .is_current_rrdp_source_member(notify_uri, &withdrawn_b.rsync_uri) + .expect("withdrawn b") + ); + assert!( + !store + .is_current_rrdp_source_member(notify_uri, &other_source.rsync_uri) + .expect("other source") + ); +} + +#[test] +fn load_current_object_bytes_by_uri_uses_repository_view_and_raw_by_hash() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let present_bytes = b"present-object".to_vec(); + let present_hash = sha256_hex(&present_bytes); + let mut present_raw = RawByHashEntry::from_bytes(present_hash.clone(), present_bytes.clone()); + present_raw + .origin_uris + .push("rsync://example.test/repo/present.roa".to_string()); + present_raw.object_type = Some("roa".to_string()); + store + .put_raw_by_hash_entry(&present_raw) + .expect("put present raw"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/present.roa".to_string(), + current_hash: Some(present_hash), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put present view"); + + let replaced_bytes = b"replaced-object".to_vec(); + let replaced_hash = sha256_hex(&replaced_bytes); + let mut replaced_raw = + RawByHashEntry::from_bytes(replaced_hash.clone(), replaced_bytes.clone()); + replaced_raw + .origin_uris + .push("rsync://example.test/repo/replaced.cer".to_string()); + replaced_raw.object_type = Some("cer".to_string()); + store + .put_raw_by_hash_entry(&replaced_raw) + .expect("put replaced raw"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/replaced.cer".to_string(), + current_hash: Some(replaced_hash), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("cer".to_string()), + state: RepositoryViewState::Replaced, + }) + .expect("put replaced view"); + + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/withdrawn.crl".to_string(), + current_hash: Some(sha256_hex(b"withdrawn")), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("crl".to_string()), + state: RepositoryViewState::Withdrawn, + }) + .expect("put withdrawn view"); + + assert_eq!( + store + .load_current_object_bytes_by_uri("rsync://example.test/repo/present.roa") + .expect("load present"), + Some(present_bytes) + ); + assert_eq!( + store + .load_current_object_bytes_by_uri("rsync://example.test/repo/replaced.cer") + .expect("load replaced"), + Some(replaced_bytes) + ); + assert_eq!( + store + .load_current_object_bytes_by_uri("rsync://example.test/repo/withdrawn.crl") + .expect("load withdrawn"), + None + ); + assert_eq!( + store + .load_current_object_bytes_by_uri("rsync://example.test/repo/missing.roa") + .expect("load missing"), + None + ); +} + +#[test] +fn load_current_object_bytes_by_uri_errors_when_raw_by_hash_is_missing() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let rsync_uri = "rsync://example.test/repo/missing.cer"; + + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(hex::encode([0x11; 32])), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("cer".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put view"); + let err = store + .load_current_object_bytes_by_uri(rsync_uri) + .expect_err("missing raw_by_hash should error"); + assert!(matches!(err, StorageError::InvalidData { .. })); +} + +#[test] +fn load_current_object_with_hash_by_uri_returns_hash_and_bytes() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let rsync_uri = "rsync://example.test/repo/present.roa"; + let bytes = b"present-object".to_vec(); + let hash = sha256_hex(&bytes); + + let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone()); + raw.origin_uris.push(rsync_uri.to_string()); + raw.object_type = Some("roa".to_string()); + store.put_raw_by_hash_entry(&raw).expect("put raw"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(hash.clone()), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put view"); + + let got = store + .load_current_object_with_hash_by_uri(rsync_uri) + .expect("load current object") + .expect("current object exists"); + assert_eq!(got.current_hash_hex, hash); + assert_eq!(got.current_hash, compute_sha256_32(&bytes)); + assert_eq!(got.bytes, bytes); +} + +#[test] +fn load_current_object_with_hash_by_uri_uses_internal_blob_cf_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let rsync_uri = "rsync://example.test/repo/blob-only.roa"; + let bytes = b"blob-only-current-object".to_vec(); + let hash = sha256_hex(&bytes); + + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("put blob bytes"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(hash.clone()), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put view"); + + let got = store + .load_current_object_with_hash_by_uri(rsync_uri) + .expect("load current object") + .expect("current object exists"); + assert_eq!(got.current_hash_hex, hash); + assert_eq!(got.current_hash, compute_sha256_32(&bytes)); + assert_eq!(got.bytes, bytes); + assert!( + store + .get_raw_by_hash_entry(&got.current_hash_hex) + .expect("get raw entry") + .is_none() + ); +} + +#[test] +fn load_current_object_bytes_by_uri_uses_internal_blob_cf_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let rsync_uri = "rsync://example.test/repo/blob-only-bytes.roa"; + let bytes = b"blob-only-current-object-bytes".to_vec(); + let hash = sha256_hex(&bytes); + + store + .put_blob_bytes_batch(&[(hash, bytes.clone())]) + .expect("put blob bytes"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(sha256_hex(&bytes)), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put view"); + + assert_eq!( + store + .load_current_object_bytes_by_uri(rsync_uri) + .expect("load current object bytes"), + Some(bytes) + ); +} + +#[test] +fn pack_file_can_lazy_load_bytes_from_external_raw_store() { + let td = tempfile::tempdir().expect("tempdir"); + let raw_store = std::sync::Arc::new( + ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store"), + ); + let bytes = b"lazy-pack-file".to_vec(); + let sha256_hex = sha256_hex(&bytes); + raw_store + .put_raw_entry(&RawByHashEntry::from_bytes( + sha256_hex.clone(), + bytes.clone(), + )) + .expect("put raw entry"); + + let file = PackFile::from_lazy_external_raw_store( + "rsync://example.test/repo/a.roa", + sha256_hex, + compute_sha256_32(&bytes), + raw_store, + ); + + assert_eq!(file.bytes().expect("lazy bytes"), bytes.as_slice()); + assert_eq!(file.bytes_cloned().expect("cloned bytes"), bytes); +} + +#[test] +fn pack_file_can_lazy_load_bytes_from_external_repo_bytes_store() { + let td = tempfile::tempdir().expect("tempdir"); + let repo_bytes_store = std::sync::Arc::new( + ExternalRepoBytesDb::open(td.path().join("repo-bytes.db")).expect("open repo bytes"), + ); + let bytes = b"repo-object-pack-file".to_vec(); + let sha256_hex = sha256_hex(&bytes); + repo_bytes_store + .put_blob_bytes_batch(&[(sha256_hex.clone(), bytes.clone())]) + .expect("put repo bytes"); + + let file = PackFile::from_lazy_repo_bytes( + "rsync://example.test/repo/a.roa", + sha256_hex, + compute_sha256_32(&bytes), + repo_bytes_store, + ); + + assert_eq!(file.bytes().expect("lazy repo bytes"), bytes.as_slice()); + assert_eq!(file.bytes_cloned().expect("cloned repo bytes"), bytes); + assert_eq!(file.compute_sha256().expect("compute sha256"), file.sha256); +} diff --git a/src/sync/repo.rs b/src/sync/repo.rs index 657d389..9aba46d 100644 --- a/src/sync/repo.rs +++ b/src/sync/repo.rs @@ -724,1345 +724,5 @@ pub(crate) fn run_rsync_transport( } #[cfg(test)] -mod tests { - use super::*; - use crate::analysis::timing::{TimingHandle, TimingMeta}; - use crate::fetch::rsync::LocalDirRsyncFetcher; - use crate::replay::archive::{ReplayArchiveIndex, sha256_hex}; - use crate::replay::delta_archive::ReplayDeltaArchiveIndex; - use crate::replay::delta_fetch_http::PayloadDeltaReplayHttpFetcher; - use crate::replay::delta_fetch_rsync::PayloadDeltaReplayRsyncFetcher; - use crate::replay::fetch_http::PayloadReplayHttpFetcher; - use crate::replay::fetch_rsync::PayloadReplayRsyncFetcher; - use crate::storage::RepositoryViewState; - use crate::sync::rrdp::Fetcher as HttpFetcher; - use crate::sync::rrdp::RrdpState; - use crate::sync::store_projection::{build_repository_view_present_entry, compute_sha256_hex}; - use base64::Engine; - use sha2::Digest; - use std::collections::HashMap; - use std::sync::Arc; - use std::sync::atomic::{AtomicUsize, Ordering}; - - struct DummyHttpFetcher; - - impl HttpFetcher for DummyHttpFetcher { - fn fetch(&self, _url: &str) -> Result, String> { - panic!("http fetcher must not be used in rsync-only mode") - } - } - - struct PanicRsyncFetcher; - impl RsyncFetcher for PanicRsyncFetcher { - fn fetch_objects( - &self, - _rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - panic!("rsync must not be used in this test") - } - } - - struct MapFetcher { - map: HashMap>, - } - - impl HttpFetcher for MapFetcher { - fn fetch(&self, uri: &str) -> Result, String> { - self.map - .get(uri) - .cloned() - .ok_or_else(|| format!("not found: {uri}")) - } - } - - fn assert_current_object(store: &RocksStore, uri: &str, expected: &[u8]) { - assert_eq!( - store - .load_current_object_bytes_by_uri(uri) - .expect("load current object"), - Some(expected.to_vec()) - ); - } - - #[test] - fn rsync_sync_uses_fetcher_dedup_scope_for_repository_view_projection() { - struct ScopeFetcher; - impl RsyncFetcher for ScopeFetcher { - fn fetch_objects( - &self, - _rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - Ok(vec![( - "rsync://example.net/repo/child/a.mft".to_string(), - b"manifest".to_vec(), - )]) - } - - fn dedup_key(&self, _rsync_base_uri: &str) -> String { - "rsync://example.net/repo/".to_string() - } - } - - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let seeded = build_repository_view_present_entry( - "rsync://example.net/repo/", - "rsync://example.net/repo/sibling/old.roa", - &compute_sha256_hex(b"old"), - ); - store - .put_projection_batch(&[seeded], &[], &[]) - .expect("seed repository view"); - - let fetcher = ScopeFetcher; - let written = rsync_sync_into_current_store( - &store, - "rsync://example.net/repo/child/", - None, - &fetcher, - None, - None, - ) - .expect("sync ok"); - assert_eq!(written, 1); - - let entries = store - .list_repository_view_entries_with_prefix("rsync://example.net/repo/") - .expect("list repository view"); - let sibling = entries - .iter() - .find(|entry| entry.rsync_uri == "rsync://example.net/repo/sibling/old.roa") - .expect("sibling entry exists"); - assert_eq!(sibling.state, RepositoryViewState::Withdrawn); - let child = entries - .iter() - .find(|entry| entry.rsync_uri == "rsync://example.net/repo/child/a.mft") - .expect("child entry exists"); - assert_eq!(child.state, RepositoryViewState::Present); - } - - fn notification_xml( - session_id: &str, - serial: u64, - snapshot_uri: &str, - snapshot_hash: &str, - ) -> Vec { - format!( - r#""# - ) - .into_bytes() - } - - fn snapshot_xml(session_id: &str, serial: u64, published: &[(&str, &[u8])]) -> Vec { - let mut out = format!( - r#""# - ); - for (uri, bytes) in published { - let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); - out.push_str(&format!(r#"{b64}"#)); - } - out.push_str(""); - out.into_bytes() - } - - fn build_replay_archive_fixture() -> ( - tempfile::TempDir, - std::path::PathBuf, - std::path::PathBuf, - String, - String, - String, - String, - ) { - let temp = tempfile::tempdir().expect("tempdir"); - let archive_root = temp.path().join("payload-archive"); - let capture = "repo-replay"; - let capture_root = archive_root.join("v1").join("captures").join(capture); - std::fs::create_dir_all(&capture_root).expect("mkdir capture root"); - std::fs::write( - capture_root.join("capture.json"), - format!( - r#"{{"version":1,"captureId":"{capture}","createdAt":"2026-03-13T00:00:00Z","notes":""}}"# - ), - ) - .expect("write capture json"); - - let notify_uri = "https://rrdp.example.test/notification.xml".to_string(); - let snapshot_uri = "https://rrdp.example.test/snapshot.xml".to_string(); - let session = "00000000-0000-0000-0000-000000000001".to_string(); - let serial = 7u64; - let published_uri = "rsync://example.test/repo/a.mft".to_string(); - let published_bytes = b"mft"; - let snapshot = snapshot_xml(&session, serial, &[(&published_uri, published_bytes)]); - let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); - let notification = notification_xml(&session, serial, &snapshot_uri, &snapshot_hash); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let session_dir = capture_root - .join("rrdp/repos") - .join(&repo_hash) - .join(&session); - std::fs::create_dir_all(&session_dir).expect("mkdir session dir"); - std::fs::write( - session_dir.parent().unwrap().join("meta.json"), - format!( - r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}}"# - ), - ) - .expect("write repo meta"); - std::fs::write(session_dir.join("notification-7.xml"), notification) - .expect("write notification"); - std::fs::write( - session_dir.join(format!("snapshot-7-{snapshot_hash}.xml")), - &snapshot, - ) - .expect("write snapshot"); - - let rsync_base_uri = "rsync://rsync.example.test/repo/".to_string(); - let rsync_locked_notify = "https://rrdp-fallback.example.test/notification.xml".to_string(); - let mod_hash = sha256_hex(rsync_base_uri.as_bytes()); - let module_bucket_dir = capture_root.join("rsync/modules").join(&mod_hash); - let module_root = module_bucket_dir - .join("tree") - .join("rsync.example.test") - .join("repo"); - std::fs::create_dir_all(module_root.join("sub")).expect("mkdir module tree"); - std::fs::write( - module_bucket_dir.join("meta.json"), - format!( - r#"{{"version":1,"module":"{rsync_base_uri}","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}}"# - ), - ) - .expect("write rsync meta"); - std::fs::write(module_root.join("sub").join("fallback.cer"), b"cer") - .expect("write rsync object"); - - let locks_path = temp.path().join("locks.json"); - std::fs::write( - &locks_path, - format!( - r#"{{ - "version":1, - "capture":"{capture}", - "rrdp":{{ - "{notify_uri}":{{"transport":"rrdp","session":"{session}","serial":{serial}}}, - "{rsync_locked_notify}":{{"transport":"rsync","session":null,"serial":null}} - }}, - "rsync":{{ - "{rsync_base_uri}":{{"transport":"rsync"}} - }} -}}"# - ), - ) - .expect("write locks"); - - ( - temp, - archive_root, - locks_path, - notify_uri, - rsync_locked_notify, - rsync_base_uri, - published_uri, - ) - } - - fn build_delta_replay_fixture() -> ( - tempfile::TempDir, - std::path::PathBuf, - std::path::PathBuf, - std::path::PathBuf, - std::path::PathBuf, - String, - String, - String, - ) { - let temp = tempfile::tempdir().expect("tempdir"); - - let base_archive = temp.path().join("payload-archive"); - let base_capture_root = base_archive.join("v1/captures/base-cap"); - std::fs::create_dir_all(&base_capture_root).expect("mkdir base capture"); - std::fs::write( - base_capture_root.join("capture.json"), - r#"{"version":1,"captureId":"base-cap","createdAt":"2026-03-16T00:00:00Z","notes":""}"#, - ) - .expect("write base capture meta"); - - let notify_uri = "https://rrdp.example.test/notification.xml".to_string(); - let snapshot_uri = "https://rrdp.example.test/snapshot.xml".to_string(); - let session = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(); - let base_serial = 10u64; - let delta1_uri = "https://rrdp.example.test/d1.xml".to_string(); - let delta2_uri = "https://rrdp.example.test/d2.xml".to_string(); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let base_session_dir = base_capture_root - .join("rrdp/repos") - .join(&repo_hash) - .join(&session); - std::fs::create_dir_all(&base_session_dir).expect("mkdir base session dir"); - std::fs::write( - base_session_dir.parent().unwrap().join("meta.json"), - format!(r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), - ) - .expect("write base rrdp meta"); - let base_snapshot = snapshot_xml( - &session, - base_serial, - &[("rsync://example.test/repo/a.mft", b"base")], - ); - let base_snapshot_hash = hex::encode(sha2::Sha256::digest(&base_snapshot)); - let base_notification = - notification_xml(&session, base_serial, &snapshot_uri, &base_snapshot_hash); - std::fs::write( - base_session_dir.join("notification-10.xml"), - base_notification, - ) - .expect("write base notif"); - std::fs::write( - base_session_dir.join(format!("snapshot-10-{base_snapshot_hash}.xml")), - base_snapshot, - ) - .expect("write base snapshot"); - - let module_uri = "rsync://rsync.example.test/repo/".to_string(); - let module_hash = sha256_hex(module_uri.as_bytes()); - let base_module_bucket = base_capture_root.join("rsync/modules").join(&module_hash); - let base_module_tree = base_module_bucket.join("tree/rsync.example.test/repo"); - std::fs::create_dir_all(base_module_tree.join("sub")).expect("mkdir base rsync tree"); - std::fs::write( - base_module_bucket.join("meta.json"), - format!(r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), - ) - .expect("write base module meta"); - std::fs::write(base_module_tree.join("a.mft"), b"base").expect("write base a.mft"); - std::fs::write(base_module_tree.join("sub").join("x.cer"), b"base-cer") - .expect("write base x.cer"); - - let base_locks = temp.path().join("base-locks.json"); - let fallback_notify = "https://rrdp-fallback.example.test/notification.xml".to_string(); - let base_locks_body = format!( - r#"{{"version":1,"capture":"base-cap","rrdp":{{"{notify_uri}":{{"transport":"rrdp","session":"{session}","serial":10}},"{fallback_notify}":{{"transport":"rsync","session":null,"serial":null}}}},"rsync":{{"{module_uri}":{{"transport":"rsync"}}}}}}"# - ); - std::fs::write(&base_locks, &base_locks_body).expect("write base locks"); - let base_locks_sha = sha256_hex(base_locks_body.as_bytes()); - - let delta_archive = temp.path().join("payload-delta-archive"); - let delta_capture_root = delta_archive.join("v1/captures/delta-cap"); - std::fs::create_dir_all(&delta_capture_root).expect("mkdir delta capture"); - std::fs::write( - delta_capture_root.join("capture.json"), - r#"{"version":1,"captureId":"delta-cap","createdAt":"2026-03-16T00:00:00Z","notes":""}"#, - ) - .expect("write delta capture meta"); - std::fs::write( - delta_capture_root.join("base.json"), - format!(r#"{{"version":1,"baseCapture":"base-cap","baseLocksSha256":"{base_locks_sha}","createdAt":"2026-03-16T00:00:00Z"}}"#), - ) - .expect("write delta base meta"); - - let delta_session_dir = delta_capture_root - .join("rrdp/repos") - .join(&repo_hash) - .join(&session); - let delta_deltas_dir = delta_session_dir.join("deltas"); - std::fs::create_dir_all(&delta_deltas_dir).expect("mkdir delta deltas"); - std::fs::write( - delta_session_dir.parent().unwrap().join("meta.json"), - format!(r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), - ) - .expect("write delta meta"); - std::fs::write( - delta_session_dir.parent().unwrap().join("transition.json"), - format!(r#"{{"kind":"delta","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}}"#), - ) - .expect("write delta transition"); - let delta1 = format!( - r#"{}"#, - base64::engine::general_purpose::STANDARD.encode(b"delta-a") - ); - let delta2 = format!( - r#"{}"#, - base64::engine::general_purpose::STANDARD.encode(b"delta-b") - ); - let delta1_hash = hex::encode(sha2::Sha256::digest(delta1.as_bytes())); - let delta2_hash = hex::encode(sha2::Sha256::digest(delta2.as_bytes())); - let target_notification = format!( - r#" - - - - -"# - ); - std::fs::write( - delta_session_dir.join("notification-target-12.xml"), - target_notification, - ) - .expect("write target notification"); - std::fs::write(delta_deltas_dir.join("delta-11-aaaa.xml"), delta1).expect("write delta11"); - std::fs::write(delta_deltas_dir.join("delta-12-bbbb.xml"), delta2).expect("write delta12"); - - let delta_module_bucket = delta_capture_root.join("rsync/modules").join(&module_hash); - let delta_module_tree = delta_module_bucket.join("tree/rsync.example.test/repo"); - std::fs::create_dir_all(delta_module_tree.join("sub")).expect("mkdir delta rsync tree"); - std::fs::write( - delta_module_bucket.join("meta.json"), - format!(r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), - ) - .expect("write delta rsync meta"); - std::fs::write( - delta_module_bucket.join("files.json"), - format!(r#"{{"version":1,"module":"{module_uri}","fileCount":1,"files":["{module_uri}sub/x.cer"]}}"#), - ) - .expect("write delta files"); - std::fs::write(delta_module_tree.join("sub").join("x.cer"), b"overlay-cer") - .expect("write overlay file"); - - let fallback_hash = sha256_hex(fallback_notify.as_bytes()); - let fallback_repo_dir = delta_capture_root.join("rrdp/repos").join(&fallback_hash); - std::fs::create_dir_all(&fallback_repo_dir).expect("mkdir fallback repo dir"); - std::fs::write( - fallback_repo_dir.join("meta.json"), - format!(r#"{{"version":1,"rpkiNotify":"{fallback_notify}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), - ) - .expect("write fallback meta"); - std::fs::write( - fallback_repo_dir.join("transition.json"), - r#"{"kind":"fallback-rsync","base":{"transport":"rsync","session":null,"serial":null},"target":{"transport":"rsync","session":null,"serial":null},"delta_count":0,"deltas":[]}"#, - ) - .expect("write fallback transition"); - - let delta_locks = temp.path().join("locks-delta.json"); - std::fs::write( - &delta_locks, - format!(r#"{{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"{base_locks_sha}","rrdp":{{"{notify_uri}":{{"kind":"delta","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}},"{fallback_notify}":{{"kind":"fallback-rsync","base":{{"transport":"rsync","session":null,"serial":null}},"target":{{"transport":"rsync","session":null,"serial":null}},"delta_count":0,"deltas":[]}}}},"rsync":{{"{module_uri}":{{"file_count":1,"overlay_only":false}}}}}}"#), - ) - .expect("write delta locks"); - - ( - temp, - base_archive, - base_locks, - delta_archive, - delta_locks, - notify_uri, - fallback_notify, - module_uri, - ) - } - - fn timing_to_json(temp_dir: &std::path::Path, timing: &TimingHandle) -> serde_json::Value { - let timing_path = temp_dir.join("timing_retry.json"); - timing.write_json(&timing_path, 50).expect("write json"); - serde_json::from_slice(&std::fs::read(&timing_path).expect("read json")) - .expect("parse json") - } - - #[test] - fn rsync_sync_writes_current_store_and_records_counts() { - let temp = tempfile::tempdir().expect("tempdir"); - - let repo_dir = temp.path().join("repo"); - std::fs::create_dir_all(repo_dir.join("sub")).expect("mkdir"); - std::fs::write(repo_dir.join("a.mft"), b"mft").expect("write"); - std::fs::write(repo_dir.join("sub").join("b.roa"), b"roa").expect("write"); - std::fs::write(repo_dir.join("sub").join("c.cer"), b"cer").expect("write"); - - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - - let timing = TimingHandle::new(TimingMeta { - recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - tal_url: None, - db_path: Some(store_dir.to_string_lossy().into_owned()), - }); - - let policy = Policy { - sync_preference: SyncPreference::RsyncOnly, - ..Policy::default() - }; - let http = DummyHttpFetcher; - let rsync = LocalDirRsyncFetcher::new(&repo_dir); - - let download_log = DownloadLogHandle::new(); - let out = sync_publication_point( - &store, - &policy, - None, - "rsync://example.test/repo/", - &http, - &rsync, - Some(&timing), - Some(&download_log), - ) - .expect("sync ok"); - - assert_eq!(out.source, RepoSyncSource::Rsync); - assert_eq!(out.objects_written, 3); - - let events = download_log.snapshot_events(); - assert_eq!(events.len(), 1); - assert_eq!(events[0].kind, AuditDownloadKind::Rsync); - assert!(events[0].success); - assert_eq!(events[0].bytes, Some(9)); - let objects = events[0].objects.as_ref().expect("objects stat"); - assert_eq!(objects.objects_count, 3); - assert_eq!(objects.objects_bytes_total, 9); - - assert_current_object(&store, "rsync://example.test/repo/a.mft", b"mft"); - assert_current_object(&store, "rsync://example.test/repo/sub/b.roa", b"roa"); - assert_current_object(&store, "rsync://example.test/repo/sub/c.cer", b"cer"); - - let view = store - .get_repository_view_entry("rsync://example.test/repo/a.mft") - .expect("get repository view") - .expect("repository view entry present"); - assert_eq!( - view.current_hash.as_deref(), - Some(hex::encode(sha2::Sha256::digest(b"mft")).as_str()) - ); - assert_eq!( - view.repository_source.as_deref(), - Some("rsync://example.test/repo/") - ); - - let current_bytes = store - .load_current_object_bytes_by_uri("rsync://example.test/repo/sub/b.roa") - .expect("load current bytes") - .expect("current object bytes exist"); - assert_eq!(current_bytes, b"roa".to_vec()); - assert!( - store - .get_raw_by_hash_entry(hex::encode(sha2::Sha256::digest(b"roa")).as_str()) - .expect("get raw_by_hash") - .is_none() - ); - - let timing_path = temp.path().join("timing.json"); - timing.write_json(&timing_path, 5).expect("write json"); - let v: serde_json::Value = - serde_json::from_slice(&std::fs::read(&timing_path).expect("read json")) - .expect("parse json"); - let counts = v.get("counts").expect("counts"); - assert_eq!( - counts - .get("rsync_objects_fetched_total") - .and_then(|v| v.as_u64()), - Some(3) - ); - assert_eq!( - counts - .get("rsync_objects_bytes_total") - .and_then(|v| v.as_u64()), - Some(3 * 3) - ); - } - - #[test] - fn rsync_second_sync_marks_missing_repository_view_entries_withdrawn() { - let temp = tempfile::tempdir().expect("tempdir"); - - let repo_dir = temp.path().join("repo"); - std::fs::create_dir_all(repo_dir.join("sub")).expect("mkdir"); - std::fs::write(repo_dir.join("a.mft"), b"mft-v1").expect("write a"); - std::fs::write(repo_dir.join("sub").join("b.roa"), b"roa-v1").expect("write b"); - - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - let policy = Policy { - sync_preference: SyncPreference::RsyncOnly, - ..Policy::default() - }; - let http = DummyHttpFetcher; - let rsync = LocalDirRsyncFetcher::new(&repo_dir); - - sync_publication_point( - &store, - &policy, - None, - "rsync://example.test/repo/", - &http, - &rsync, - None, - None, - ) - .expect("first sync ok"); - - std::fs::remove_file(repo_dir.join("sub").join("b.roa")).expect("remove b"); - std::fs::write(repo_dir.join("c.crl"), b"crl-v2").expect("write c"); - - sync_publication_point( - &store, - &policy, - None, - "rsync://example.test/repo/", - &http, - &rsync, - None, - None, - ) - .expect("second sync ok"); - - let withdrawn = store - .get_repository_view_entry("rsync://example.test/repo/sub/b.roa") - .expect("get withdrawn repo view") - .expect("withdrawn entry exists"); - assert_eq!( - withdrawn.state, - crate::storage::RepositoryViewState::Withdrawn - ); - assert_eq!( - withdrawn.repository_source.as_deref(), - Some("rsync://example.test/repo/") - ); - - let added = store - .get_repository_view_entry("rsync://example.test/repo/c.crl") - .expect("get added repo view") - .expect("added entry exists"); - assert_eq!(added.state, crate::storage::RepositoryViewState::Present); - } - - #[test] - fn rrdp_fetch_error_falls_back_to_rsync_without_retry() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - - let timing = TimingHandle::new(TimingMeta { - recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - tal_url: None, - db_path: Some(store_dir.to_string_lossy().into_owned()), - }); - - let notification_uri = "https://example.test/notification.xml"; - let published_uri = "rsync://example.test/repo/a.mft"; - let published_bytes = b"x"; - struct AlwaysFailHttp { - notification_calls: AtomicUsize, - } - - impl HttpFetcher for AlwaysFailHttp { - fn fetch(&self, _uri: &str) -> Result, String> { - self.notification_calls.fetch_add(1, Ordering::SeqCst); - Err("http request failed: simulated transient".to_string()) - } - } - - struct SingleObjectRsync { - uri: String, - bytes: Vec, - } - impl RsyncFetcher for SingleObjectRsync { - fn fetch_objects( - &self, - _rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - Ok(vec![(self.uri.clone(), self.bytes.clone())]) - } - } - - let http = AlwaysFailHttp { - notification_calls: AtomicUsize::new(0), - }; - - let policy = Policy { - sync_preference: SyncPreference::RrdpThenRsync, - ..Policy::default() - }; - - let download_log = DownloadLogHandle::new(); - let out = sync_publication_point( - &store, - &policy, - Some(notification_uri), - "rsync://example.test/repo/", - &http, - &SingleObjectRsync { - uri: published_uri.to_string(), - bytes: published_bytes.to_vec(), - }, - Some(&timing), - Some(&download_log), - ) - .expect("sync ok"); - - assert_eq!(out.source, RepoSyncSource::Rsync); - assert_current_object(&store, published_uri, published_bytes); - assert_eq!(http.notification_calls.load(Ordering::SeqCst), 1); - - let events = download_log.snapshot_events(); - assert_eq!(events.len(), 2, "expected 1x notification + 1x rsync"); - assert_eq!( - events - .iter() - .filter(|e| e.kind == AuditDownloadKind::RrdpNotification) - .count(), - 1 - ); - assert_eq!( - events - .iter() - .filter(|e| e.kind == AuditDownloadKind::RrdpNotification && !e.success) - .count(), - 1 - ); - assert_eq!( - events - .iter() - .filter(|e| e.kind == AuditDownloadKind::Rsync) - .count(), - 1 - ); - - let v = timing_to_json(temp.path(), &timing); - let counts = v.get("counts").expect("counts"); - assert_eq!( - counts - .get("rrdp_retry_attempt_total") - .and_then(|v| v.as_u64()), - Some(1) - ); - assert_eq!( - counts - .get("repo_sync_rrdp_failed_total") - .and_then(|v| v.as_u64()), - Some(1) - ); - assert_eq!( - counts - .get("repo_sync_rsync_fallback_ok_total") - .and_then(|v| v.as_u64()), - Some(1) - ); - } - - #[test] - fn rrdp_protocol_error_does_not_retry_and_falls_back_to_rsync() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - - let timing = TimingHandle::new(TimingMeta { - recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - tal_url: None, - db_path: Some(store_dir.to_string_lossy().into_owned()), - }); - - let notification_uri = "https://example.test/notification.xml"; - let snapshot_uri = "https://example.test/snapshot.xml"; - let published_uri = "rsync://example.test/repo/a.mft"; - let published_bytes = b"x"; - - let snapshot = snapshot_xml( - "9df4b597-af9e-4dca-bdda-719cce2c4e28", - 1, - &[(published_uri, published_bytes)], - ); - // Intentionally wrong hash to trigger protocol error (SnapshotHashMismatch). - let wrong_hash = "00".repeat(32); - let notif = notification_xml( - "9df4b597-af9e-4dca-bdda-719cce2c4e28", - 1, - snapshot_uri, - &wrong_hash, - ); - - let mut map = HashMap::new(); - map.insert(notification_uri.to_string(), notif); - map.insert(snapshot_uri.to_string(), snapshot); - let http = MapFetcher { map }; - - struct EmptyRsyncFetcher; - impl RsyncFetcher for EmptyRsyncFetcher { - fn fetch_objects( - &self, - _rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - Ok(Vec::new()) - } - } - - let policy = Policy { - sync_preference: SyncPreference::RrdpThenRsync, - ..Policy::default() - }; - - let download_log = DownloadLogHandle::new(); - let out = sync_publication_point( - &store, - &policy, - Some(notification_uri), - "rsync://example.test/repo/", - &http, - &EmptyRsyncFetcher, - Some(&timing), - Some(&download_log), - ) - .expect("sync ok"); - - assert_eq!(out.source, RepoSyncSource::Rsync); - assert!( - out.warnings - .iter() - .any(|w| w.message.contains("RRDP failed; falling back to rsync")), - "expected RRDP fallback warning" - ); - - let events = download_log.snapshot_events(); - assert_eq!( - events.len(), - 3, - "expected notification + snapshot + rsync fallback" - ); - assert_eq!(events[0].kind, AuditDownloadKind::RrdpNotification); - assert!(events[0].success); - assert_eq!(events[1].kind, AuditDownloadKind::RrdpSnapshot); - assert!(!events[1].success); - assert_eq!(events[2].kind, AuditDownloadKind::Rsync); - assert!(events[2].success); - - let v = timing_to_json(temp.path(), &timing); - let counts = v.get("counts").expect("counts"); - assert_eq!( - counts - .get("rrdp_retry_attempt_total") - .and_then(|v| v.as_u64()), - Some(1) - ); - assert_eq!( - counts - .get("rrdp_failed_protocol_total") - .and_then(|v| v.as_u64()), - Some(1) - ); - assert_eq!( - counts - .get("repo_sync_rrdp_failed_total") - .and_then(|v| v.as_u64()), - Some(1) - ); - assert_eq!( - counts - .get("repo_sync_rsync_fallback_ok_total") - .and_then(|v| v.as_u64()), - Some(1) - ); - } - - #[test] - fn rrdp_delta_fetches_are_logged_even_if_snapshot_fallback_is_used() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - - let timing = TimingHandle::new(TimingMeta { - recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - tal_url: None, - db_path: Some(store_dir.to_string_lossy().into_owned()), - }); - - let notification_uri = "https://example.test/notification.xml"; - let snapshot_uri = "https://example.test/snapshot.xml"; - let delta_2_uri = "https://example.test/delta_2.xml"; - let delta_3_uri = "https://example.test/delta_3.xml"; - let published_uri = "rsync://example.test/repo/a.mft"; - let published_bytes = b"x"; - - let sid = "9df4b597-af9e-4dca-bdda-719cce2c4e28"; - - // Seed old RRDP state so sync_from_notification tries deltas (RFC 8182 §3.4.1). - let state = RrdpState { - session_id: sid.to_string(), - serial: 1, - }; - persist_rrdp_local_state( - &store, - notification_uri, - &state, - RrdpSourceSyncState::DeltaReady, - Some(snapshot_uri), - None, - ) - .expect("seed state"); - - let delta_2 = format!( - r#""# - ) - .into_bytes(); - let delta_3 = format!( - r#""# - ) - .into_bytes(); - let delta_2_hash = hex::encode(sha2::Sha256::digest(&delta_2)); - let delta_3_hash = hex::encode(sha2::Sha256::digest(&delta_3)); - - let snapshot = snapshot_xml(sid, 3, &[(published_uri, published_bytes)]); - let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); - let notif = format!( - r#""# - ) - .into_bytes(); - - let mut map = HashMap::new(); - map.insert(notification_uri.to_string(), notif); - map.insert(snapshot_uri.to_string(), snapshot); - map.insert(delta_2_uri.to_string(), delta_2); - map.insert(delta_3_uri.to_string(), delta_3); - let http = MapFetcher { map }; - - let policy = Policy { - sync_preference: SyncPreference::RrdpThenRsync, - ..Policy::default() - }; - - let download_log = DownloadLogHandle::new(); - let out = sync_publication_point( - &store, - &policy, - Some(notification_uri), - "rsync://example.test/repo/", - &http, - &PanicRsyncFetcher, - Some(&timing), - Some(&download_log), - ) - .expect("sync ok"); - - assert_eq!(out.source, RepoSyncSource::Rrdp); - assert_eq!(out.objects_written, 1); - assert_current_object(&store, published_uri, published_bytes); - - let events = download_log.snapshot_events(); - assert_eq!(events.len(), 4); - assert_eq!( - events - .iter() - .filter(|e| e.kind == AuditDownloadKind::RrdpNotification) - .count(), - 1 - ); - assert_eq!( - events - .iter() - .filter(|e| e.kind == AuditDownloadKind::RrdpDelta) - .count(), - 2 - ); - assert_eq!( - events - .iter() - .filter(|e| e.kind == AuditDownloadKind::RrdpSnapshot) - .count(), - 1 - ); - assert!(events.iter().all(|e| e.success)); - } - - #[test] - fn replay_sync_uses_rrdp_when_locked_to_rrdp() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - - let ( - _archive_temp, - archive_root, - locks_path, - notify_uri, - _rsync_locked_notify, - _rsync_base_uri, - published_uri, - ) = build_replay_archive_fixture(); - let replay_index = - ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); - let http = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) - .expect("build replay http fetcher"); - let rsync = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) - .expect("build replay rsync fetcher"); - - let out = sync_publication_point_replay( - &store, - &replay_index, - Some(¬ify_uri), - "rsync://example.test/repo/", - &http, - &rsync, - None, - None, - ) - .expect("replay sync ok"); - - assert_eq!(out.source, RepoSyncSource::Rrdp); - assert_eq!(out.objects_written, 1); - assert_current_object(&store, &published_uri, b"mft"); - } - - #[test] - fn replay_sync_uses_rsync_when_notification_is_locked_to_rsync() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - - let ( - _archive_temp, - archive_root, - locks_path, - _notify_uri, - rsync_locked_notify, - rsync_base_uri, - _published_uri, - ) = build_replay_archive_fixture(); - let replay_index = - ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); - let http = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) - .expect("build replay http fetcher"); - let rsync = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) - .expect("build replay rsync fetcher"); - - let out = sync_publication_point_replay( - &store, - &replay_index, - Some(&rsync_locked_notify), - &rsync_base_uri, - &http, - &rsync, - None, - None, - ) - .expect("replay rsync sync ok"); - - assert_eq!(out.source, RepoSyncSource::Rsync); - assert_eq!(out.objects_written, 1); - assert_eq!(out.warnings.len(), 0); - assert_current_object( - &store, - "rsync://rsync.example.test/repo/sub/fallback.cer", - b"cer", - ); - } - - #[test] - fn replay_sync_errors_when_lock_is_missing() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - - let ( - _archive_temp, - archive_root, - locks_path, - _notify_uri, - _rsync_locked_notify, - _rsync_base_uri, - _published_uri, - ) = build_replay_archive_fixture(); - let replay_index = - ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); - let http = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) - .expect("build replay http fetcher"); - let rsync = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) - .expect("build replay rsync fetcher"); - - let err = sync_publication_point_replay( - &store, - &replay_index, - Some("https://missing.example/notification.xml"), - "rsync://missing.example/repo/", - &http, - &rsync, - None, - None, - ) - .unwrap_err(); - assert!(matches!(err, RepoSyncError::Replay(_)), "{err}"); - } - - #[test] - fn delta_replay_sync_applies_rrdp_deltas_when_base_state_matches() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - let ( - _fixture, - base_archive, - base_locks, - delta_archive, - delta_locks, - notify_uri, - _fallback_notify, - module_uri, - ) = build_delta_replay_fixture(); - let base_index = Arc::new( - ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index"), - ); - let delta_index = Arc::new( - ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), - ); - let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) - .expect("build delta http fetcher"); - let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); - - let state = RrdpState { - session_id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(), - serial: 10, - }; - persist_rrdp_local_state( - &store, - ¬ify_uri, - &state, - RrdpSourceSyncState::DeltaReady, - None, - None, - ) - .expect("seed base state"); - - let out = sync_publication_point_replay_delta( - &store, - &delta_index, - Some(¬ify_uri), - &module_uri, - &http, - &rsync, - None, - None, - ) - .expect("delta sync ok"); - - assert_eq!(out.source, RepoSyncSource::Rrdp); - assert_eq!(out.objects_written, 2); - assert_current_object(&store, "rsync://example.test/repo/a.mft", b"delta-a"); - assert_current_object(&store, "rsync://example.test/repo/sub/b.roa", b"delta-b"); - let new_state = load_rrdp_local_state(&store, ¬ify_uri) - .expect("load current state") - .expect("rrdp state present"); - assert_eq!(new_state.serial, 12); - } - - #[test] - fn delta_replay_sync_rejects_base_state_mismatch() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - let ( - _fixture, - base_archive, - base_locks, - delta_archive, - delta_locks, - notify_uri, - _fallback_notify, - module_uri, - ) = build_delta_replay_fixture(); - let base_index = Arc::new( - ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index"), - ); - let delta_index = Arc::new( - ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), - ); - let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) - .expect("build delta http fetcher"); - let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); - - let err = sync_publication_point_replay_delta( - &store, - &delta_index, - Some(¬ify_uri), - &module_uri, - &http, - &rsync, - None, - None, - ) - .unwrap_err(); - assert!(matches!(err, RepoSyncError::Replay(_)), "{err}"); - } - - #[test] - fn delta_replay_sync_noops_unchanged_rrdp_repo() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - let ( - _fixture, - base_archive, - base_locks, - delta_archive, - delta_locks, - notify_uri, - _fallback_notify, - module_uri, - ) = build_delta_replay_fixture(); - let state = RrdpState { - session_id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(), - serial: 10, - }; - persist_rrdp_local_state( - &store, - ¬ify_uri, - &state, - RrdpSourceSyncState::DeltaReady, - None, - None, - ) - .expect("seed base state"); - - let base_locks_body = std::fs::read_to_string(&base_locks).expect("read base locks"); - let base_locks_sha = sha256_hex(base_locks_body.as_bytes()); - std::fs::write( - &delta_locks, - format!(r#"{{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"{base_locks_sha}","rrdp":{{"{notify_uri}":{{"kind":"unchanged","base":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10}},"target":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10}},"delta_count":0,"deltas":[]}},"https://rrdp-fallback.example.test/notification.xml":{{"kind":"fallback-rsync","base":{{"transport":"rsync","session":null,"serial":null}},"target":{{"transport":"rsync","session":null,"serial":null}},"delta_count":0,"deltas":[]}}}},"rsync":{{"rsync://rsync.example.test/repo/":{{"file_count":1,"overlay_only":true}}}}}}"#), - ) - .expect("rewrite delta locks"); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let repo_dir = delta_archive - .join("v1/captures/delta-cap/rrdp/repos") - .join(&repo_hash); - std::fs::write( - repo_dir.join("transition.json"), - r#"{"kind":"unchanged","base":{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10},"target":{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10},"delta_count":0,"deltas":[]}"#, - ).expect("rewrite transition"); - let delta_index = - ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"); - let http = PayloadDeltaReplayHttpFetcher::from_index(Arc::new(delta_index.clone())) - .expect("build delta http fetcher"); - let base_index = Arc::new( - ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index"), - ); - let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, Arc::new(delta_index.clone())); - - let out = sync_publication_point_replay_delta( - &store, - &delta_index, - Some(¬ify_uri), - &module_uri, - &http, - &rsync, - None, - None, - ) - .expect("unchanged delta sync ok"); - assert_eq!(out.source, RepoSyncSource::Rrdp); - assert_eq!(out.objects_written, 0); - } - - #[test] - fn delta_replay_sync_uses_rsync_overlay_for_fallback_rsync_kind() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - let ( - _fixture, - base_archive, - base_locks, - delta_archive, - delta_locks, - _notify_uri, - fallback_notify, - module_uri, - ) = build_delta_replay_fixture(); - let base_index = Arc::new( - ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index"), - ); - let delta_index = Arc::new( - ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), - ); - let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) - .expect("build delta http fetcher"); - let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); - - let out = sync_publication_point_replay_delta( - &store, - &delta_index, - Some(&fallback_notify), - &module_uri, - &http, - &rsync, - None, - None, - ) - .expect("fallback-rsync delta sync ok"); - assert_eq!(out.source, RepoSyncSource::Rsync); - assert_eq!(out.objects_written, 2); - assert_current_object(&store, "rsync://rsync.example.test/repo/a.mft", b"base"); - assert_current_object( - &store, - "rsync://rsync.example.test/repo/sub/x.cer", - b"overlay-cer", - ); - } - - #[test] - fn delta_replay_sync_rejects_session_reset_and_gap() { - for kind in ["session-reset", "gap"] { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - let ( - _fixture, - base_archive, - base_locks, - delta_archive, - delta_locks, - notify_uri, - _fallback_notify, - module_uri, - ) = build_delta_replay_fixture(); - let base_index = Arc::new( - ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index"), - ); - let state = RrdpState { - session_id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(), - serial: 10, - }; - persist_rrdp_local_state( - &store, - ¬ify_uri, - &state, - RrdpSourceSyncState::DeltaReady, - None, - None, - ) - .expect("seed base state"); - - let locks_body = std::fs::read_to_string(&delta_locks).expect("read delta locks"); - let rewritten = - locks_body.replace("\"kind\":\"delta\"", &format!("\"kind\":\"{}\"", kind)); - std::fs::write(&delta_locks, rewritten).expect("rewrite locks kind"); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let repo_dir = delta_archive - .join("v1/captures/delta-cap/rrdp/repos") - .join(&repo_hash); - std::fs::write( - repo_dir.join("transition.json"), - format!( - r#"{{"kind":"{kind}","base":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10}},"target":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":12}},"delta_count":2,"deltas":[11,12]}}"#, - ), - ) - .expect("rewrite transition kind"); - let delta_index = Arc::new( - ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks) - .expect("load delta index"), - ); - let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) - .expect("build delta http fetcher"); - let rsync = - PayloadDeltaReplayRsyncFetcher::new(base_index.clone(), delta_index.clone()); - let err = sync_publication_point_replay_delta( - &store, - &delta_index, - Some(¬ify_uri), - &module_uri, - &http, - &rsync, - None, - None, - ) - .unwrap_err(); - assert!(matches!(err, RepoSyncError::Replay(_)), "{err}"); - } - } -} +#[path = "repo/tests.rs"] +mod tests; diff --git a/src/sync/repo/tests.rs b/src/sync/repo/tests.rs new file mode 100644 index 0000000..d35cc6f --- /dev/null +++ b/src/sync/repo/tests.rs @@ -0,0 +1,1332 @@ +use super::*; +use crate::analysis::timing::{TimingHandle, TimingMeta}; +use crate::fetch::rsync::LocalDirRsyncFetcher; +use crate::replay::archive::{ReplayArchiveIndex, sha256_hex}; +use crate::replay::delta_archive::ReplayDeltaArchiveIndex; +use crate::replay::delta_fetch_http::PayloadDeltaReplayHttpFetcher; +use crate::replay::delta_fetch_rsync::PayloadDeltaReplayRsyncFetcher; +use crate::replay::fetch_http::PayloadReplayHttpFetcher; +use crate::replay::fetch_rsync::PayloadReplayRsyncFetcher; +use crate::storage::RepositoryViewState; +use crate::sync::rrdp::Fetcher as HttpFetcher; +use crate::sync::rrdp::RrdpState; +use crate::sync::store_projection::{build_repository_view_present_entry, compute_sha256_hex}; +use base64::Engine; +use sha2::Digest; +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +struct DummyHttpFetcher; + +impl HttpFetcher for DummyHttpFetcher { + fn fetch(&self, _url: &str) -> Result, String> { + panic!("http fetcher must not be used in rsync-only mode") + } +} + +struct PanicRsyncFetcher; +impl RsyncFetcher for PanicRsyncFetcher { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + panic!("rsync must not be used in this test") + } +} + +struct MapFetcher { + map: HashMap>, +} + +impl HttpFetcher for MapFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + self.map + .get(uri) + .cloned() + .ok_or_else(|| format!("not found: {uri}")) + } +} + +fn assert_current_object(store: &RocksStore, uri: &str, expected: &[u8]) { + assert_eq!( + store + .load_current_object_bytes_by_uri(uri) + .expect("load current object"), + Some(expected.to_vec()) + ); +} + +#[test] +fn rsync_sync_uses_fetcher_dedup_scope_for_repository_view_projection() { + struct ScopeFetcher; + impl RsyncFetcher for ScopeFetcher { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + Ok(vec![( + "rsync://example.net/repo/child/a.mft".to_string(), + b"manifest".to_vec(), + )]) + } + + fn dedup_key(&self, _rsync_base_uri: &str) -> String { + "rsync://example.net/repo/".to_string() + } + } + + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let seeded = build_repository_view_present_entry( + "rsync://example.net/repo/", + "rsync://example.net/repo/sibling/old.roa", + &compute_sha256_hex(b"old"), + ); + store + .put_projection_batch(&[seeded], &[], &[]) + .expect("seed repository view"); + + let fetcher = ScopeFetcher; + let written = rsync_sync_into_current_store( + &store, + "rsync://example.net/repo/child/", + None, + &fetcher, + None, + None, + ) + .expect("sync ok"); + assert_eq!(written, 1); + + let entries = store + .list_repository_view_entries_with_prefix("rsync://example.net/repo/") + .expect("list repository view"); + let sibling = entries + .iter() + .find(|entry| entry.rsync_uri == "rsync://example.net/repo/sibling/old.roa") + .expect("sibling entry exists"); + assert_eq!(sibling.state, RepositoryViewState::Withdrawn); + let child = entries + .iter() + .find(|entry| entry.rsync_uri == "rsync://example.net/repo/child/a.mft") + .expect("child entry exists"); + assert_eq!(child.state, RepositoryViewState::Present); +} + +fn notification_xml( + session_id: &str, + serial: u64, + snapshot_uri: &str, + snapshot_hash: &str, +) -> Vec { + format!( + r#""# + ) + .into_bytes() +} + +fn snapshot_xml(session_id: &str, serial: u64, published: &[(&str, &[u8])]) -> Vec { + let mut out = format!( + r#""# + ); + for (uri, bytes) in published { + let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); + out.push_str(&format!(r#"{b64}"#)); + } + out.push_str(""); + out.into_bytes() +} + +fn build_replay_archive_fixture() -> ( + tempfile::TempDir, + std::path::PathBuf, + std::path::PathBuf, + String, + String, + String, + String, +) { + let temp = tempfile::tempdir().expect("tempdir"); + let archive_root = temp.path().join("payload-archive"); + let capture = "repo-replay"; + let capture_root = archive_root.join("v1").join("captures").join(capture); + std::fs::create_dir_all(&capture_root).expect("mkdir capture root"); + std::fs::write( + capture_root.join("capture.json"), + format!( + r#"{{"version":1,"captureId":"{capture}","createdAt":"2026-03-13T00:00:00Z","notes":""}}"# + ), + ) + .expect("write capture json"); + + let notify_uri = "https://rrdp.example.test/notification.xml".to_string(); + let snapshot_uri = "https://rrdp.example.test/snapshot.xml".to_string(); + let session = "00000000-0000-0000-0000-000000000001".to_string(); + let serial = 7u64; + let published_uri = "rsync://example.test/repo/a.mft".to_string(); + let published_bytes = b"mft"; + let snapshot = snapshot_xml(&session, serial, &[(&published_uri, published_bytes)]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notification = notification_xml(&session, serial, &snapshot_uri, &snapshot_hash); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let session_dir = capture_root + .join("rrdp/repos") + .join(&repo_hash) + .join(&session); + std::fs::create_dir_all(&session_dir).expect("mkdir session dir"); + std::fs::write( + session_dir.parent().unwrap().join("meta.json"), + format!( + r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}}"# + ), + ) + .expect("write repo meta"); + std::fs::write(session_dir.join("notification-7.xml"), notification) + .expect("write notification"); + std::fs::write( + session_dir.join(format!("snapshot-7-{snapshot_hash}.xml")), + &snapshot, + ) + .expect("write snapshot"); + + let rsync_base_uri = "rsync://rsync.example.test/repo/".to_string(); + let rsync_locked_notify = "https://rrdp-fallback.example.test/notification.xml".to_string(); + let mod_hash = sha256_hex(rsync_base_uri.as_bytes()); + let module_bucket_dir = capture_root.join("rsync/modules").join(&mod_hash); + let module_root = module_bucket_dir + .join("tree") + .join("rsync.example.test") + .join("repo"); + std::fs::create_dir_all(module_root.join("sub")).expect("mkdir module tree"); + std::fs::write( + module_bucket_dir.join("meta.json"), + format!( + r#"{{"version":1,"module":"{rsync_base_uri}","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}}"# + ), + ) + .expect("write rsync meta"); + std::fs::write(module_root.join("sub").join("fallback.cer"), b"cer") + .expect("write rsync object"); + + let locks_path = temp.path().join("locks.json"); + std::fs::write( + &locks_path, + format!( + r#"{{ + "version":1, + "capture":"{capture}", + "rrdp":{{ + "{notify_uri}":{{"transport":"rrdp","session":"{session}","serial":{serial}}}, + "{rsync_locked_notify}":{{"transport":"rsync","session":null,"serial":null}} + }}, + "rsync":{{ + "{rsync_base_uri}":{{"transport":"rsync"}} + }} +}}"# + ), + ) + .expect("write locks"); + + ( + temp, + archive_root, + locks_path, + notify_uri, + rsync_locked_notify, + rsync_base_uri, + published_uri, + ) +} + +fn build_delta_replay_fixture() -> ( + tempfile::TempDir, + std::path::PathBuf, + std::path::PathBuf, + std::path::PathBuf, + std::path::PathBuf, + String, + String, + String, +) { + let temp = tempfile::tempdir().expect("tempdir"); + + let base_archive = temp.path().join("payload-archive"); + let base_capture_root = base_archive.join("v1/captures/base-cap"); + std::fs::create_dir_all(&base_capture_root).expect("mkdir base capture"); + std::fs::write( + base_capture_root.join("capture.json"), + r#"{"version":1,"captureId":"base-cap","createdAt":"2026-03-16T00:00:00Z","notes":""}"#, + ) + .expect("write base capture meta"); + + let notify_uri = "https://rrdp.example.test/notification.xml".to_string(); + let snapshot_uri = "https://rrdp.example.test/snapshot.xml".to_string(); + let session = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(); + let base_serial = 10u64; + let delta1_uri = "https://rrdp.example.test/d1.xml".to_string(); + let delta2_uri = "https://rrdp.example.test/d2.xml".to_string(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let base_session_dir = base_capture_root + .join("rrdp/repos") + .join(&repo_hash) + .join(&session); + std::fs::create_dir_all(&base_session_dir).expect("mkdir base session dir"); + std::fs::write( + base_session_dir.parent().unwrap().join("meta.json"), + format!(r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), + ) + .expect("write base rrdp meta"); + let base_snapshot = snapshot_xml( + &session, + base_serial, + &[("rsync://example.test/repo/a.mft", b"base")], + ); + let base_snapshot_hash = hex::encode(sha2::Sha256::digest(&base_snapshot)); + let base_notification = + notification_xml(&session, base_serial, &snapshot_uri, &base_snapshot_hash); + std::fs::write( + base_session_dir.join("notification-10.xml"), + base_notification, + ) + .expect("write base notif"); + std::fs::write( + base_session_dir.join(format!("snapshot-10-{base_snapshot_hash}.xml")), + base_snapshot, + ) + .expect("write base snapshot"); + + let module_uri = "rsync://rsync.example.test/repo/".to_string(); + let module_hash = sha256_hex(module_uri.as_bytes()); + let base_module_bucket = base_capture_root.join("rsync/modules").join(&module_hash); + let base_module_tree = base_module_bucket.join("tree/rsync.example.test/repo"); + std::fs::create_dir_all(base_module_tree.join("sub")).expect("mkdir base rsync tree"); + std::fs::write( + base_module_bucket.join("meta.json"), + format!(r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), + ) + .expect("write base module meta"); + std::fs::write(base_module_tree.join("a.mft"), b"base").expect("write base a.mft"); + std::fs::write(base_module_tree.join("sub").join("x.cer"), b"base-cer") + .expect("write base x.cer"); + + let base_locks = temp.path().join("base-locks.json"); + let fallback_notify = "https://rrdp-fallback.example.test/notification.xml".to_string(); + let base_locks_body = format!( + r#"{{"version":1,"capture":"base-cap","rrdp":{{"{notify_uri}":{{"transport":"rrdp","session":"{session}","serial":10}},"{fallback_notify}":{{"transport":"rsync","session":null,"serial":null}}}},"rsync":{{"{module_uri}":{{"transport":"rsync"}}}}}}"# + ); + std::fs::write(&base_locks, &base_locks_body).expect("write base locks"); + let base_locks_sha = sha256_hex(base_locks_body.as_bytes()); + + let delta_archive = temp.path().join("payload-delta-archive"); + let delta_capture_root = delta_archive.join("v1/captures/delta-cap"); + std::fs::create_dir_all(&delta_capture_root).expect("mkdir delta capture"); + std::fs::write( + delta_capture_root.join("capture.json"), + r#"{"version":1,"captureId":"delta-cap","createdAt":"2026-03-16T00:00:00Z","notes":""}"#, + ) + .expect("write delta capture meta"); + std::fs::write( + delta_capture_root.join("base.json"), + format!(r#"{{"version":1,"baseCapture":"base-cap","baseLocksSha256":"{base_locks_sha}","createdAt":"2026-03-16T00:00:00Z"}}"#), + ) + .expect("write delta base meta"); + + let delta_session_dir = delta_capture_root + .join("rrdp/repos") + .join(&repo_hash) + .join(&session); + let delta_deltas_dir = delta_session_dir.join("deltas"); + std::fs::create_dir_all(&delta_deltas_dir).expect("mkdir delta deltas"); + std::fs::write( + delta_session_dir.parent().unwrap().join("meta.json"), + format!(r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), + ) + .expect("write delta meta"); + std::fs::write( + delta_session_dir.parent().unwrap().join("transition.json"), + format!(r#"{{"kind":"delta","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}}"#), + ) + .expect("write delta transition"); + let delta1 = format!( + r#"{}"#, + base64::engine::general_purpose::STANDARD.encode(b"delta-a") + ); + let delta2 = format!( + r#"{}"#, + base64::engine::general_purpose::STANDARD.encode(b"delta-b") + ); + let delta1_hash = hex::encode(sha2::Sha256::digest(delta1.as_bytes())); + let delta2_hash = hex::encode(sha2::Sha256::digest(delta2.as_bytes())); + let target_notification = format!( + r#" + + + + +"# + ); + std::fs::write( + delta_session_dir.join("notification-target-12.xml"), + target_notification, + ) + .expect("write target notification"); + std::fs::write(delta_deltas_dir.join("delta-11-aaaa.xml"), delta1).expect("write delta11"); + std::fs::write(delta_deltas_dir.join("delta-12-bbbb.xml"), delta2).expect("write delta12"); + + let delta_module_bucket = delta_capture_root.join("rsync/modules").join(&module_hash); + let delta_module_tree = delta_module_bucket.join("tree/rsync.example.test/repo"); + std::fs::create_dir_all(delta_module_tree.join("sub")).expect("mkdir delta rsync tree"); + std::fs::write( + delta_module_bucket.join("meta.json"), + format!(r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), + ) + .expect("write delta rsync meta"); + std::fs::write( + delta_module_bucket.join("files.json"), + format!(r#"{{"version":1,"module":"{module_uri}","fileCount":1,"files":["{module_uri}sub/x.cer"]}}"#), + ) + .expect("write delta files"); + std::fs::write(delta_module_tree.join("sub").join("x.cer"), b"overlay-cer") + .expect("write overlay file"); + + let fallback_hash = sha256_hex(fallback_notify.as_bytes()); + let fallback_repo_dir = delta_capture_root.join("rrdp/repos").join(&fallback_hash); + std::fs::create_dir_all(&fallback_repo_dir).expect("mkdir fallback repo dir"); + std::fs::write( + fallback_repo_dir.join("meta.json"), + format!(r#"{{"version":1,"rpkiNotify":"{fallback_notify}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), + ) + .expect("write fallback meta"); + std::fs::write( + fallback_repo_dir.join("transition.json"), + r#"{"kind":"fallback-rsync","base":{"transport":"rsync","session":null,"serial":null},"target":{"transport":"rsync","session":null,"serial":null},"delta_count":0,"deltas":[]}"#, + ) + .expect("write fallback transition"); + + let delta_locks = temp.path().join("locks-delta.json"); + std::fs::write( + &delta_locks, + format!(r#"{{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"{base_locks_sha}","rrdp":{{"{notify_uri}":{{"kind":"delta","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}},"{fallback_notify}":{{"kind":"fallback-rsync","base":{{"transport":"rsync","session":null,"serial":null}},"target":{{"transport":"rsync","session":null,"serial":null}},"delta_count":0,"deltas":[]}}}},"rsync":{{"{module_uri}":{{"file_count":1,"overlay_only":false}}}}}}"#), + ) + .expect("write delta locks"); + + ( + temp, + base_archive, + base_locks, + delta_archive, + delta_locks, + notify_uri, + fallback_notify, + module_uri, + ) +} + +fn timing_to_json(temp_dir: &std::path::Path, timing: &TimingHandle) -> serde_json::Value { + let timing_path = temp_dir.join("timing_retry.json"); + timing.write_json(&timing_path, 50).expect("write json"); + serde_json::from_slice(&std::fs::read(&timing_path).expect("read json")).expect("parse json") +} + +#[test] +fn rsync_sync_writes_current_store_and_records_counts() { + let temp = tempfile::tempdir().expect("tempdir"); + + let repo_dir = temp.path().join("repo"); + std::fs::create_dir_all(repo_dir.join("sub")).expect("mkdir"); + std::fs::write(repo_dir.join("a.mft"), b"mft").expect("write"); + std::fs::write(repo_dir.join("sub").join("b.roa"), b"roa").expect("write"); + std::fs::write(repo_dir.join("sub").join("c.cer"), b"cer").expect("write"); + + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + tal_url: None, + db_path: Some(store_dir.to_string_lossy().into_owned()), + }); + + let policy = Policy { + sync_preference: SyncPreference::RsyncOnly, + ..Policy::default() + }; + let http = DummyHttpFetcher; + let rsync = LocalDirRsyncFetcher::new(&repo_dir); + + let download_log = DownloadLogHandle::new(); + let out = sync_publication_point( + &store, + &policy, + None, + "rsync://example.test/repo/", + &http, + &rsync, + Some(&timing), + Some(&download_log), + ) + .expect("sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rsync); + assert_eq!(out.objects_written, 3); + + let events = download_log.snapshot_events(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].kind, AuditDownloadKind::Rsync); + assert!(events[0].success); + assert_eq!(events[0].bytes, Some(9)); + let objects = events[0].objects.as_ref().expect("objects stat"); + assert_eq!(objects.objects_count, 3); + assert_eq!(objects.objects_bytes_total, 9); + + assert_current_object(&store, "rsync://example.test/repo/a.mft", b"mft"); + assert_current_object(&store, "rsync://example.test/repo/sub/b.roa", b"roa"); + assert_current_object(&store, "rsync://example.test/repo/sub/c.cer", b"cer"); + + let view = store + .get_repository_view_entry("rsync://example.test/repo/a.mft") + .expect("get repository view") + .expect("repository view entry present"); + assert_eq!( + view.current_hash.as_deref(), + Some(hex::encode(sha2::Sha256::digest(b"mft")).as_str()) + ); + assert_eq!( + view.repository_source.as_deref(), + Some("rsync://example.test/repo/") + ); + + let current_bytes = store + .load_current_object_bytes_by_uri("rsync://example.test/repo/sub/b.roa") + .expect("load current bytes") + .expect("current object bytes exist"); + assert_eq!(current_bytes, b"roa".to_vec()); + assert!( + store + .get_raw_by_hash_entry(hex::encode(sha2::Sha256::digest(b"roa")).as_str()) + .expect("get raw_by_hash") + .is_none() + ); + + let timing_path = temp.path().join("timing.json"); + timing.write_json(&timing_path, 5).expect("write json"); + let v: serde_json::Value = + serde_json::from_slice(&std::fs::read(&timing_path).expect("read json")) + .expect("parse json"); + let counts = v.get("counts").expect("counts"); + assert_eq!( + counts + .get("rsync_objects_fetched_total") + .and_then(|v| v.as_u64()), + Some(3) + ); + assert_eq!( + counts + .get("rsync_objects_bytes_total") + .and_then(|v| v.as_u64()), + Some(3 * 3) + ); +} + +#[test] +fn rsync_second_sync_marks_missing_repository_view_entries_withdrawn() { + let temp = tempfile::tempdir().expect("tempdir"); + + let repo_dir = temp.path().join("repo"); + std::fs::create_dir_all(repo_dir.join("sub")).expect("mkdir"); + std::fs::write(repo_dir.join("a.mft"), b"mft-v1").expect("write a"); + std::fs::write(repo_dir.join("sub").join("b.roa"), b"roa-v1").expect("write b"); + + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let policy = Policy { + sync_preference: SyncPreference::RsyncOnly, + ..Policy::default() + }; + let http = DummyHttpFetcher; + let rsync = LocalDirRsyncFetcher::new(&repo_dir); + + sync_publication_point( + &store, + &policy, + None, + "rsync://example.test/repo/", + &http, + &rsync, + None, + None, + ) + .expect("first sync ok"); + + std::fs::remove_file(repo_dir.join("sub").join("b.roa")).expect("remove b"); + std::fs::write(repo_dir.join("c.crl"), b"crl-v2").expect("write c"); + + sync_publication_point( + &store, + &policy, + None, + "rsync://example.test/repo/", + &http, + &rsync, + None, + None, + ) + .expect("second sync ok"); + + let withdrawn = store + .get_repository_view_entry("rsync://example.test/repo/sub/b.roa") + .expect("get withdrawn repo view") + .expect("withdrawn entry exists"); + assert_eq!( + withdrawn.state, + crate::storage::RepositoryViewState::Withdrawn + ); + assert_eq!( + withdrawn.repository_source.as_deref(), + Some("rsync://example.test/repo/") + ); + + let added = store + .get_repository_view_entry("rsync://example.test/repo/c.crl") + .expect("get added repo view") + .expect("added entry exists"); + assert_eq!(added.state, crate::storage::RepositoryViewState::Present); +} + +#[test] +fn rrdp_fetch_error_falls_back_to_rsync_without_retry() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + tal_url: None, + db_path: Some(store_dir.to_string_lossy().into_owned()), + }); + + let notification_uri = "https://example.test/notification.xml"; + let published_uri = "rsync://example.test/repo/a.mft"; + let published_bytes = b"x"; + struct AlwaysFailHttp { + notification_calls: AtomicUsize, + } + + impl HttpFetcher for AlwaysFailHttp { + fn fetch(&self, _uri: &str) -> Result, String> { + self.notification_calls.fetch_add(1, Ordering::SeqCst); + Err("http request failed: simulated transient".to_string()) + } + } + + struct SingleObjectRsync { + uri: String, + bytes: Vec, + } + impl RsyncFetcher for SingleObjectRsync { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + Ok(vec![(self.uri.clone(), self.bytes.clone())]) + } + } + + let http = AlwaysFailHttp { + notification_calls: AtomicUsize::new(0), + }; + + let policy = Policy { + sync_preference: SyncPreference::RrdpThenRsync, + ..Policy::default() + }; + + let download_log = DownloadLogHandle::new(); + let out = sync_publication_point( + &store, + &policy, + Some(notification_uri), + "rsync://example.test/repo/", + &http, + &SingleObjectRsync { + uri: published_uri.to_string(), + bytes: published_bytes.to_vec(), + }, + Some(&timing), + Some(&download_log), + ) + .expect("sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rsync); + assert_current_object(&store, published_uri, published_bytes); + assert_eq!(http.notification_calls.load(Ordering::SeqCst), 1); + + let events = download_log.snapshot_events(); + assert_eq!(events.len(), 2, "expected 1x notification + 1x rsync"); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::RrdpNotification) + .count(), + 1 + ); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::RrdpNotification && !e.success) + .count(), + 1 + ); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::Rsync) + .count(), + 1 + ); + + let v = timing_to_json(temp.path(), &timing); + let counts = v.get("counts").expect("counts"); + assert_eq!( + counts + .get("rrdp_retry_attempt_total") + .and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + counts + .get("repo_sync_rrdp_failed_total") + .and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + counts + .get("repo_sync_rsync_fallback_ok_total") + .and_then(|v| v.as_u64()), + Some(1) + ); +} + +#[test] +fn rrdp_protocol_error_does_not_retry_and_falls_back_to_rsync() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + tal_url: None, + db_path: Some(store_dir.to_string_lossy().into_owned()), + }); + + let notification_uri = "https://example.test/notification.xml"; + let snapshot_uri = "https://example.test/snapshot.xml"; + let published_uri = "rsync://example.test/repo/a.mft"; + let published_bytes = b"x"; + + let snapshot = snapshot_xml( + "9df4b597-af9e-4dca-bdda-719cce2c4e28", + 1, + &[(published_uri, published_bytes)], + ); + // Intentionally wrong hash to trigger protocol error (SnapshotHashMismatch). + let wrong_hash = "00".repeat(32); + let notif = notification_xml( + "9df4b597-af9e-4dca-bdda-719cce2c4e28", + 1, + snapshot_uri, + &wrong_hash, + ); + + let mut map = HashMap::new(); + map.insert(notification_uri.to_string(), notif); + map.insert(snapshot_uri.to_string(), snapshot); + let http = MapFetcher { map }; + + struct EmptyRsyncFetcher; + impl RsyncFetcher for EmptyRsyncFetcher { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + Ok(Vec::new()) + } + } + + let policy = Policy { + sync_preference: SyncPreference::RrdpThenRsync, + ..Policy::default() + }; + + let download_log = DownloadLogHandle::new(); + let out = sync_publication_point( + &store, + &policy, + Some(notification_uri), + "rsync://example.test/repo/", + &http, + &EmptyRsyncFetcher, + Some(&timing), + Some(&download_log), + ) + .expect("sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rsync); + assert!( + out.warnings + .iter() + .any(|w| w.message.contains("RRDP failed; falling back to rsync")), + "expected RRDP fallback warning" + ); + + let events = download_log.snapshot_events(); + assert_eq!( + events.len(), + 3, + "expected notification + snapshot + rsync fallback" + ); + assert_eq!(events[0].kind, AuditDownloadKind::RrdpNotification); + assert!(events[0].success); + assert_eq!(events[1].kind, AuditDownloadKind::RrdpSnapshot); + assert!(!events[1].success); + assert_eq!(events[2].kind, AuditDownloadKind::Rsync); + assert!(events[2].success); + + let v = timing_to_json(temp.path(), &timing); + let counts = v.get("counts").expect("counts"); + assert_eq!( + counts + .get("rrdp_retry_attempt_total") + .and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + counts + .get("rrdp_failed_protocol_total") + .and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + counts + .get("repo_sync_rrdp_failed_total") + .and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + counts + .get("repo_sync_rsync_fallback_ok_total") + .and_then(|v| v.as_u64()), + Some(1) + ); +} + +#[test] +fn rrdp_delta_fetches_are_logged_even_if_snapshot_fallback_is_used() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + tal_url: None, + db_path: Some(store_dir.to_string_lossy().into_owned()), + }); + + let notification_uri = "https://example.test/notification.xml"; + let snapshot_uri = "https://example.test/snapshot.xml"; + let delta_2_uri = "https://example.test/delta_2.xml"; + let delta_3_uri = "https://example.test/delta_3.xml"; + let published_uri = "rsync://example.test/repo/a.mft"; + let published_bytes = b"x"; + + let sid = "9df4b597-af9e-4dca-bdda-719cce2c4e28"; + + // Seed old RRDP state so sync_from_notification tries deltas (RFC 8182 §3.4.1). + let state = RrdpState { + session_id: sid.to_string(), + serial: 1, + }; + persist_rrdp_local_state( + &store, + notification_uri, + &state, + RrdpSourceSyncState::DeltaReady, + Some(snapshot_uri), + None, + ) + .expect("seed state"); + + let delta_2 = format!( + r#""# + ) + .into_bytes(); + let delta_3 = format!( + r#""# + ) + .into_bytes(); + let delta_2_hash = hex::encode(sha2::Sha256::digest(&delta_2)); + let delta_3_hash = hex::encode(sha2::Sha256::digest(&delta_3)); + + let snapshot = snapshot_xml(sid, 3, &[(published_uri, published_bytes)]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = format!( + r#""# + ) + .into_bytes(); + + let mut map = HashMap::new(); + map.insert(notification_uri.to_string(), notif); + map.insert(snapshot_uri.to_string(), snapshot); + map.insert(delta_2_uri.to_string(), delta_2); + map.insert(delta_3_uri.to_string(), delta_3); + let http = MapFetcher { map }; + + let policy = Policy { + sync_preference: SyncPreference::RrdpThenRsync, + ..Policy::default() + }; + + let download_log = DownloadLogHandle::new(); + let out = sync_publication_point( + &store, + &policy, + Some(notification_uri), + "rsync://example.test/repo/", + &http, + &PanicRsyncFetcher, + Some(&timing), + Some(&download_log), + ) + .expect("sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rrdp); + assert_eq!(out.objects_written, 1); + assert_current_object(&store, published_uri, published_bytes); + + let events = download_log.snapshot_events(); + assert_eq!(events.len(), 4); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::RrdpNotification) + .count(), + 1 + ); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::RrdpDelta) + .count(), + 2 + ); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::RrdpSnapshot) + .count(), + 1 + ); + assert!(events.iter().all(|e| e.success)); +} + +#[test] +fn replay_sync_uses_rrdp_when_locked_to_rrdp() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let ( + _archive_temp, + archive_root, + locks_path, + notify_uri, + _rsync_locked_notify, + _rsync_base_uri, + published_uri, + ) = build_replay_archive_fixture(); + let replay_index = + ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); + let http = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay http fetcher"); + let rsync = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay rsync fetcher"); + + let out = sync_publication_point_replay( + &store, + &replay_index, + Some(¬ify_uri), + "rsync://example.test/repo/", + &http, + &rsync, + None, + None, + ) + .expect("replay sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rrdp); + assert_eq!(out.objects_written, 1); + assert_current_object(&store, &published_uri, b"mft"); +} + +#[test] +fn replay_sync_uses_rsync_when_notification_is_locked_to_rsync() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let ( + _archive_temp, + archive_root, + locks_path, + _notify_uri, + rsync_locked_notify, + rsync_base_uri, + _published_uri, + ) = build_replay_archive_fixture(); + let replay_index = + ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); + let http = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay http fetcher"); + let rsync = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay rsync fetcher"); + + let out = sync_publication_point_replay( + &store, + &replay_index, + Some(&rsync_locked_notify), + &rsync_base_uri, + &http, + &rsync, + None, + None, + ) + .expect("replay rsync sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rsync); + assert_eq!(out.objects_written, 1); + assert_eq!(out.warnings.len(), 0); + assert_current_object( + &store, + "rsync://rsync.example.test/repo/sub/fallback.cer", + b"cer", + ); +} + +#[test] +fn replay_sync_errors_when_lock_is_missing() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let ( + _archive_temp, + archive_root, + locks_path, + _notify_uri, + _rsync_locked_notify, + _rsync_base_uri, + _published_uri, + ) = build_replay_archive_fixture(); + let replay_index = + ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); + let http = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay http fetcher"); + let rsync = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay rsync fetcher"); + + let err = sync_publication_point_replay( + &store, + &replay_index, + Some("https://missing.example/notification.xml"), + "rsync://missing.example/repo/", + &http, + &rsync, + None, + None, + ) + .unwrap_err(); + assert!(matches!(err, RepoSyncError::Replay(_)), "{err}"); +} + +#[test] +fn delta_replay_sync_applies_rrdp_deltas_when_base_state_matches() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let ( + _fixture, + base_archive, + base_locks, + delta_archive, + delta_locks, + notify_uri, + _fallback_notify, + module_uri, + ) = build_delta_replay_fixture(); + let base_index = + Arc::new(ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index")); + let delta_index = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) + .expect("build delta http fetcher"); + let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); + + let state = RrdpState { + session_id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(), + serial: 10, + }; + persist_rrdp_local_state( + &store, + ¬ify_uri, + &state, + RrdpSourceSyncState::DeltaReady, + None, + None, + ) + .expect("seed base state"); + + let out = sync_publication_point_replay_delta( + &store, + &delta_index, + Some(¬ify_uri), + &module_uri, + &http, + &rsync, + None, + None, + ) + .expect("delta sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rrdp); + assert_eq!(out.objects_written, 2); + assert_current_object(&store, "rsync://example.test/repo/a.mft", b"delta-a"); + assert_current_object(&store, "rsync://example.test/repo/sub/b.roa", b"delta-b"); + let new_state = load_rrdp_local_state(&store, ¬ify_uri) + .expect("load current state") + .expect("rrdp state present"); + assert_eq!(new_state.serial, 12); +} + +#[test] +fn delta_replay_sync_rejects_base_state_mismatch() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let ( + _fixture, + base_archive, + base_locks, + delta_archive, + delta_locks, + notify_uri, + _fallback_notify, + module_uri, + ) = build_delta_replay_fixture(); + let base_index = + Arc::new(ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index")); + let delta_index = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) + .expect("build delta http fetcher"); + let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); + + let err = sync_publication_point_replay_delta( + &store, + &delta_index, + Some(¬ify_uri), + &module_uri, + &http, + &rsync, + None, + None, + ) + .unwrap_err(); + assert!(matches!(err, RepoSyncError::Replay(_)), "{err}"); +} + +#[test] +fn delta_replay_sync_noops_unchanged_rrdp_repo() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let ( + _fixture, + base_archive, + base_locks, + delta_archive, + delta_locks, + notify_uri, + _fallback_notify, + module_uri, + ) = build_delta_replay_fixture(); + let state = RrdpState { + session_id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(), + serial: 10, + }; + persist_rrdp_local_state( + &store, + ¬ify_uri, + &state, + RrdpSourceSyncState::DeltaReady, + None, + None, + ) + .expect("seed base state"); + + let base_locks_body = std::fs::read_to_string(&base_locks).expect("read base locks"); + let base_locks_sha = sha256_hex(base_locks_body.as_bytes()); + std::fs::write( + &delta_locks, + format!(r#"{{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"{base_locks_sha}","rrdp":{{"{notify_uri}":{{"kind":"unchanged","base":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10}},"target":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10}},"delta_count":0,"deltas":[]}},"https://rrdp-fallback.example.test/notification.xml":{{"kind":"fallback-rsync","base":{{"transport":"rsync","session":null,"serial":null}},"target":{{"transport":"rsync","session":null,"serial":null}},"delta_count":0,"deltas":[]}}}},"rsync":{{"rsync://rsync.example.test/repo/":{{"file_count":1,"overlay_only":true}}}}}}"#), + ) + .expect("rewrite delta locks"); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = delta_archive + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + std::fs::write( + repo_dir.join("transition.json"), + r#"{"kind":"unchanged","base":{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10},"target":{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10},"delta_count":0,"deltas":[]}"#, + ).expect("rewrite transition"); + let delta_index = + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"); + let http = PayloadDeltaReplayHttpFetcher::from_index(Arc::new(delta_index.clone())) + .expect("build delta http fetcher"); + let base_index = + Arc::new(ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index")); + let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, Arc::new(delta_index.clone())); + + let out = sync_publication_point_replay_delta( + &store, + &delta_index, + Some(¬ify_uri), + &module_uri, + &http, + &rsync, + None, + None, + ) + .expect("unchanged delta sync ok"); + assert_eq!(out.source, RepoSyncSource::Rrdp); + assert_eq!(out.objects_written, 0); +} + +#[test] +fn delta_replay_sync_uses_rsync_overlay_for_fallback_rsync_kind() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let ( + _fixture, + base_archive, + base_locks, + delta_archive, + delta_locks, + _notify_uri, + fallback_notify, + module_uri, + ) = build_delta_replay_fixture(); + let base_index = + Arc::new(ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index")); + let delta_index = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) + .expect("build delta http fetcher"); + let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); + + let out = sync_publication_point_replay_delta( + &store, + &delta_index, + Some(&fallback_notify), + &module_uri, + &http, + &rsync, + None, + None, + ) + .expect("fallback-rsync delta sync ok"); + assert_eq!(out.source, RepoSyncSource::Rsync); + assert_eq!(out.objects_written, 2); + assert_current_object(&store, "rsync://rsync.example.test/repo/a.mft", b"base"); + assert_current_object( + &store, + "rsync://rsync.example.test/repo/sub/x.cer", + b"overlay-cer", + ); +} + +#[test] +fn delta_replay_sync_rejects_session_reset_and_gap() { + for kind in ["session-reset", "gap"] { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let ( + _fixture, + base_archive, + base_locks, + delta_archive, + delta_locks, + notify_uri, + _fallback_notify, + module_uri, + ) = build_delta_replay_fixture(); + let base_index = Arc::new( + ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index"), + ); + let state = RrdpState { + session_id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(), + serial: 10, + }; + persist_rrdp_local_state( + &store, + ¬ify_uri, + &state, + RrdpSourceSyncState::DeltaReady, + None, + None, + ) + .expect("seed base state"); + + let locks_body = std::fs::read_to_string(&delta_locks).expect("read delta locks"); + let rewritten = locks_body.replace("\"kind\":\"delta\"", &format!("\"kind\":\"{}\"", kind)); + std::fs::write(&delta_locks, rewritten).expect("rewrite locks kind"); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = delta_archive + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + std::fs::write( + repo_dir.join("transition.json"), + format!( + r#"{{"kind":"{kind}","base":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10}},"target":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":12}},"delta_count":2,"deltas":[11,12]}}"#, + ), + ) + .expect("rewrite transition kind"); + let delta_index = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) + .expect("build delta http fetcher"); + let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index.clone(), delta_index.clone()); + let err = sync_publication_point_replay_delta( + &store, + &delta_index, + Some(¬ify_uri), + &module_uri, + &http, + &rsync, + None, + None, + ) + .unwrap_err(); + assert!(matches!(err, RepoSyncError::Replay(_)), "{err}"); + } +} diff --git a/src/sync/rrdp.rs b/src/sync/rrdp.rs index 4e7dacd..5d2ccda 100644 --- a/src/sync/rrdp.rs +++ b/src/sync/rrdp.rs @@ -5,21 +5,19 @@ use crate::current_repo_index::CurrentRepoIndexHandle; use crate::storage::{ RepositoryViewEntry, RepositoryViewState, RocksStore, RrdpDeltaOp, RrdpSourceSyncState, }; +mod snapshot_apply; + use crate::sync::store_projection::{ build_repository_view_present_entry, build_repository_view_withdrawn_entry, - build_rrdp_source_member_present_record, build_rrdp_source_member_withdrawn_record, - build_rrdp_uri_owner_active_record, build_rrdp_uri_owner_withdrawn_record, compute_sha256_hex, - current_rrdp_owner_is, ensure_rrdp_uri_can_be_owned_by, prepare_repo_bytes_batch, - put_repository_view_present, put_repository_view_withdrawn, put_rrdp_source_member_present, + current_rrdp_owner_is, ensure_rrdp_uri_can_be_owned_by, put_repository_view_present, + put_repository_view_withdrawn, put_rrdp_source_member_present, put_rrdp_source_member_withdrawn, put_rrdp_uri_owner_active, put_rrdp_uri_owner_withdrawn, update_rrdp_source_record_on_success, upsert_repo_blob_bytes, }; use base64::Engine; -use quick_xml::Reader; -use quick_xml::events::Event; use serde::{Deserialize, Serialize}; use sha2::Digest; -use std::io::{BufRead, Seek, SeekFrom, Write}; +use std::io::Write; use uuid::Uuid; const RRDP_XMLNS: &str = "http://www.ripe.net/rpki/rrdp"; @@ -1268,433 +1266,9 @@ fn apply_delta( Ok(ops.len()) } -fn apply_snapshot( - store: &RocksStore, - notification_uri: &str, - current_repo_index: Option<&CurrentRepoIndexHandle>, - snapshot_xml: &[u8], - expected_session_id: Uuid, - expected_serial: u64, -) -> Result { - if snapshot_xml.iter().any(|&b| b > 0x7F) { - return Err(RrdpError::NotAscii.into()); - } - apply_snapshot_from_bufread( - store, - notification_uri, - current_repo_index, - std::io::Cursor::new(snapshot_xml), - expected_session_id, - expected_serial, - ) -} - -fn apply_snapshot_from_bufread( - store: &RocksStore, - notification_uri: &str, - current_repo_index: Option<&CurrentRepoIndexHandle>, - input: R, - expected_session_id: Uuid, - expected_serial: u64, -) -> Result { - let previous_members: Vec = store - .list_current_rrdp_source_members(notification_uri) - .map_err(|e| RrdpSyncError::Storage(e.to_string()))? - .into_iter() - .map(|record| record.rsync_uri) - .collect(); - let session_id = expected_session_id.to_string(); - let mut new_set: std::collections::HashSet = std::collections::HashSet::new(); - let mut batch_published: Vec<(String, Vec)> = - Vec::with_capacity(RRDP_SNAPSHOT_APPLY_BATCH_SIZE); - let mut published_count = 0usize; - - let mut reader = Reader::from_reader(input); - reader.config_mut().trim_text(false); - let mut buf = Vec::new(); - let mut root_seen = false; - let mut in_publish = false; - let mut publish_nested_depth = 0usize; - let mut current_publish_uri: Option = None; - let mut current_publish_text = String::new(); - - loop { - match reader.read_event_into(&mut buf) { - Ok(Event::Start(e)) => { - let local_name = e.local_name(); - let local_name = local_name.as_ref(); - if !root_seen { - root_seen = true; - if local_name != b"snapshot" { - let got = String::from_utf8_lossy(local_name).to_string(); - return Err(RrdpError::UnexpectedRoot(got).into()); - } - let mut xmlns = String::new(); - let mut version = String::new(); - let mut session_id_attr = String::new(); - let mut serial_attr = String::new(); - for attr in e.attributes().with_checks(false) { - let attr = match attr { - Ok(attr) => attr, - Err(e) => return Err(RrdpError::Xml(e.to_string()).into()), - }; - let key = attr.key.as_ref(); - let value = attr - .decode_and_unescape_value(reader.decoder()) - .map_err(|e| RrdpError::Xml(e.to_string()))? - .into_owned(); - match key { - b"xmlns" => xmlns = value, - b"version" => version = value, - b"session_id" => session_id_attr = value, - b"serial" => serial_attr = value, - _ => {} - } - } - if xmlns != RRDP_XMLNS { - return Err(RrdpError::InvalidNamespace(xmlns).into()); - } - if version != "1" { - return Err(RrdpError::InvalidVersion(version).into()); - } - let got_session_id = Uuid::parse_str(&session_id_attr) - .map_err(|_| RrdpError::InvalidSessionId(session_id_attr.clone()))?; - if got_session_id != expected_session_id { - return Err(RrdpError::SnapshotSessionIdMismatch { - expected: expected_session_id.to_string(), - got: got_session_id.to_string(), - } - .into()); - } - let got_serial = parse_u64_str(&serial_attr)?; - if got_serial != expected_serial { - return Err(RrdpError::SnapshotSerialMismatch { - expected: expected_serial, - got: got_serial, - } - .into()); - } - } else if in_publish { - publish_nested_depth += 1; - } else if local_name == b"publish" { - let mut uri = None; - for attr in e.attributes().with_checks(false) { - let attr = match attr { - Ok(attr) => attr, - Err(e) => return Err(RrdpError::Xml(e.to_string()).into()), - }; - if attr.key.as_ref() == b"uri" { - uri = Some( - attr.decode_and_unescape_value(reader.decoder()) - .map_err(|e| RrdpError::Xml(e.to_string()))? - .into_owned(), - ); - } - } - let uri = uri.ok_or(RrdpError::PublishUriMissing)?; - ensure_rrdp_uri_can_be_owned_by(store, notification_uri, &uri) - .map_err(RrdpSyncError::Storage)?; - in_publish = true; - publish_nested_depth = 0; - current_publish_uri = Some(uri); - current_publish_text.clear(); - } - } - Ok(Event::Empty(e)) => { - let local_name = e.local_name(); - let local_name = local_name.as_ref(); - if !root_seen { - let got = String::from_utf8_lossy(local_name).to_string(); - return Err(RrdpError::UnexpectedRoot(got).into()); - } - if local_name == b"publish" { - let mut has_uri = false; - for attr in e.attributes().with_checks(false) { - let attr = match attr { - Ok(attr) => attr, - Err(e) => return Err(RrdpError::Xml(e.to_string()).into()), - }; - if attr.key.as_ref() == b"uri" { - has_uri = true; - break; - } - } - if !has_uri { - return Err(RrdpError::PublishUriMissing.into()); - } - return Err(RrdpError::PublishContentMissing.into()); - } - } - Ok(Event::Text(e)) => { - if in_publish && publish_nested_depth == 0 { - let text = reader - .decoder() - .decode(e.as_ref()) - .map_err(|e| RrdpError::Xml(e.to_string()))?; - current_publish_text.push_str(&text); - } - } - Ok(Event::CData(e)) => { - if in_publish && publish_nested_depth == 0 { - let text = reader - .decoder() - .decode(e.as_ref()) - .map_err(|e| RrdpError::Xml(e.to_string()))?; - current_publish_text.push_str(&text); - } - } - Ok(Event::End(e)) => { - let local_name = e.local_name(); - let local_name = local_name.as_ref(); - if in_publish { - if publish_nested_depth > 0 { - publish_nested_depth -= 1; - } else if local_name == b"publish" { - let uri = current_publish_uri - .take() - .ok_or_else(|| RrdpError::Xml("publish uri missing in state".into()))?; - let content_b64 = strip_all_ascii_whitespace(¤t_publish_text); - current_publish_text.clear(); - if content_b64.is_empty() { - return Err(RrdpError::PublishContentMissing.into()); - } - let bytes = base64::engine::general_purpose::STANDARD - .decode(content_b64.as_bytes()) - .map_err(|e| RrdpError::PublishBase64(e.to_string()))?; - new_set.insert(uri.clone()); - batch_published.push((uri, bytes)); - published_count += 1; - if batch_published.len() >= RRDP_SNAPSHOT_APPLY_BATCH_SIZE { - flush_snapshot_publish_batch( - store, - notification_uri, - current_repo_index, - &session_id, - expected_serial, - &batch_published, - )?; - batch_published.clear(); - } - in_publish = false; - } - } - } - Ok(Event::Eof) => break, - Ok(Event::Decl(_) | Event::PI(_) | Event::Comment(_) | Event::DocType(_)) => {} - Err(e) => return Err(RrdpError::Xml(e.to_string()).into()), - } - buf.clear(); - } - - if !root_seen { - return Err(RrdpError::Xml("missing root element".to_string()).into()); - } - if in_publish { - return Err(RrdpError::PublishContentMissing.into()); - } - if !batch_published.is_empty() { - flush_snapshot_publish_batch( - store, - notification_uri, - current_repo_index, - &session_id, - expected_serial, - &batch_published, - )?; - batch_published.clear(); - } - - let mut withdrawn: Vec<(String, Option)> = Vec::new(); - for old_uri in &previous_members { - if new_set.contains(old_uri) { - continue; - } - let previous_hash = store - .get_repository_view_entry(old_uri) - .map_err(|e| RrdpSyncError::Storage(e.to_string()))? - .and_then(|entry| entry.current_hash) - .or_else(|| { - store - .load_current_object_bytes_by_uri(old_uri) - .ok() - .flatten() - .map(|bytes| compute_sha256_hex(&bytes)) - }); - withdrawn.push((old_uri.clone(), previous_hash)); - } - - let mut repository_view_entries = Vec::with_capacity(withdrawn.len()); - let mut member_records = Vec::with_capacity(withdrawn.len()); - let mut owner_records = Vec::with_capacity(withdrawn.len()); - for (uri, previous_hash) in withdrawn { - member_records.push(build_rrdp_source_member_withdrawn_record( - notification_uri, - &session_id, - expected_serial, - &uri, - previous_hash.clone(), - )); - if current_rrdp_owner_is(store, notification_uri, &uri).map_err(RrdpSyncError::Storage)? { - repository_view_entries.push(build_repository_view_withdrawn_entry( - notification_uri, - &uri, - previous_hash.clone(), - )); - owner_records.push(build_rrdp_uri_owner_withdrawn_record( - notification_uri, - &session_id, - expected_serial, - &uri, - previous_hash, - )); - } - } - store - .put_projection_batch(&repository_view_entries, &member_records, &owner_records) - .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; - if let Some(index) = current_repo_index { - index - .lock() - .map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))? - .apply_repository_view_entries(&repository_view_entries) - .map_err(RrdpSyncError::Storage)?; - } - - Ok(published_count) -} - -fn flush_snapshot_publish_batch( - store: &RocksStore, - notification_uri: &str, - current_repo_index: Option<&CurrentRepoIndexHandle>, - session_id: &str, - serial: u64, - published: &[(String, Vec)], -) -> Result<(), RrdpSyncError> { - let prepared_bytes = prepare_repo_bytes_batch(published).map_err(RrdpSyncError::Storage)?; - let mut repository_view_entries = Vec::with_capacity(published.len()); - let mut member_records = Vec::with_capacity(published.len()); - let mut owner_records = Vec::with_capacity(published.len()); - - for (uri, _bytes) in published { - let current_hash = prepared_bytes - .uri_to_hash - .get(uri) - .cloned() - .ok_or_else(|| { - RrdpSyncError::Storage(format!("missing raw_by_hash mapping for {uri}")) - })?; - repository_view_entries.push(build_repository_view_present_entry( - notification_uri, - uri, - ¤t_hash, - )); - member_records.push(build_rrdp_source_member_present_record( - notification_uri, - session_id, - serial, - uri, - ¤t_hash, - )); - owner_records.push(build_rrdp_uri_owner_active_record( - notification_uri, - session_id, - serial, - uri, - ¤t_hash, - )); - } - - store - .put_blob_bytes_batch(&prepared_bytes.blobs_to_write) - .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; - store - .put_projection_batch(&repository_view_entries, &member_records, &owner_records) - .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; - if let Some(index) = current_repo_index { - index - .lock() - .map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))? - .apply_repository_view_entries(&repository_view_entries) - .map_err(RrdpSyncError::Storage)?; - } - - Ok(()) -} - -const SNAPSHOT_NON_ASCII_ERROR: &str = "snapshot body contains non-ASCII bytes"; - -struct SnapshotSpoolWriter<'a, W: Write> { - inner: &'a mut W, - hasher: sha2::Sha256, - bytes: u64, -} - -impl<'a, W: Write> SnapshotSpoolWriter<'a, W> { - fn new(inner: &'a mut W) -> Self { - Self { - inner, - hasher: sha2::Sha256::new(), - bytes: 0, - } - } - - fn bytes_written(&self) -> u64 { - self.bytes - } - - fn finalize_hash(self) -> [u8; 32] { - let digest = self.hasher.finalize(); - let mut out = [0u8; 32]; - out.copy_from_slice(&digest); - out - } -} - -impl Write for SnapshotSpoolWriter<'_, W> { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - if buf.iter().any(|&b| b > 0x7F) { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - SNAPSHOT_NON_ASCII_ERROR, - )); - } - let n = self.inner.write(buf)?; - self.hasher.update(&buf[..n]); - self.bytes += n as u64; - Ok(n) - } - - fn flush(&mut self) -> std::io::Result<()> { - self.inner.flush() - } -} - -fn fetch_snapshot_into_tempfile( - fetcher: &dyn Fetcher, - snapshot_uri: &str, - expected_hash_sha256: &[u8; 32], -) -> Result<(tempfile::NamedTempFile, u64), RrdpSyncError> { - let mut tmp = tempfile::NamedTempFile::new() - .map_err(|e| RrdpSyncError::Fetch(format!("tempfile create failed: {e}")))?; - let mut spool = SnapshotSpoolWriter::new(tmp.as_file_mut()); - let bytes_written = match fetcher.fetch_to_writer(snapshot_uri, &mut spool) { - Ok(bytes) => bytes, - Err(e) if e.contains(SNAPSHOT_NON_ASCII_ERROR) => return Err(RrdpError::NotAscii.into()), - Err(e) => return Err(RrdpSyncError::Fetch(e)), - }; - let computed = spool.finalize_hash(); - if computed.as_slice() != expected_hash_sha256.as_slice() { - return Err(RrdpError::SnapshotHashMismatch.into()); - } - tmp.as_file_mut() - .flush() - .map_err(|e| RrdpSyncError::Fetch(format!("tempfile flush failed: {e}")))?; - tmp.as_file_mut() - .seek(SeekFrom::Start(0)) - .map_err(|e| RrdpSyncError::Fetch(format!("tempfile rewind failed: {e}")))?; - Ok((tmp, bytes_written)) -} +#[cfg(test)] +use snapshot_apply::apply_snapshot; +use snapshot_apply::{apply_snapshot_from_bufread, fetch_snapshot_into_tempfile}; fn parse_rrdp_xml(xml: &[u8]) -> Result, RrdpError> { if xml.iter().any(|&b| b > 0x7F) { @@ -1779,1254 +1353,5 @@ fn strip_all_ascii_whitespace(s: &str) -> String { } #[cfg(test)] -mod tests { - use super::*; - use crate::analysis::timing::{TimingHandle, TimingMeta}; - use crate::current_repo_index::CurrentRepoIndex; - use crate::storage::RocksStore; - use std::collections::HashMap; - use std::time::Duration; - - struct MapFetcher { - map: HashMap>, - } - - impl Fetcher for MapFetcher { - fn fetch(&self, uri: &str) -> Result, String> { - self.map - .get(uri) - .cloned() - .ok_or_else(|| format!("not found: {uri}")) - } - } - - struct SleepyFetcher { - inner: MapFetcher, - sleep_uri: String, - sleep: Duration, - } - - impl Fetcher for SleepyFetcher { - fn fetch(&self, uri: &str) -> Result, String> { - if uri == self.sleep_uri { - std::thread::sleep(self.sleep); - } - self.inner.fetch(uri) - } - } - - fn assert_current_object(store: &RocksStore, uri: &str, expected: &[u8]) { - assert_eq!( - store - .load_current_object_bytes_by_uri(uri) - .expect("load current object"), - Some(expected.to_vec()) - ); - } - - fn notification_xml( - session_id: &str, - serial: u64, - snapshot_uri: &str, - snapshot_hash: &str, - ) -> Vec { - format!( - r#""# - ) - .into_bytes() - } - - fn notification_xml_with_deltas( - session_id: &str, - serial: u64, - snapshot_uri: &str, - snapshot_hash: &str, - deltas: &[(&str, u64, &str, &str)], - ) -> Vec { - let mut out = format!( - r#""# - ); - for (_name, delta_serial, uri, hash) in deltas { - out.push_str(&format!( - r#""# - )); - } - out.push_str(""); - out.into_bytes() - } - - fn snapshot_xml(session_id: &str, serial: u64, published: &[(&str, &[u8])]) -> Vec { - let mut out = format!( - r#""# - ); - for (uri, bytes) in published { - let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); - out.push_str(&format!(r#"{b64}"#)); - } - out.push_str(""); - out.into_bytes() - } - - #[test] - fn timing_rrdp_repo_step_spans_cover_snapshot_fetch_duration() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - - let notification_uri = "https://example.test/notification.xml"; - let snapshot_uri = "https://example.test/snapshot.xml"; - let published_uri = "rsync://example.test/repo/a.mft"; - let published_bytes = b"x"; - let session_id = "550e8400-e29b-41d4-a716-446655440000"; - - let snapshot = snapshot_xml(session_id, 1, &[(published_uri, published_bytes)]); - let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); - let notif = notification_xml(session_id, 1, snapshot_uri, &snapshot_hash); - - let mut map = HashMap::new(); - map.insert(snapshot_uri.to_string(), snapshot); - let fetcher = SleepyFetcher { - inner: MapFetcher { map }, - sleep_uri: snapshot_uri.to_string(), - sleep: Duration::from_millis(25), - }; - - let timing = TimingHandle::new(TimingMeta { - recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - tal_url: None, - db_path: Some(store_dir.to_string_lossy().into_owned()), - }); - - sync_from_notification_snapshot_with_timing( - &store, - notification_uri, - ¬if, - &fetcher, - Some(&timing), - ) - .expect("rrdp snapshot sync ok"); - - let timing_path = temp.path().join("timing.json"); - timing.write_json(&timing_path, 200).expect("write timing"); - let rep: serde_json::Value = - serde_json::from_slice(&std::fs::read(&timing_path).expect("read timing")) - .expect("parse timing"); - - let want = format!("{notification_uri}::fetch_snapshot"); - let steps = rep - .get("top_rrdp_repo_steps") - .and_then(|v| v.as_array()) - .expect("top_rrdp_repo_steps array"); - let entry = steps - .iter() - .find(|e| e.get("key").and_then(|k| k.as_str()) == Some(want.as_str())) - .unwrap_or_else(|| panic!("missing timing step entry for {want}")); - let nanos = entry - .get("total_nanos") - .and_then(|v| v.as_u64()) - .expect("total_nanos"); - assert!( - nanos >= 20_000_000, - "expected fetch_snapshot timing to include the fetch duration; got {nanos}ns" - ); - } - - #[test] - fn parse_notification_snapshot_rejects_non_ascii() { - let mut xml = b"".to_vec(); - xml.push(0x80); - let err = parse_notification_snapshot(&xml).unwrap_err(); - assert!(matches!(err, RrdpError::NotAscii)); - } - - #[test] - fn parse_notification_snapshot_parses_valid_minimal_notification() { - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let snapshot_uri = "https://example.net/snapshot.xml"; - let hash = "00".repeat(32); - let xml = notification_xml(sid, 7, snapshot_uri, &hash); - let n = parse_notification_snapshot(&xml).expect("parse"); - assert_eq!(n.session_id, Uuid::parse_str(sid).unwrap()); - assert_eq!(n.serial, 7); - assert_eq!(n.snapshot_uri, snapshot_uri); - assert_eq!(hex::encode(n.snapshot_hash_sha256), hash); - } - - #[test] - fn parse_notification_parses_deltas_and_validates_contiguity() { - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let snapshot_uri = "https://example.net/snapshot.xml"; - let hash = "00".repeat(32); - let d_hash_2 = "11".repeat(32); - let d_hash_3 = "22".repeat(32); - // Provide deltas in reverse order to ensure we sort. - let xml = notification_xml_with_deltas( - sid, - 3, - snapshot_uri, - &hash, - &[ - ("d3", 3, "https://example.net/delta-3.xml", &d_hash_3), - ("d2", 2, "https://example.net/delta-2.xml", &d_hash_2), - ], - ); - let n = parse_notification(&xml).expect("parse notification"); - assert_eq!(n.serial, 3); - assert_eq!(n.deltas.len(), 2); - assert_eq!(n.deltas[0].serial, 2); - assert_eq!(n.deltas[1].serial, 3); - assert_eq!(n.deltas[0].uri, "https://example.net/delta-2.xml"); - assert_eq!(hex::encode(n.deltas[1].hash_sha256), d_hash_3); - } - - #[test] - fn parse_notification_rejects_non_contiguous_deltas() { - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let snapshot_uri = "https://example.net/snapshot.xml"; - let hash = "00".repeat(32); - let d_hash_1 = "11".repeat(32); - let d_hash_3 = "22".repeat(32); - // Missing delta serial 2. - let xml = notification_xml_with_deltas( - sid, - 3, - snapshot_uri, - &hash, - &[ - ("d3", 3, "https://example.net/delta-3.xml", &d_hash_3), - ("d1", 1, "https://example.net/delta-1.xml", &d_hash_1), - ], - ); - let err = parse_notification(&xml).unwrap_err(); - assert!(matches!(err, RrdpError::DeltaRefChainNotContiguous { .. })); - } - - fn delta_xml(session_id: &str, serial: u64, elements: &[&str]) -> Vec { - let mut out = format!( - r#""# - ); - for e in elements { - out.push_str(e); - } - out.push_str(""); - out.into_bytes() - } - - #[test] - fn parse_delta_file_parses_publish_and_withdraw() { - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let serial = 3u64; - let publish_bytes = b"abc"; - let publish_b64 = base64::engine::general_purpose::STANDARD.encode(publish_bytes); - let withdraw_hash = "33".repeat(32); - - let xml = delta_xml( - sid, - serial, - &[ - &format!( - r#"{publish_b64}"# - ), - &format!( - r#""# - ), - ], - ); - - let d = parse_delta_file(&xml).expect("parse delta"); - assert_eq!(d.session_id, Uuid::parse_str(sid).unwrap()); - assert_eq!(d.serial, serial); - assert_eq!(d.elements.len(), 2); - match &d.elements[0] { - DeltaElement::Publish { - uri, - hash_sha256, - bytes, - } => { - assert_eq!(uri, "rsync://example.net/repo/a.mft"); - assert_eq!(*hash_sha256, None); - assert_eq!(bytes, publish_bytes); - } - _ => panic!("expected publish"), - } - match &d.elements[1] { - DeltaElement::Withdraw { uri, hash_sha256 } => { - assert_eq!(uri, "rsync://example.net/repo/b.cer"); - assert_eq!(hex::encode(hash_sha256), withdraw_hash); - } - _ => panic!("expected withdraw"), - } - } - - #[test] - fn parse_delta_file_rejects_withdraw_with_content() { - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let serial = 1u64; - let withdraw_hash = "33".repeat(32); - let xml = delta_xml( - sid, - serial, - &[&format!( - r#"AA=="# - )], - ); - let err = parse_delta_file(&xml).unwrap_err(); - assert!(matches!(err, RrdpError::DeltaWithdrawUnexpectedContent)); - } - - #[test] - fn apply_delta_applies_publish_replace_and_withdraw_with_membership_checks() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - - // Start from snapshot state with a + b - let snapshot_uri = "https://example.net/snapshot.xml"; - let snapshot = snapshot_xml( - sid, - 1, - &[ - ("rsync://example.net/repo/a.mft", b"a1"), - ("rsync://example.net/repo/b.roa", b"b1"), - ], - ); - let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); - let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); - let fetcher = MapFetcher { - map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher) - .expect("sync snapshot"); - - let old_b = store - .load_current_object_bytes_by_uri("rsync://example.net/repo/b.roa") - .expect("load current b") - .expect("b present"); - let old_b_hash = hex::encode(sha2::Sha256::digest(old_b.as_slice())); - - let withdraw_a_hash = hex::encode(sha2::Sha256::digest(b"a1".as_slice())); - let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2"); - let replace_b_b64 = base64::engine::general_purpose::STANDARD.encode(b"b2"); - - let delta = delta_xml( - sid, - 2, - &[ - &format!( - r#""# - ), - &format!( - r#"{replace_b_b64}"# - ), - &format!( - r#"{publish_c_b64}"# - ), - ], - ); - let delta_hash = sha2::Sha256::digest(&delta); - let mut expected_hash = [0u8; 32]; - expected_hash.copy_from_slice(delta_hash.as_slice()); - - let applied = apply_delta( - &store, - notif_uri, - None, - &delta, - expected_hash, - Uuid::parse_str(sid).unwrap(), - 2, - ) - .expect("apply delta"); - assert_eq!(applied, 3); - - assert_eq!( - store - .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") - .expect("load current a"), - None, - "a withdrawn" - ); - let b = store - .load_current_object_bytes_by_uri("rsync://example.net/repo/b.roa") - .expect("load current b") - .expect("b present"); - assert_eq!(b, b"b2"); - let c = store - .load_current_object_bytes_by_uri("rsync://example.net/repo/c.crl") - .expect("load current c") - .expect("c present"); - assert_eq!(c, b"c2"); - - assert!( - !store - .is_current_rrdp_source_member(notif_uri, "rsync://example.net/repo/a.mft") - .expect("is member"), - "a removed from rrdp repo index" - ); - assert!( - store - .is_current_rrdp_source_member(notif_uri, "rsync://example.net/repo/c.crl") - .expect("is member"), - "c added to rrdp repo index" - ); - - let a_view = store - .get_repository_view_entry("rsync://example.net/repo/a.mft") - .expect("get a view") - .expect("a view exists"); - assert_eq!(a_view.state, crate::storage::RepositoryViewState::Withdrawn); - let b_view = store - .get_repository_view_entry("rsync://example.net/repo/b.roa") - .expect("get b view") - .expect("b view exists"); - assert_eq!(b_view.state, crate::storage::RepositoryViewState::Present); - assert_eq!( - b_view.current_hash.as_deref(), - Some(hex::encode(sha2::Sha256::digest(b"b2")).as_str()) - ); - let c_owner = store - .get_rrdp_uri_owner_record("rsync://example.net/repo/c.crl") - .expect("get c owner") - .expect("c owner exists"); - assert_eq!( - c_owner.owner_state, - crate::storage::RrdpUriOwnerState::Active - ); - let a_member = store - .get_rrdp_source_member_record(notif_uri, "rsync://example.net/repo/a.mft") - .expect("get a member") - .expect("a member exists"); - assert!(!a_member.present); - let current_members = store - .list_current_rrdp_source_members(notif_uri) - .expect("list current members"); - assert_eq!( - current_members - .iter() - .map(|record| record.rsync_uri.as_str()) - .collect::>(), - vec![ - "rsync://example.net/repo/b.roa", - "rsync://example.net/repo/c.crl", - ] - ); - } - - #[test] - fn apply_delta_rejects_hash_mismatch() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); - let notif_uri = "https://example.net/notification.xml"; - - let delta = delta_xml( - sid.to_string().as_str(), - 1, - &[r#"QQ=="#], - ); - let mut wrong = [0u8; 32]; - wrong[0] = 1; - let err = apply_delta(&store, notif_uri, None, &delta, wrong, sid, 1).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::DeltaHashMismatch) - )); - } - - #[test] - fn apply_delta_rejects_withdraw_of_non_member() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); - let notif_uri = "https://example.net/notification.xml"; - - let withdraw_hash = "00".repeat(32); - let delta = delta_xml( - sid.to_string().as_str(), - 1, - &[&format!( - r#""# - )], - ); - let delta_hash = sha2::Sha256::digest(&delta); - let mut expected_hash = [0u8; 32]; - expected_hash.copy_from_slice(delta_hash.as_slice()); - - let err = apply_delta(&store, notif_uri, None, &delta, expected_hash, sid, 1).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::DeltaTargetNotFromRepository { .. }) - )); - } - - #[test] - fn apply_delta_rejects_publish_without_hash_for_existing_object() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - - // Seed snapshot with a.mft. - let snapshot_uri = "https://example.net/snapshot.xml"; - let snapshot = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]); - let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); - let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); - let fetcher = MapFetcher { - map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("seed"); - - // Replace publish for an existing URI must have @hash. - let publish_b64 = base64::engine::general_purpose::STANDARD.encode(b"a2"); - let delta = delta_xml( - sid, - 2, - &[&format!( - r#"{publish_b64}"# - )], - ); - let delta_hash = sha2::Sha256::digest(&delta); - let mut expected_hash = [0u8; 32]; - expected_hash.copy_from_slice(delta_hash.as_slice()); - - let err = apply_delta( - &store, - notif_uri, - None, - &delta, - expected_hash, - Uuid::parse_str(sid).unwrap(), - 2, - ) - .unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::DeltaPublishWithoutHashForExisting { .. }) - )); - } - - #[test] - fn apply_delta_rejects_target_missing_and_hash_mismatch() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - - // Seed snapshot with a.mft. - let snapshot_uri = "https://example.net/snapshot.xml"; - let snapshot = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]); - let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); - let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); - let fetcher = MapFetcher { - map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("seed"); - - let old_bytes = store - .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") - .expect("get") - .expect("present"); - let old_hash = hex::encode(sha2::Sha256::digest(old_bytes.as_slice())); - - // Hash mismatch on withdraw. - let wrong_hash = "11".repeat(32); - let delta = delta_xml( - sid, - 2, - &[&format!( - r#""# - )], - ); - let delta_hash = sha2::Sha256::digest(&delta); - let mut expected_hash = [0u8; 32]; - expected_hash.copy_from_slice(delta_hash.as_slice()); - let err = apply_delta( - &store, - notif_uri, - None, - &delta, - expected_hash, - Uuid::parse_str(sid).unwrap(), - 2, - ) - .unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::DeltaTargetHashMismatch { .. }) - )); - - // Target missing in local cache (index still says it's a member). - store - .delete_repository_view_entry("rsync://example.net/repo/a.mft") - .expect("delete current repository view entry"); - let delta = delta_xml( - sid, - 2, - &[&format!( - r#""# - )], - ); - let delta_hash = sha2::Sha256::digest(&delta); - let mut expected_hash = [0u8; 32]; - expected_hash.copy_from_slice(delta_hash.as_slice()); - let err = apply_delta( - &store, - notif_uri, - None, - &delta, - expected_hash, - Uuid::parse_str(sid).unwrap(), - 2, - ) - .unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::DeltaTargetMissing { .. }) - )); - } - - #[test] - fn apply_delta_rejects_session_and_serial_mismatch() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - - let publish_b64 = base64::engine::general_purpose::STANDARD.encode(b"x"); - let delta = delta_xml( - sid, - 2, - &[&format!( - r#"{publish_b64}"# - )], - ); - let delta_hash = sha2::Sha256::digest(&delta); - let mut expected_hash = [0u8; 32]; - expected_hash.copy_from_slice(delta_hash.as_slice()); - - // Session mismatch. - let other_sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440001").unwrap(); - let err = - apply_delta(&store, notif_uri, None, &delta, expected_hash, other_sid, 2).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::DeltaSessionIdMismatch { .. }) - )); - - // Serial mismatch. - let err = apply_delta( - &store, - notif_uri, - None, - &delta, - expected_hash, - Uuid::parse_str(sid).unwrap(), - 3, - ) - .unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::DeltaSerialMismatch { .. }) - )); - } - - #[test] - fn sync_from_notification_snapshot_rejects_cross_source_owner_conflict() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid_a = "550e8400-e29b-41d4-a716-446655440000"; - let sid_b = "550e8400-e29b-41d4-a716-446655440001"; - let uri = "rsync://example.net/repo/a.mft"; - - let notif_a_uri = "https://example.net/a/notification.xml"; - let snapshot_a_uri = "https://example.net/a/snapshot.xml"; - let snapshot_a = snapshot_xml(sid_a, 1, &[(uri, b"a1")]); - let snapshot_a_hash = hex::encode(sha2::Sha256::digest(&snapshot_a)); - let notif_a = notification_xml(sid_a, 1, snapshot_a_uri, &snapshot_a_hash); - let fetcher_a = MapFetcher { - map: HashMap::from([(snapshot_a_uri.to_string(), snapshot_a)]), - }; - sync_from_notification_snapshot(&store, notif_a_uri, ¬if_a, &fetcher_a) - .expect("seed source a"); - - let notif_b_uri = "https://example.net/b/notification.xml"; - let snapshot_b_uri = "https://example.net/b/snapshot.xml"; - let snapshot_b = snapshot_xml(sid_b, 1, &[(uri, b"b1")]); - let snapshot_b_hash = hex::encode(sha2::Sha256::digest(&snapshot_b)); - let notif_b = notification_xml(sid_b, 1, snapshot_b_uri, &snapshot_b_hash); - let fetcher_b = MapFetcher { - map: HashMap::from([(snapshot_b_uri.to_string(), snapshot_b)]), - }; - - let err = sync_from_notification_snapshot(&store, notif_b_uri, ¬if_b, &fetcher_b) - .expect_err("cross-source overwrite must fail"); - assert!(matches!(err, RrdpSyncError::Storage(_))); - assert!(err.to_string().contains("owner conflict"), "{err}"); - } - - #[test] - fn sync_from_notification_snapshot_applies_snapshot_and_stores_state() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let serial = 9u64; - let notif_uri = "https://example.net/notification.xml"; - let snapshot_uri = "https://example.net/snapshot.xml"; - - let snapshot = snapshot_xml( - sid, - serial, - &[ - ("rsync://example.net/repo/a.mft", b"mft-bytes"), - ("rsync://example.net/repo/b.roa", b"roa-bytes"), - ], - ); - let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); - let notif = notification_xml(sid, serial, snapshot_uri, &snapshot_hash); - - let fetcher = MapFetcher { - map: HashMap::from([(snapshot_uri.to_string(), snapshot.clone())]), - }; - - let published = - sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("sync"); - assert_eq!(published, 2); - - assert_current_object(&store, "rsync://example.net/repo/a.mft", b"mft-bytes"); - assert_current_object(&store, "rsync://example.net/repo/b.roa", b"roa-bytes"); - - let state = load_rrdp_local_state(&store, notif_uri) - .expect("get rrdp state") - .expect("state present"); - assert_eq!(state.session_id, sid); - assert_eq!(state.serial, serial); - - let source = store - .get_rrdp_source_record(notif_uri) - .expect("get rrdp source") - .expect("rrdp source exists"); - assert_eq!(source.last_session_id.as_deref(), Some(sid)); - assert_eq!(source.last_serial, Some(serial)); - assert_eq!( - source.sync_state, - crate::storage::RrdpSourceSyncState::SnapshotOnly - ); - - let view = store - .get_repository_view_entry("rsync://example.net/repo/a.mft") - .expect("get repository view") - .expect("repository view exists"); - assert_eq!(view.state, crate::storage::RepositoryViewState::Present); - assert_eq!(view.repository_source.as_deref(), Some(notif_uri)); - - let current_bytes = store - .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") - .expect("load current bytes") - .expect("current object bytes exist"); - assert_eq!(current_bytes, b"mft-bytes".to_vec()); - assert!( - store - .get_raw_by_hash_entry(hex::encode(sha2::Sha256::digest(b"mft-bytes")).as_str()) - .expect("get raw_by_hash") - .is_none() - ); - - let member = store - .get_rrdp_source_member_record(notif_uri, "rsync://example.net/repo/a.mft") - .expect("get member") - .expect("member exists"); - assert!(member.present); - let owner = store - .get_rrdp_uri_owner_record("rsync://example.net/repo/a.mft") - .expect("get owner") - .expect("owner exists"); - assert_eq!(owner.notify_uri, notif_uri); - assert_eq!(owner.owner_state, crate::storage::RrdpUriOwnerState::Active); - } - - #[test] - fn sync_from_notification_snapshot_deletes_objects_not_in_new_snapshot() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - - // serial 1: publish a + b - let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; - let snapshot_1 = snapshot_xml( - sid, - 1, - &[ - ("rsync://example.net/repo/a.mft", b"a1"), - ("rsync://example.net/repo/b.roa", b"b1"), - ], - ); - let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); - let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); - - let fetcher_1 = MapFetcher { - map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("sync 1"); - - // serial 2: publish b (new bytes) + c, and drop a - let snapshot_uri_2 = "https://example.net/snapshot-2.xml"; - let snapshot_2 = snapshot_xml( - sid, - 2, - &[ - ("rsync://example.net/repo/b.roa", b"b2"), - ("rsync://example.net/repo/c.crl", b"c2"), - ], - ); - let snapshot_hash_2 = hex::encode(sha2::Sha256::digest(&snapshot_2)); - let notif_2 = notification_xml(sid, 2, snapshot_uri_2, &snapshot_hash_2); - - let fetcher_2 = MapFetcher { - map: HashMap::from([(snapshot_uri_2.to_string(), snapshot_2)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if_2, &fetcher_2).expect("sync 2"); - - assert!( - store - .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") - .expect("get current a") - .is_none(), - "a should be deleted by full-state snapshot apply" - ); - - assert_current_object(&store, "rsync://example.net/repo/b.roa", b"b2"); - assert_current_object(&store, "rsync://example.net/repo/c.crl", b"c2"); - } - - #[test] - fn sync_from_notification_uses_deltas_when_available_for_local_state() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - - // Seed state with snapshot serial=1 containing a+b. - let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; - let snapshot_1 = snapshot_xml( - sid, - 1, - &[ - ("rsync://example.net/repo/a.mft", b"a1"), - ("rsync://example.net/repo/b.roa", b"b1"), - ], - ); - let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); - let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); - let fetcher_1 = MapFetcher { - map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed"); - - // Notification serial=3 with deltas 2 and 3. Snapshot URI is intentionally not fetchable - // to assert we really use deltas. - let snapshot_uri_3 = "https://example.net/snapshot-3.xml"; - let snapshot_hash_3 = "00".repeat(32); - - let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2"); - let delta_2 = delta_xml( - sid, - 2, - &[&format!( - r#"{publish_c_b64}"# - )], - ); - let delta_2_hash_hex = hex::encode(sha2::Sha256::digest(&delta_2)); - - let c_hash_hex = hex::encode(sha2::Sha256::digest(b"c2".as_slice())); - let delta_3 = delta_xml( - sid, - 3, - &[&format!( - r#""# - )], - ); - let delta_3_hash_hex = hex::encode(sha2::Sha256::digest(&delta_3)); - - let notif_3 = notification_xml_with_deltas( - sid, - 3, - snapshot_uri_3, - &snapshot_hash_3, - &[ - ( - "d3", - 3, - "https://example.net/delta-3.xml", - &delta_3_hash_hex, - ), - ( - "d2", - 2, - "https://example.net/delta-2.xml", - &delta_2_hash_hex, - ), - ], - ); - - let fetcher = MapFetcher { - map: HashMap::from([ - ("https://example.net/delta-2.xml".to_string(), delta_2), - ("https://example.net/delta-3.xml".to_string(), delta_3), - ]), - }; - - let applied = sync_from_notification(&store, notif_uri, ¬if_3, &fetcher).expect("sync"); - assert!(applied > 0); - - // Delta 2 publishes c then delta 3 withdraws it => final state should not contain c. - assert!( - store - .load_current_object_bytes_by_uri("rsync://example.net/repo/c.crl") - .expect("get current") - .is_none() - ); - - let state = load_rrdp_local_state(&store, notif_uri) - .expect("get rrdp state") - .expect("state present"); - assert_eq!(state.session_id, Uuid::parse_str(sid).unwrap().to_string()); - assert_eq!(state.serial, 3); - } - - #[test] - fn sync_from_notification_same_serial_hydrates_current_repo_index() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - let snapshot_uri = "https://example.net/snapshot.xml"; - let uri_a = "rsync://example.net/repo/a.mft"; - let uri_b = "rsync://example.net/repo/b.roa"; - - let snapshot = snapshot_xml(sid, 1, &[(uri_a, b"a1"), (uri_b, b"b1")]); - let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); - let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); - let fetcher_1 = MapFetcher { - map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher_1).expect("seed"); - - let index = CurrentRepoIndex::shared(); - let no_fetcher = MapFetcher { - map: HashMap::new(), - }; - let applied = sync_from_notification_with_timing_and_download_log( - &store, - notif_uri, - Some(&index), - ¬if, - &no_fetcher, - None, - None, - ) - .expect("same serial no-op"); - assert_eq!(applied, 0); - - let index = index.lock().expect("lock index"); - assert_eq!(index.active_uri_count(), 2); - assert!(index.get_by_uri(uri_a).is_some()); - assert!(index.get_by_uri(uri_b).is_some()); - } - - #[test] - fn sync_from_notification_delta_hydrates_unchanged_current_repo_entries() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; - let uri_a = "rsync://example.net/repo/a.mft"; - let uri_b = "rsync://example.net/repo/b.roa"; - let uri_c = "rsync://example.net/repo/c.crl"; - - let snapshot_1 = snapshot_xml(sid, 1, &[(uri_a, b"a1"), (uri_b, b"b1")]); - let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); - let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); - let fetcher_1 = MapFetcher { - map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed"); - - let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2"); - let delta_2 = delta_xml( - sid, - 2, - &[&format!( - r#"{publish_c_b64}"# - )], - ); - let delta_2_hash_hex = hex::encode(sha2::Sha256::digest(&delta_2)); - let notif_2 = notification_xml_with_deltas( - sid, - 2, - "https://example.net/snapshot-2.xml", - &"00".repeat(32), - &[( - "d2", - 2, - "https://example.net/delta-2.xml", - &delta_2_hash_hex, - )], - ); - let fetcher_2 = MapFetcher { - map: HashMap::from([("https://example.net/delta-2.xml".to_string(), delta_2)]), - }; - - let index = CurrentRepoIndex::shared(); - let applied = sync_from_notification_with_timing_and_download_log( - &store, - notif_uri, - Some(&index), - ¬if_2, - &fetcher_2, - None, - None, - ) - .expect("delta sync"); - assert_eq!(applied, 1); - - let index = index.lock().expect("lock index"); - assert_eq!(index.active_uri_count(), 3); - assert!( - index.get_by_uri(uri_a).is_some(), - "unchanged object from the previous serial must be visible" - ); - assert!( - index.get_by_uri(uri_b).is_some(), - "unchanged object from the previous serial must be visible" - ); - assert!(index.get_by_uri(uri_c).is_some(), "delta publish visible"); - } - - #[test] - fn load_rrdp_local_state_uses_source_record_only() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - let notif_uri = "https://example.net/notification.xml"; - assert_eq!( - load_rrdp_local_state(&store, notif_uri).expect("load empty"), - None - ); - - update_rrdp_source_record_on_success( - &store, - notif_uri, - "source-session", - 9, - crate::storage::RrdpSourceSyncState::DeltaReady, - Some("https://example.net/snapshot.xml"), - Some(&hex::encode([0x11; 32])), - ) - .expect("write source record"); - - let got = load_rrdp_local_state(&store, notif_uri) - .expect("load source preferred") - .expect("source present"); - assert_eq!( - got, - RrdpState { - session_id: "source-session".to_string(), - serial: 9, - } - ); - } - - #[test] - fn sync_from_notification_falls_back_to_snapshot_if_missing_required_deltas() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - - // Seed state serial=1 with a only. - let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; - let snapshot_1 = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]); - let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); - let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); - let fetcher_1 = MapFetcher { - map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed"); - - // Notification serial=3 only includes delta serial=3 (contiguous per RFC, but missing - // serial=2 relative to our local state, so we must use snapshot). - let snapshot_uri_3 = "https://example.net/snapshot-3.xml"; - let snapshot_3 = snapshot_xml(sid, 3, &[("rsync://example.net/repo/z.roa", b"z3")]); - let snapshot_hash_3 = hex::encode(sha2::Sha256::digest(&snapshot_3)); - let delta_3_hash_hex = "11".repeat(32); - let notif_3 = notification_xml_with_deltas( - sid, - 3, - snapshot_uri_3, - &snapshot_hash_3, - &[( - "d3", - 3, - "https://example.net/delta-3.xml", - &delta_3_hash_hex, - )], - ); - - let fetcher = MapFetcher { - map: HashMap::from([(snapshot_uri_3.to_string(), snapshot_3)]), - }; - - let published = - sync_from_notification(&store, notif_uri, ¬if_3, &fetcher).expect("sync"); - assert_eq!(published, 1); - assert!( - store - .load_current_object_bytes_by_uri("rsync://example.net/repo/z.roa") - .expect("get current") - .is_some() - ); - } - - #[test] - fn sync_from_notification_snapshot_rejects_snapshot_hash_mismatch() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let serial = 1u64; - let notif_uri = "https://example.net/notification.xml"; - let snapshot_uri = "https://example.net/snapshot.xml"; - - let snapshot = snapshot_xml(sid, serial, &[("rsync://example.net/repo/a.mft", b"x")]); - let notif = notification_xml(sid, serial, snapshot_uri, &"00".repeat(32)); - - let fetcher = MapFetcher { - map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), - }; - let err = sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::SnapshotHashMismatch) - )); - } - - #[test] - fn apply_snapshot_rejects_session_id_and_serial_mismatch() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - let notif_uri = "https://example.net/notification.xml"; - - let expected_sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); - let got_sid = "550e8400-e29b-41d4-a716-446655440001"; - - let snapshot = snapshot_xml(got_sid, 2, &[("rsync://example.net/repo/a.mft", b"x")]); - let err = apply_snapshot(&store, notif_uri, None, &snapshot, expected_sid, 2).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::SnapshotSessionIdMismatch { .. }) - )); - - let snapshot = snapshot_xml( - expected_sid.to_string().as_str(), - 3, - &[("rsync://example.net/repo/a.mft", b"x")], - ); - let err = apply_snapshot(&store, notif_uri, None, &snapshot, expected_sid, 2).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::SnapshotSerialMismatch { .. }) - )); - } - - #[test] - fn strip_all_ascii_whitespace_removes_newlines_and_spaces() { - assert_eq!(strip_all_ascii_whitespace(" a \n b\tc "), "abc"); - } - - #[test] - fn apply_snapshot_reports_publish_errors() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - let notif_uri = "https://example.net/notification.xml"; - let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); - - // Missing publish/@uri - let xml = format!( - r#"AA=="# - ) - .into_bytes(); - let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::PublishUriMissing) - )); - - // Missing base64 content (no text nodes). - let xml = format!( - r#""# - ) - .into_bytes(); - let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::PublishContentMissing) - )); - - // Invalid base64 content. - let xml = format!( - r#"!!!"# - ) - .into_bytes(); - let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::PublishBase64(_)) - )); - } - - #[test] - fn apply_snapshot_handles_multiple_publish_batches() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - let notif_uri = "https://example.net/notification.xml"; - let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); - - let total = RRDP_SNAPSHOT_APPLY_BATCH_SIZE + 7; - let mut xml = - format!(r#""#); - for i in 0..total { - let uri = format!("rsync://example.net/repo/{i:04}.roa"); - let bytes = format!("payload-{i}").into_bytes(); - let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); - xml.push_str(&format!(r#"{b64}"#)); - } - xml.push_str(""); - - let published = apply_snapshot(&store, notif_uri, None, xml.as_bytes(), sid, 1) - .expect("apply snapshot"); - assert_eq!(published, total); - - for idx in [0usize, RRDP_SNAPSHOT_APPLY_BATCH_SIZE - 1, total - 1] { - let uri = format!("rsync://example.net/repo/{idx:04}.roa"); - let got = store - .load_current_object_bytes_by_uri(&uri) - .expect("load object") - .expect("object exists"); - assert_eq!(got, format!("payload-{idx}").into_bytes()); - } - } -} +#[path = "rrdp/tests.rs"] +mod tests; diff --git a/src/sync/rrdp/snapshot_apply.rs b/src/sync/rrdp/snapshot_apply.rs new file mode 100644 index 0000000..731febf --- /dev/null +++ b/src/sync/rrdp/snapshot_apply.rs @@ -0,0 +1,445 @@ +use base64::Engine; +use quick_xml::Reader; +use quick_xml::events::Event; +use sha2::Digest; +use std::io::{BufRead, Seek, SeekFrom, Write}; +use uuid::Uuid; + +use crate::current_repo_index::CurrentRepoIndexHandle; +use crate::storage::RocksStore; +use crate::sync::store_projection::{ + build_repository_view_present_entry, build_repository_view_withdrawn_entry, + build_rrdp_source_member_present_record, build_rrdp_source_member_withdrawn_record, + build_rrdp_uri_owner_active_record, build_rrdp_uri_owner_withdrawn_record, compute_sha256_hex, + current_rrdp_owner_is, ensure_rrdp_uri_can_be_owned_by, prepare_repo_bytes_batch, +}; + +use super::{ + Fetcher, RRDP_SNAPSHOT_APPLY_BATCH_SIZE, RRDP_XMLNS, RrdpError, RrdpSyncError, parse_u64_str, + strip_all_ascii_whitespace, +}; + +#[cfg(test)] +pub(super) fn apply_snapshot( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + snapshot_xml: &[u8], + expected_session_id: Uuid, + expected_serial: u64, +) -> Result { + if snapshot_xml.iter().any(|&b| b > 0x7F) { + return Err(RrdpError::NotAscii.into()); + } + apply_snapshot_from_bufread( + store, + notification_uri, + current_repo_index, + std::io::Cursor::new(snapshot_xml), + expected_session_id, + expected_serial, + ) +} + +pub(super) fn apply_snapshot_from_bufread( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + input: R, + expected_session_id: Uuid, + expected_serial: u64, +) -> Result { + let previous_members: Vec = store + .list_current_rrdp_source_members(notification_uri) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))? + .into_iter() + .map(|record| record.rsync_uri) + .collect(); + let session_id = expected_session_id.to_string(); + let mut new_set: std::collections::HashSet = std::collections::HashSet::new(); + let mut batch_published: Vec<(String, Vec)> = + Vec::with_capacity(RRDP_SNAPSHOT_APPLY_BATCH_SIZE); + let mut published_count = 0usize; + + let mut reader = Reader::from_reader(input); + reader.config_mut().trim_text(false); + let mut buf = Vec::new(); + let mut root_seen = false; + let mut in_publish = false; + let mut publish_nested_depth = 0usize; + let mut current_publish_uri: Option = None; + let mut current_publish_text = String::new(); + + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(e)) => { + let local_name = e.local_name(); + let local_name = local_name.as_ref(); + if !root_seen { + root_seen = true; + if local_name != b"snapshot" { + let got = String::from_utf8_lossy(local_name).to_string(); + return Err(RrdpError::UnexpectedRoot(got).into()); + } + let mut xmlns = String::new(); + let mut version = String::new(); + let mut session_id_attr = String::new(); + let mut serial_attr = String::new(); + for attr in e.attributes().with_checks(false) { + let attr = match attr { + Ok(attr) => attr, + Err(e) => return Err(RrdpError::Xml(e.to_string()).into()), + }; + let key = attr.key.as_ref(); + let value = attr + .decode_and_unescape_value(reader.decoder()) + .map_err(|e| RrdpError::Xml(e.to_string()))? + .into_owned(); + match key { + b"xmlns" => xmlns = value, + b"version" => version = value, + b"session_id" => session_id_attr = value, + b"serial" => serial_attr = value, + _ => {} + } + } + if xmlns != RRDP_XMLNS { + return Err(RrdpError::InvalidNamespace(xmlns).into()); + } + if version != "1" { + return Err(RrdpError::InvalidVersion(version).into()); + } + let got_session_id = Uuid::parse_str(&session_id_attr) + .map_err(|_| RrdpError::InvalidSessionId(session_id_attr.clone()))?; + if got_session_id != expected_session_id { + return Err(RrdpError::SnapshotSessionIdMismatch { + expected: expected_session_id.to_string(), + got: got_session_id.to_string(), + } + .into()); + } + let got_serial = parse_u64_str(&serial_attr)?; + if got_serial != expected_serial { + return Err(RrdpError::SnapshotSerialMismatch { + expected: expected_serial, + got: got_serial, + } + .into()); + } + } else if in_publish { + publish_nested_depth += 1; + } else if local_name == b"publish" { + let mut uri = None; + for attr in e.attributes().with_checks(false) { + let attr = match attr { + Ok(attr) => attr, + Err(e) => return Err(RrdpError::Xml(e.to_string()).into()), + }; + if attr.key.as_ref() == b"uri" { + uri = Some( + attr.decode_and_unescape_value(reader.decoder()) + .map_err(|e| RrdpError::Xml(e.to_string()))? + .into_owned(), + ); + } + } + let uri = uri.ok_or(RrdpError::PublishUriMissing)?; + ensure_rrdp_uri_can_be_owned_by(store, notification_uri, &uri) + .map_err(RrdpSyncError::Storage)?; + in_publish = true; + publish_nested_depth = 0; + current_publish_uri = Some(uri); + current_publish_text.clear(); + } + } + Ok(Event::Empty(e)) => { + let local_name = e.local_name(); + let local_name = local_name.as_ref(); + if !root_seen { + let got = String::from_utf8_lossy(local_name).to_string(); + return Err(RrdpError::UnexpectedRoot(got).into()); + } + if local_name == b"publish" { + let mut has_uri = false; + for attr in e.attributes().with_checks(false) { + let attr = match attr { + Ok(attr) => attr, + Err(e) => return Err(RrdpError::Xml(e.to_string()).into()), + }; + if attr.key.as_ref() == b"uri" { + has_uri = true; + break; + } + } + if !has_uri { + return Err(RrdpError::PublishUriMissing.into()); + } + return Err(RrdpError::PublishContentMissing.into()); + } + } + Ok(Event::Text(e)) => { + if in_publish && publish_nested_depth == 0 { + let text = reader + .decoder() + .decode(e.as_ref()) + .map_err(|e| RrdpError::Xml(e.to_string()))?; + current_publish_text.push_str(&text); + } + } + Ok(Event::CData(e)) => { + if in_publish && publish_nested_depth == 0 { + let text = reader + .decoder() + .decode(e.as_ref()) + .map_err(|e| RrdpError::Xml(e.to_string()))?; + current_publish_text.push_str(&text); + } + } + Ok(Event::End(e)) => { + let local_name = e.local_name(); + let local_name = local_name.as_ref(); + if in_publish { + if publish_nested_depth > 0 { + publish_nested_depth -= 1; + } else if local_name == b"publish" { + let uri = current_publish_uri + .take() + .ok_or_else(|| RrdpError::Xml("publish uri missing in state".into()))?; + let content_b64 = strip_all_ascii_whitespace(¤t_publish_text); + current_publish_text.clear(); + if content_b64.is_empty() { + return Err(RrdpError::PublishContentMissing.into()); + } + let bytes = base64::engine::general_purpose::STANDARD + .decode(content_b64.as_bytes()) + .map_err(|e| RrdpError::PublishBase64(e.to_string()))?; + new_set.insert(uri.clone()); + batch_published.push((uri, bytes)); + published_count += 1; + if batch_published.len() >= RRDP_SNAPSHOT_APPLY_BATCH_SIZE { + flush_snapshot_publish_batch( + store, + notification_uri, + current_repo_index, + &session_id, + expected_serial, + &batch_published, + )?; + batch_published.clear(); + } + in_publish = false; + } + } + } + Ok(Event::Eof) => break, + Ok(Event::Decl(_) | Event::PI(_) | Event::Comment(_) | Event::DocType(_)) => {} + Err(e) => return Err(RrdpError::Xml(e.to_string()).into()), + } + buf.clear(); + } + + if !root_seen { + return Err(RrdpError::Xml("missing root element".to_string()).into()); + } + if in_publish { + return Err(RrdpError::PublishContentMissing.into()); + } + if !batch_published.is_empty() { + flush_snapshot_publish_batch( + store, + notification_uri, + current_repo_index, + &session_id, + expected_serial, + &batch_published, + )?; + batch_published.clear(); + } + + let mut withdrawn: Vec<(String, Option)> = Vec::new(); + for old_uri in &previous_members { + if new_set.contains(old_uri) { + continue; + } + let previous_hash = store + .get_repository_view_entry(old_uri) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))? + .and_then(|entry| entry.current_hash) + .or_else(|| { + store + .load_current_object_bytes_by_uri(old_uri) + .ok() + .flatten() + .map(|bytes| compute_sha256_hex(&bytes)) + }); + withdrawn.push((old_uri.clone(), previous_hash)); + } + + let mut repository_view_entries = Vec::with_capacity(withdrawn.len()); + let mut member_records = Vec::with_capacity(withdrawn.len()); + let mut owner_records = Vec::with_capacity(withdrawn.len()); + for (uri, previous_hash) in withdrawn { + member_records.push(build_rrdp_source_member_withdrawn_record( + notification_uri, + &session_id, + expected_serial, + &uri, + previous_hash.clone(), + )); + if current_rrdp_owner_is(store, notification_uri, &uri).map_err(RrdpSyncError::Storage)? { + repository_view_entries.push(build_repository_view_withdrawn_entry( + notification_uri, + &uri, + previous_hash.clone(), + )); + owner_records.push(build_rrdp_uri_owner_withdrawn_record( + notification_uri, + &session_id, + expected_serial, + &uri, + previous_hash, + )); + } + } + store + .put_projection_batch(&repository_view_entries, &member_records, &owner_records) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; + if let Some(index) = current_repo_index { + index + .lock() + .map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))? + .apply_repository_view_entries(&repository_view_entries) + .map_err(RrdpSyncError::Storage)?; + } + + Ok(published_count) +} + +fn flush_snapshot_publish_batch( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + session_id: &str, + serial: u64, + published: &[(String, Vec)], +) -> Result<(), RrdpSyncError> { + let prepared_bytes = prepare_repo_bytes_batch(published).map_err(RrdpSyncError::Storage)?; + let mut repository_view_entries = Vec::with_capacity(published.len()); + let mut member_records = Vec::with_capacity(published.len()); + let mut owner_records = Vec::with_capacity(published.len()); + + for (uri, _bytes) in published { + let current_hash = prepared_bytes + .uri_to_hash + .get(uri) + .cloned() + .ok_or_else(|| { + RrdpSyncError::Storage(format!("missing raw_by_hash mapping for {uri}")) + })?; + repository_view_entries.push(build_repository_view_present_entry( + notification_uri, + uri, + ¤t_hash, + )); + member_records.push(build_rrdp_source_member_present_record( + notification_uri, + session_id, + serial, + uri, + ¤t_hash, + )); + owner_records.push(build_rrdp_uri_owner_active_record( + notification_uri, + session_id, + serial, + uri, + ¤t_hash, + )); + } + + store + .put_blob_bytes_batch(&prepared_bytes.blobs_to_write) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; + store + .put_projection_batch(&repository_view_entries, &member_records, &owner_records) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; + if let Some(index) = current_repo_index { + index + .lock() + .map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))? + .apply_repository_view_entries(&repository_view_entries) + .map_err(RrdpSyncError::Storage)?; + } + + Ok(()) +} + +const SNAPSHOT_NON_ASCII_ERROR: &str = "snapshot body contains non-ASCII bytes"; + +struct SnapshotSpoolWriter<'a, W: Write> { + inner: &'a mut W, + hasher: sha2::Sha256, + bytes: u64, +} + +impl<'a, W: Write> SnapshotSpoolWriter<'a, W> { + fn new(inner: &'a mut W) -> Self { + Self { + inner, + hasher: sha2::Sha256::new(), + bytes: 0, + } + } + + fn finalize_hash(self) -> [u8; 32] { + let digest = self.hasher.finalize(); + let mut out = [0u8; 32]; + out.copy_from_slice(&digest); + out + } +} + +impl Write for SnapshotSpoolWriter<'_, W> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + if buf.iter().any(|&b| b > 0x7F) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + SNAPSHOT_NON_ASCII_ERROR, + )); + } + let n = self.inner.write(buf)?; + self.hasher.update(&buf[..n]); + self.bytes += n as u64; + Ok(n) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + +pub(super) fn fetch_snapshot_into_tempfile( + fetcher: &dyn Fetcher, + snapshot_uri: &str, + expected_hash_sha256: &[u8; 32], +) -> Result<(tempfile::NamedTempFile, u64), RrdpSyncError> { + let mut tmp = tempfile::NamedTempFile::new() + .map_err(|e| RrdpSyncError::Fetch(format!("tempfile create failed: {e}")))?; + let mut spool = SnapshotSpoolWriter::new(tmp.as_file_mut()); + let bytes_written = match fetcher.fetch_to_writer(snapshot_uri, &mut spool) { + Ok(bytes) => bytes, + Err(e) if e.contains(SNAPSHOT_NON_ASCII_ERROR) => return Err(RrdpError::NotAscii.into()), + Err(e) => return Err(RrdpSyncError::Fetch(e)), + }; + let computed = spool.finalize_hash(); + if computed.as_slice() != expected_hash_sha256.as_slice() { + return Err(RrdpError::SnapshotHashMismatch.into()); + } + tmp.as_file_mut() + .flush() + .map_err(|e| RrdpSyncError::Fetch(format!("tempfile flush failed: {e}")))?; + tmp.as_file_mut() + .seek(SeekFrom::Start(0)) + .map_err(|e| RrdpSyncError::Fetch(format!("tempfile rewind failed: {e}")))?; + Ok((tmp, bytes_written)) +} diff --git a/src/sync/rrdp/tests.rs b/src/sync/rrdp/tests.rs new file mode 100644 index 0000000..5dea86f --- /dev/null +++ b/src/sync/rrdp/tests.rs @@ -0,0 +1,1316 @@ +use super::*; +use crate::analysis::timing::{TimingHandle, TimingMeta}; +use crate::current_repo_index::CurrentRepoIndex; +use crate::storage::RocksStore; +use std::collections::HashMap; +use std::io::Read; +use std::time::Duration; + +struct MapFetcher { + map: HashMap>, +} + +impl Fetcher for MapFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + self.map + .get(uri) + .cloned() + .ok_or_else(|| format!("not found: {uri}")) + } +} + +struct SleepyFetcher { + inner: MapFetcher, + sleep_uri: String, + sleep: Duration, +} + +impl Fetcher for SleepyFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + if uri == self.sleep_uri { + std::thread::sleep(self.sleep); + } + self.inner.fetch(uri) + } +} + +struct WriterOnlyFetcher { + map: HashMap>, +} + +impl Fetcher for WriterOnlyFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + Err(format!("unexpected buffered fetch: {uri}")) + } + + fn fetch_to_writer(&self, uri: &str, out: &mut dyn std::io::Write) -> Result { + let bytes = self + .map + .get(uri) + .ok_or_else(|| format!("not found: {uri}"))?; + out.write_all(bytes) + .map_err(|e| format!("write sink failed: {e}"))?; + Ok(bytes.len() as u64) + } +} + +struct NonAsciiWriterFetcher; + +impl Fetcher for NonAsciiWriterFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + Err(format!("unexpected buffered fetch: {uri}")) + } + + fn fetch_to_writer(&self, _uri: &str, out: &mut dyn std::io::Write) -> Result { + out.write_all(&[0x80]) + .map_err(|e| format!("write sink failed: {e}"))?; + Err("snapshot body contains non-ASCII bytes".to_string()) + } +} + +fn assert_current_object(store: &RocksStore, uri: &str, expected: &[u8]) { + assert_eq!( + store + .load_current_object_bytes_by_uri(uri) + .expect("load current object"), + Some(expected.to_vec()) + ); +} + +fn notification_xml( + session_id: &str, + serial: u64, + snapshot_uri: &str, + snapshot_hash: &str, +) -> Vec { + format!( + r#""# + ) + .into_bytes() +} + +fn notification_xml_with_deltas( + session_id: &str, + serial: u64, + snapshot_uri: &str, + snapshot_hash: &str, + deltas: &[(&str, u64, &str, &str)], +) -> Vec { + let mut out = format!( + r#""# + ); + for (_name, delta_serial, uri, hash) in deltas { + out.push_str(&format!( + r#""# + )); + } + out.push_str(""); + out.into_bytes() +} + +fn snapshot_xml(session_id: &str, serial: u64, published: &[(&str, &[u8])]) -> Vec { + let mut out = format!( + r#""# + ); + for (uri, bytes) in published { + let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); + out.push_str(&format!(r#"{b64}"#)); + } + out.push_str(""); + out.into_bytes() +} + +#[test] +fn fetch_snapshot_into_tempfile_streams_and_validates_hash() { + let snapshot_uri = "https://example.test/snapshot.xml"; + let snapshot = b"".to_vec(); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(&sha2::Sha256::digest(&snapshot)); + let fetcher = WriterOnlyFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot.clone())]), + }; + + let (mut file, bytes_written) = + fetch_snapshot_into_tempfile(&fetcher, snapshot_uri, &expected_hash) + .expect("fetch snapshot into tempfile"); + assert_eq!(bytes_written, snapshot.len() as u64); + let mut got = Vec::new(); + file.as_file_mut() + .read_to_end(&mut got) + .expect("read tempfile"); + assert_eq!(got, snapshot); + + let mut wrong_hash = expected_hash; + wrong_hash[0] ^= 0xff; + let err = fetch_snapshot_into_tempfile(&fetcher, snapshot_uri, &wrong_hash).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::SnapshotHashMismatch) + )); +} + +#[test] +fn fetch_snapshot_into_tempfile_maps_stream_non_ascii_error() { + let err = fetch_snapshot_into_tempfile( + &NonAsciiWriterFetcher, + "https://example.test/snapshot.xml", + &[0u8; 32], + ) + .unwrap_err(); + assert!(matches!(err, RrdpSyncError::Rrdp(RrdpError::NotAscii))); +} + +#[test] +fn timing_rrdp_repo_step_spans_cover_snapshot_fetch_duration() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let notification_uri = "https://example.test/notification.xml"; + let snapshot_uri = "https://example.test/snapshot.xml"; + let published_uri = "rsync://example.test/repo/a.mft"; + let published_bytes = b"x"; + let session_id = "550e8400-e29b-41d4-a716-446655440000"; + + let snapshot = snapshot_xml(session_id, 1, &[(published_uri, published_bytes)]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(session_id, 1, snapshot_uri, &snapshot_hash); + + let mut map = HashMap::new(); + map.insert(snapshot_uri.to_string(), snapshot); + let fetcher = SleepyFetcher { + inner: MapFetcher { map }, + sleep_uri: snapshot_uri.to_string(), + sleep: Duration::from_millis(25), + }; + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + tal_url: None, + db_path: Some(store_dir.to_string_lossy().into_owned()), + }); + + sync_from_notification_snapshot_with_timing( + &store, + notification_uri, + ¬if, + &fetcher, + Some(&timing), + ) + .expect("rrdp snapshot sync ok"); + + let timing_path = temp.path().join("timing.json"); + timing.write_json(&timing_path, 200).expect("write timing"); + let rep: serde_json::Value = + serde_json::from_slice(&std::fs::read(&timing_path).expect("read timing")) + .expect("parse timing"); + + let want = format!("{notification_uri}::fetch_snapshot"); + let steps = rep + .get("top_rrdp_repo_steps") + .and_then(|v| v.as_array()) + .expect("top_rrdp_repo_steps array"); + let entry = steps + .iter() + .find(|e| e.get("key").and_then(|k| k.as_str()) == Some(want.as_str())) + .unwrap_or_else(|| panic!("missing timing step entry for {want}")); + let nanos = entry + .get("total_nanos") + .and_then(|v| v.as_u64()) + .expect("total_nanos"); + assert!( + nanos >= 20_000_000, + "expected fetch_snapshot timing to include the fetch duration; got {nanos}ns" + ); +} + +#[test] +fn parse_notification_snapshot_rejects_non_ascii() { + let mut xml = b"".to_vec(); + xml.push(0x80); + let err = parse_notification_snapshot(&xml).unwrap_err(); + assert!(matches!(err, RrdpError::NotAscii)); +} + +#[test] +fn parse_notification_snapshot_parses_valid_minimal_notification() { + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let snapshot_uri = "https://example.net/snapshot.xml"; + let hash = "00".repeat(32); + let xml = notification_xml(sid, 7, snapshot_uri, &hash); + let n = parse_notification_snapshot(&xml).expect("parse"); + assert_eq!(n.session_id, Uuid::parse_str(sid).unwrap()); + assert_eq!(n.serial, 7); + assert_eq!(n.snapshot_uri, snapshot_uri); + assert_eq!(hex::encode(n.snapshot_hash_sha256), hash); +} + +#[test] +fn parse_notification_parses_deltas_and_validates_contiguity() { + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let snapshot_uri = "https://example.net/snapshot.xml"; + let hash = "00".repeat(32); + let d_hash_2 = "11".repeat(32); + let d_hash_3 = "22".repeat(32); + // Provide deltas in reverse order to ensure we sort. + let xml = notification_xml_with_deltas( + sid, + 3, + snapshot_uri, + &hash, + &[ + ("d3", 3, "https://example.net/delta-3.xml", &d_hash_3), + ("d2", 2, "https://example.net/delta-2.xml", &d_hash_2), + ], + ); + let n = parse_notification(&xml).expect("parse notification"); + assert_eq!(n.serial, 3); + assert_eq!(n.deltas.len(), 2); + assert_eq!(n.deltas[0].serial, 2); + assert_eq!(n.deltas[1].serial, 3); + assert_eq!(n.deltas[0].uri, "https://example.net/delta-2.xml"); + assert_eq!(hex::encode(n.deltas[1].hash_sha256), d_hash_3); +} + +#[test] +fn parse_notification_rejects_non_contiguous_deltas() { + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let snapshot_uri = "https://example.net/snapshot.xml"; + let hash = "00".repeat(32); + let d_hash_1 = "11".repeat(32); + let d_hash_3 = "22".repeat(32); + // Missing delta serial 2. + let xml = notification_xml_with_deltas( + sid, + 3, + snapshot_uri, + &hash, + &[ + ("d3", 3, "https://example.net/delta-3.xml", &d_hash_3), + ("d1", 1, "https://example.net/delta-1.xml", &d_hash_1), + ], + ); + let err = parse_notification(&xml).unwrap_err(); + assert!(matches!(err, RrdpError::DeltaRefChainNotContiguous { .. })); +} + +fn delta_xml(session_id: &str, serial: u64, elements: &[&str]) -> Vec { + let mut out = format!( + r#""# + ); + for e in elements { + out.push_str(e); + } + out.push_str(""); + out.into_bytes() +} + +#[test] +fn parse_delta_file_parses_publish_and_withdraw() { + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let serial = 3u64; + let publish_bytes = b"abc"; + let publish_b64 = base64::engine::general_purpose::STANDARD.encode(publish_bytes); + let withdraw_hash = "33".repeat(32); + + let xml = delta_xml( + sid, + serial, + &[ + &format!(r#"{publish_b64}"#), + &format!(r#""#), + ], + ); + + let d = parse_delta_file(&xml).expect("parse delta"); + assert_eq!(d.session_id, Uuid::parse_str(sid).unwrap()); + assert_eq!(d.serial, serial); + assert_eq!(d.elements.len(), 2); + match &d.elements[0] { + DeltaElement::Publish { + uri, + hash_sha256, + bytes, + } => { + assert_eq!(uri, "rsync://example.net/repo/a.mft"); + assert_eq!(*hash_sha256, None); + assert_eq!(bytes, publish_bytes); + } + _ => panic!("expected publish"), + } + match &d.elements[1] { + DeltaElement::Withdraw { uri, hash_sha256 } => { + assert_eq!(uri, "rsync://example.net/repo/b.cer"); + assert_eq!(hex::encode(hash_sha256), withdraw_hash); + } + _ => panic!("expected withdraw"), + } +} + +#[test] +fn parse_delta_file_rejects_withdraw_with_content() { + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let serial = 1u64; + let withdraw_hash = "33".repeat(32); + let xml = delta_xml( + sid, + serial, + &[&format!( + r#"AA=="# + )], + ); + let err = parse_delta_file(&xml).unwrap_err(); + assert!(matches!(err, RrdpError::DeltaWithdrawUnexpectedContent)); +} + +#[test] +fn apply_delta_applies_publish_replace_and_withdraw_with_membership_checks() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // Start from snapshot state with a + b + let snapshot_uri = "https://example.net/snapshot.xml"; + let snapshot = snapshot_xml( + sid, + 1, + &[ + ("rsync://example.net/repo/a.mft", b"a1"), + ("rsync://example.net/repo/b.roa", b"b1"), + ], + ); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("sync snapshot"); + + let old_b = store + .load_current_object_bytes_by_uri("rsync://example.net/repo/b.roa") + .expect("load current b") + .expect("b present"); + let old_b_hash = hex::encode(sha2::Sha256::digest(old_b.as_slice())); + + let withdraw_a_hash = hex::encode(sha2::Sha256::digest(b"a1".as_slice())); + let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2"); + let replace_b_b64 = base64::engine::general_purpose::STANDARD.encode(b"b2"); + + let delta = delta_xml( + sid, + 2, + &[ + &format!( + r#""# + ), + &format!( + r#"{replace_b_b64}"# + ), + &format!(r#"{publish_c_b64}"#), + ], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + + let applied = apply_delta( + &store, + notif_uri, + None, + &delta, + expected_hash, + Uuid::parse_str(sid).unwrap(), + 2, + ) + .expect("apply delta"); + assert_eq!(applied, 3); + + assert_eq!( + store + .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") + .expect("load current a"), + None, + "a withdrawn" + ); + let b = store + .load_current_object_bytes_by_uri("rsync://example.net/repo/b.roa") + .expect("load current b") + .expect("b present"); + assert_eq!(b, b"b2"); + let c = store + .load_current_object_bytes_by_uri("rsync://example.net/repo/c.crl") + .expect("load current c") + .expect("c present"); + assert_eq!(c, b"c2"); + + assert!( + !store + .is_current_rrdp_source_member(notif_uri, "rsync://example.net/repo/a.mft") + .expect("is member"), + "a removed from rrdp repo index" + ); + assert!( + store + .is_current_rrdp_source_member(notif_uri, "rsync://example.net/repo/c.crl") + .expect("is member"), + "c added to rrdp repo index" + ); + + let a_view = store + .get_repository_view_entry("rsync://example.net/repo/a.mft") + .expect("get a view") + .expect("a view exists"); + assert_eq!(a_view.state, crate::storage::RepositoryViewState::Withdrawn); + let b_view = store + .get_repository_view_entry("rsync://example.net/repo/b.roa") + .expect("get b view") + .expect("b view exists"); + assert_eq!(b_view.state, crate::storage::RepositoryViewState::Present); + assert_eq!( + b_view.current_hash.as_deref(), + Some(hex::encode(sha2::Sha256::digest(b"b2")).as_str()) + ); + let c_owner = store + .get_rrdp_uri_owner_record("rsync://example.net/repo/c.crl") + .expect("get c owner") + .expect("c owner exists"); + assert_eq!( + c_owner.owner_state, + crate::storage::RrdpUriOwnerState::Active + ); + let a_member = store + .get_rrdp_source_member_record(notif_uri, "rsync://example.net/repo/a.mft") + .expect("get a member") + .expect("a member exists"); + assert!(!a_member.present); + let current_members = store + .list_current_rrdp_source_members(notif_uri) + .expect("list current members"); + assert_eq!( + current_members + .iter() + .map(|record| record.rsync_uri.as_str()) + .collect::>(), + vec![ + "rsync://example.net/repo/b.roa", + "rsync://example.net/repo/c.crl", + ] + ); +} + +#[test] +fn apply_delta_rejects_hash_mismatch() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + let notif_uri = "https://example.net/notification.xml"; + + let delta = delta_xml( + sid.to_string().as_str(), + 1, + &[r#"QQ=="#], + ); + let mut wrong = [0u8; 32]; + wrong[0] = 1; + let err = apply_delta(&store, notif_uri, None, &delta, wrong, sid, 1).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaHashMismatch) + )); +} + +#[test] +fn apply_delta_rejects_withdraw_of_non_member() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + let notif_uri = "https://example.net/notification.xml"; + + let withdraw_hash = "00".repeat(32); + let delta = delta_xml( + sid.to_string().as_str(), + 1, + &[&format!( + r#""# + )], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + + let err = apply_delta(&store, notif_uri, None, &delta, expected_hash, sid, 1).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaTargetNotFromRepository { .. }) + )); +} + +#[test] +fn apply_delta_rejects_publish_without_hash_for_existing_object() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // Seed snapshot with a.mft. + let snapshot_uri = "https://example.net/snapshot.xml"; + let snapshot = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("seed"); + + // Replace publish for an existing URI must have @hash. + let publish_b64 = base64::engine::general_purpose::STANDARD.encode(b"a2"); + let delta = delta_xml( + sid, + 2, + &[&format!( + r#"{publish_b64}"# + )], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + + let err = apply_delta( + &store, + notif_uri, + None, + &delta, + expected_hash, + Uuid::parse_str(sid).unwrap(), + 2, + ) + .unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaPublishWithoutHashForExisting { .. }) + )); +} + +#[test] +fn apply_delta_rejects_target_missing_and_hash_mismatch() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // Seed snapshot with a.mft. + let snapshot_uri = "https://example.net/snapshot.xml"; + let snapshot = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("seed"); + + let old_bytes = store + .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") + .expect("get") + .expect("present"); + let old_hash = hex::encode(sha2::Sha256::digest(old_bytes.as_slice())); + + // Hash mismatch on withdraw. + let wrong_hash = "11".repeat(32); + let delta = delta_xml( + sid, + 2, + &[&format!( + r#""# + )], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + let err = apply_delta( + &store, + notif_uri, + None, + &delta, + expected_hash, + Uuid::parse_str(sid).unwrap(), + 2, + ) + .unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaTargetHashMismatch { .. }) + )); + + // Target missing in local cache (index still says it's a member). + store + .delete_repository_view_entry("rsync://example.net/repo/a.mft") + .expect("delete current repository view entry"); + let delta = delta_xml( + sid, + 2, + &[&format!( + r#""# + )], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + let err = apply_delta( + &store, + notif_uri, + None, + &delta, + expected_hash, + Uuid::parse_str(sid).unwrap(), + 2, + ) + .unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaTargetMissing { .. }) + )); +} + +#[test] +fn apply_delta_rejects_session_and_serial_mismatch() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + let publish_b64 = base64::engine::general_purpose::STANDARD.encode(b"x"); + let delta = delta_xml( + sid, + 2, + &[&format!( + r#"{publish_b64}"# + )], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + + // Session mismatch. + let other_sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440001").unwrap(); + let err = + apply_delta(&store, notif_uri, None, &delta, expected_hash, other_sid, 2).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaSessionIdMismatch { .. }) + )); + + // Serial mismatch. + let err = apply_delta( + &store, + notif_uri, + None, + &delta, + expected_hash, + Uuid::parse_str(sid).unwrap(), + 3, + ) + .unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaSerialMismatch { .. }) + )); +} + +#[test] +fn sync_from_notification_snapshot_rejects_cross_source_owner_conflict() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid_a = "550e8400-e29b-41d4-a716-446655440000"; + let sid_b = "550e8400-e29b-41d4-a716-446655440001"; + let uri = "rsync://example.net/repo/a.mft"; + + let notif_a_uri = "https://example.net/a/notification.xml"; + let snapshot_a_uri = "https://example.net/a/snapshot.xml"; + let snapshot_a = snapshot_xml(sid_a, 1, &[(uri, b"a1")]); + let snapshot_a_hash = hex::encode(sha2::Sha256::digest(&snapshot_a)); + let notif_a = notification_xml(sid_a, 1, snapshot_a_uri, &snapshot_a_hash); + let fetcher_a = MapFetcher { + map: HashMap::from([(snapshot_a_uri.to_string(), snapshot_a)]), + }; + sync_from_notification_snapshot(&store, notif_a_uri, ¬if_a, &fetcher_a) + .expect("seed source a"); + + let notif_b_uri = "https://example.net/b/notification.xml"; + let snapshot_b_uri = "https://example.net/b/snapshot.xml"; + let snapshot_b = snapshot_xml(sid_b, 1, &[(uri, b"b1")]); + let snapshot_b_hash = hex::encode(sha2::Sha256::digest(&snapshot_b)); + let notif_b = notification_xml(sid_b, 1, snapshot_b_uri, &snapshot_b_hash); + let fetcher_b = MapFetcher { + map: HashMap::from([(snapshot_b_uri.to_string(), snapshot_b)]), + }; + + let err = sync_from_notification_snapshot(&store, notif_b_uri, ¬if_b, &fetcher_b) + .expect_err("cross-source overwrite must fail"); + assert!(matches!(err, RrdpSyncError::Storage(_))); + assert!(err.to_string().contains("owner conflict"), "{err}"); +} + +#[test] +fn sync_from_notification_snapshot_applies_snapshot_and_stores_state() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let serial = 9u64; + let notif_uri = "https://example.net/notification.xml"; + let snapshot_uri = "https://example.net/snapshot.xml"; + + let snapshot = snapshot_xml( + sid, + serial, + &[ + ("rsync://example.net/repo/a.mft", b"mft-bytes"), + ("rsync://example.net/repo/b.roa", b"roa-bytes"), + ], + ); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(sid, serial, snapshot_uri, &snapshot_hash); + + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot.clone())]), + }; + + let published = + sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("sync"); + assert_eq!(published, 2); + + assert_current_object(&store, "rsync://example.net/repo/a.mft", b"mft-bytes"); + assert_current_object(&store, "rsync://example.net/repo/b.roa", b"roa-bytes"); + + let state = load_rrdp_local_state(&store, notif_uri) + .expect("get rrdp state") + .expect("state present"); + assert_eq!(state.session_id, sid); + assert_eq!(state.serial, serial); + + let source = store + .get_rrdp_source_record(notif_uri) + .expect("get rrdp source") + .expect("rrdp source exists"); + assert_eq!(source.last_session_id.as_deref(), Some(sid)); + assert_eq!(source.last_serial, Some(serial)); + assert_eq!( + source.sync_state, + crate::storage::RrdpSourceSyncState::SnapshotOnly + ); + + let view = store + .get_repository_view_entry("rsync://example.net/repo/a.mft") + .expect("get repository view") + .expect("repository view exists"); + assert_eq!(view.state, crate::storage::RepositoryViewState::Present); + assert_eq!(view.repository_source.as_deref(), Some(notif_uri)); + + let current_bytes = store + .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") + .expect("load current bytes") + .expect("current object bytes exist"); + assert_eq!(current_bytes, b"mft-bytes".to_vec()); + assert!( + store + .get_raw_by_hash_entry(hex::encode(sha2::Sha256::digest(b"mft-bytes")).as_str()) + .expect("get raw_by_hash") + .is_none() + ); + + let member = store + .get_rrdp_source_member_record(notif_uri, "rsync://example.net/repo/a.mft") + .expect("get member") + .expect("member exists"); + assert!(member.present); + let owner = store + .get_rrdp_uri_owner_record("rsync://example.net/repo/a.mft") + .expect("get owner") + .expect("owner exists"); + assert_eq!(owner.notify_uri, notif_uri); + assert_eq!(owner.owner_state, crate::storage::RrdpUriOwnerState::Active); +} + +#[test] +fn sync_from_notification_snapshot_deletes_objects_not_in_new_snapshot() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // serial 1: publish a + b + let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; + let snapshot_1 = snapshot_xml( + sid, + 1, + &[ + ("rsync://example.net/repo/a.mft", b"a1"), + ("rsync://example.net/repo/b.roa", b"b1"), + ], + ); + let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); + let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); + + let fetcher_1 = MapFetcher { + map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("sync 1"); + + // serial 2: publish b (new bytes) + c, and drop a + let snapshot_uri_2 = "https://example.net/snapshot-2.xml"; + let snapshot_2 = snapshot_xml( + sid, + 2, + &[ + ("rsync://example.net/repo/b.roa", b"b2"), + ("rsync://example.net/repo/c.crl", b"c2"), + ], + ); + let snapshot_hash_2 = hex::encode(sha2::Sha256::digest(&snapshot_2)); + let notif_2 = notification_xml(sid, 2, snapshot_uri_2, &snapshot_hash_2); + + let fetcher_2 = MapFetcher { + map: HashMap::from([(snapshot_uri_2.to_string(), snapshot_2)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if_2, &fetcher_2).expect("sync 2"); + + assert!( + store + .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") + .expect("get current a") + .is_none(), + "a should be deleted by full-state snapshot apply" + ); + + assert_current_object(&store, "rsync://example.net/repo/b.roa", b"b2"); + assert_current_object(&store, "rsync://example.net/repo/c.crl", b"c2"); +} + +#[test] +fn sync_from_notification_uses_deltas_when_available_for_local_state() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // Seed state with snapshot serial=1 containing a+b. + let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; + let snapshot_1 = snapshot_xml( + sid, + 1, + &[ + ("rsync://example.net/repo/a.mft", b"a1"), + ("rsync://example.net/repo/b.roa", b"b1"), + ], + ); + let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); + let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); + let fetcher_1 = MapFetcher { + map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed"); + + // Notification serial=3 with deltas 2 and 3. Snapshot URI is intentionally not fetchable + // to assert we really use deltas. + let snapshot_uri_3 = "https://example.net/snapshot-3.xml"; + let snapshot_hash_3 = "00".repeat(32); + + let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2"); + let delta_2 = delta_xml( + sid, + 2, + &[&format!( + r#"{publish_c_b64}"# + )], + ); + let delta_2_hash_hex = hex::encode(sha2::Sha256::digest(&delta_2)); + + let c_hash_hex = hex::encode(sha2::Sha256::digest(b"c2".as_slice())); + let delta_3 = delta_xml( + sid, + 3, + &[&format!( + r#""# + )], + ); + let delta_3_hash_hex = hex::encode(sha2::Sha256::digest(&delta_3)); + + let notif_3 = notification_xml_with_deltas( + sid, + 3, + snapshot_uri_3, + &snapshot_hash_3, + &[ + ( + "d3", + 3, + "https://example.net/delta-3.xml", + &delta_3_hash_hex, + ), + ( + "d2", + 2, + "https://example.net/delta-2.xml", + &delta_2_hash_hex, + ), + ], + ); + + let fetcher = MapFetcher { + map: HashMap::from([ + ("https://example.net/delta-2.xml".to_string(), delta_2), + ("https://example.net/delta-3.xml".to_string(), delta_3), + ]), + }; + + let applied = sync_from_notification(&store, notif_uri, ¬if_3, &fetcher).expect("sync"); + assert!(applied > 0); + + // Delta 2 publishes c then delta 3 withdraws it => final state should not contain c. + assert!( + store + .load_current_object_bytes_by_uri("rsync://example.net/repo/c.crl") + .expect("get current") + .is_none() + ); + + let state = load_rrdp_local_state(&store, notif_uri) + .expect("get rrdp state") + .expect("state present"); + assert_eq!(state.session_id, Uuid::parse_str(sid).unwrap().to_string()); + assert_eq!(state.serial, 3); +} + +#[test] +fn sync_from_notification_same_serial_hydrates_current_repo_index() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + let snapshot_uri = "https://example.net/snapshot.xml"; + let uri_a = "rsync://example.net/repo/a.mft"; + let uri_b = "rsync://example.net/repo/b.roa"; + + let snapshot = snapshot_xml(sid, 1, &[(uri_a, b"a1"), (uri_b, b"b1")]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); + let fetcher_1 = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher_1).expect("seed"); + + let index = CurrentRepoIndex::shared(); + let no_fetcher = MapFetcher { + map: HashMap::new(), + }; + let applied = sync_from_notification_with_timing_and_download_log( + &store, + notif_uri, + Some(&index), + ¬if, + &no_fetcher, + None, + None, + ) + .expect("same serial no-op"); + assert_eq!(applied, 0); + + let index = index.lock().expect("lock index"); + assert_eq!(index.active_uri_count(), 2); + assert!(index.get_by_uri(uri_a).is_some()); + assert!(index.get_by_uri(uri_b).is_some()); +} + +#[test] +fn sync_from_notification_delta_hydrates_unchanged_current_repo_entries() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; + let uri_a = "rsync://example.net/repo/a.mft"; + let uri_b = "rsync://example.net/repo/b.roa"; + let uri_c = "rsync://example.net/repo/c.crl"; + + let snapshot_1 = snapshot_xml(sid, 1, &[(uri_a, b"a1"), (uri_b, b"b1")]); + let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); + let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); + let fetcher_1 = MapFetcher { + map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed"); + + let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2"); + let delta_2 = delta_xml( + sid, + 2, + &[&format!( + r#"{publish_c_b64}"# + )], + ); + let delta_2_hash_hex = hex::encode(sha2::Sha256::digest(&delta_2)); + let notif_2 = notification_xml_with_deltas( + sid, + 2, + "https://example.net/snapshot-2.xml", + &"00".repeat(32), + &[( + "d2", + 2, + "https://example.net/delta-2.xml", + &delta_2_hash_hex, + )], + ); + let fetcher_2 = MapFetcher { + map: HashMap::from([("https://example.net/delta-2.xml".to_string(), delta_2)]), + }; + + let index = CurrentRepoIndex::shared(); + let applied = sync_from_notification_with_timing_and_download_log( + &store, + notif_uri, + Some(&index), + ¬if_2, + &fetcher_2, + None, + None, + ) + .expect("delta sync"); + assert_eq!(applied, 1); + + let index = index.lock().expect("lock index"); + assert_eq!(index.active_uri_count(), 3); + assert!( + index.get_by_uri(uri_a).is_some(), + "unchanged object from the previous serial must be visible" + ); + assert!( + index.get_by_uri(uri_b).is_some(), + "unchanged object from the previous serial must be visible" + ); + assert!(index.get_by_uri(uri_c).is_some(), "delta publish visible"); +} + +#[test] +fn load_rrdp_local_state_uses_source_record_only() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let notif_uri = "https://example.net/notification.xml"; + assert_eq!( + load_rrdp_local_state(&store, notif_uri).expect("load empty"), + None + ); + + update_rrdp_source_record_on_success( + &store, + notif_uri, + "source-session", + 9, + crate::storage::RrdpSourceSyncState::DeltaReady, + Some("https://example.net/snapshot.xml"), + Some(&hex::encode([0x11; 32])), + ) + .expect("write source record"); + + let got = load_rrdp_local_state(&store, notif_uri) + .expect("load source preferred") + .expect("source present"); + assert_eq!( + got, + RrdpState { + session_id: "source-session".to_string(), + serial: 9, + } + ); +} + +#[test] +fn sync_from_notification_falls_back_to_snapshot_if_missing_required_deltas() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // Seed state serial=1 with a only. + let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; + let snapshot_1 = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]); + let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); + let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); + let fetcher_1 = MapFetcher { + map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed"); + + // Notification serial=3 only includes delta serial=3 (contiguous per RFC, but missing + // serial=2 relative to our local state, so we must use snapshot). + let snapshot_uri_3 = "https://example.net/snapshot-3.xml"; + let snapshot_3 = snapshot_xml(sid, 3, &[("rsync://example.net/repo/z.roa", b"z3")]); + let snapshot_hash_3 = hex::encode(sha2::Sha256::digest(&snapshot_3)); + let delta_3_hash_hex = "11".repeat(32); + let notif_3 = notification_xml_with_deltas( + sid, + 3, + snapshot_uri_3, + &snapshot_hash_3, + &[( + "d3", + 3, + "https://example.net/delta-3.xml", + &delta_3_hash_hex, + )], + ); + + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri_3.to_string(), snapshot_3)]), + }; + + let published = sync_from_notification(&store, notif_uri, ¬if_3, &fetcher).expect("sync"); + assert_eq!(published, 1); + assert!( + store + .load_current_object_bytes_by_uri("rsync://example.net/repo/z.roa") + .expect("get current") + .is_some() + ); +} + +#[test] +fn sync_from_notification_snapshot_rejects_snapshot_hash_mismatch() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let serial = 1u64; + let notif_uri = "https://example.net/notification.xml"; + let snapshot_uri = "https://example.net/snapshot.xml"; + + let snapshot = snapshot_xml(sid, serial, &[("rsync://example.net/repo/a.mft", b"x")]); + let notif = notification_xml(sid, serial, snapshot_uri, &"00".repeat(32)); + + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), + }; + let err = sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::SnapshotHashMismatch) + )); +} + +#[test] +fn apply_snapshot_rejects_session_id_and_serial_mismatch() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let notif_uri = "https://example.net/notification.xml"; + + let expected_sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + let got_sid = "550e8400-e29b-41d4-a716-446655440001"; + + let snapshot = snapshot_xml(got_sid, 2, &[("rsync://example.net/repo/a.mft", b"x")]); + let err = apply_snapshot(&store, notif_uri, None, &snapshot, expected_sid, 2).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::SnapshotSessionIdMismatch { .. }) + )); + + let snapshot = snapshot_xml( + expected_sid.to_string().as_str(), + 3, + &[("rsync://example.net/repo/a.mft", b"x")], + ); + let err = apply_snapshot(&store, notif_uri, None, &snapshot, expected_sid, 2).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::SnapshotSerialMismatch { .. }) + )); +} + +#[test] +fn strip_all_ascii_whitespace_removes_newlines_and_spaces() { + assert_eq!(strip_all_ascii_whitespace(" a \n b\tc "), "abc"); +} + +#[test] +fn apply_snapshot_reports_publish_errors() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let notif_uri = "https://example.net/notification.xml"; + let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + + // Missing publish/@uri + let xml = format!( + r#"AA=="# + ) + .into_bytes(); + let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::PublishUriMissing) + )); + + // Missing base64 content (no text nodes). + let xml = format!( + r#""# + ) + .into_bytes(); + let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::PublishContentMissing) + )); + + // Invalid base64 content. + let xml = format!( + r#"!!!"# + ) + .into_bytes(); + let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::PublishBase64(_)) + )); +} + +#[test] +fn apply_snapshot_handles_multiple_publish_batches() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let notif_uri = "https://example.net/notification.xml"; + let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + + let total = RRDP_SNAPSHOT_APPLY_BATCH_SIZE + 7; + let mut xml = + format!(r#""#); + for i in 0..total { + let uri = format!("rsync://example.net/repo/{i:04}.roa"); + let bytes = format!("payload-{i}").into_bytes(); + let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); + xml.push_str(&format!(r#"{b64}"#)); + } + xml.push_str(""); + + let published = + apply_snapshot(&store, notif_uri, None, xml.as_bytes(), sid, 1).expect("apply snapshot"); + assert_eq!(published, total); + + for idx in [0usize, RRDP_SNAPSHOT_APPLY_BATCH_SIZE - 1, total - 1] { + let uri = format!("rsync://example.net/repo/{idx:04}.roa"); + let got = store + .load_current_object_bytes_by_uri(&uri) + .expect("load object") + .expect("object exists"); + assert_eq!(got, format!("payload-{idx}").into_bytes()); + } +} diff --git a/src/tools/mod.rs b/src/tools/mod.rs new file mode 100644 index 0000000..c68c8fc --- /dev/null +++ b/src/tools/mod.rs @@ -0,0 +1,3 @@ +pub mod rpki_artifact_metrics; +pub mod rpki_daemon; +pub mod sequence_triage_ccr_cir; diff --git a/src/tools/rpki_artifact_metrics.rs b/src/tools/rpki_artifact_metrics.rs new file mode 100644 index 0000000..5ed9b26 --- /dev/null +++ b/src/tools/rpki_artifact_metrics.rs @@ -0,0 +1,2111 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, RwLock}; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use crate::ccr::decode_content_info; +use crate::cir::decode_cir; +use serde::Serialize; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +const LARGE_PP_OBJECT_THRESHOLDS: &[u64] = &[10, 50, 100, 500, 1000, 5000, 10000, 50000]; +const PP_SYNC_SECONDS_BUCKETS: &[f64] = &[0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0]; + +#[derive(Clone, Debug, PartialEq, Eq)] +struct Args { + run_root: PathBuf, + listen: String, + poll_secs: u64, + instance: String, + once: bool, + out_metrics: Option, + out_status: Option, +} + +fn usage() -> &'static str { + "Usage: rpki_artifact_metrics --run-root [--listen ] [--poll-secs ] [--instance ] [--once] [--out-metrics ] [--out-status ]" +} + +pub fn main_entry() -> Result<(), String> { + real_main() +} + +fn real_main() -> Result<(), String> { + let args = parse_args(&std::env::args().collect::>())?; + if args.once { + let snapshot = scan_run_root(&args.run_root, &args.instance)?; + let metrics = render_metrics(&snapshot); + let status = render_status_json(&snapshot)?; + if let Some(path) = args.out_metrics.as_ref() { + write_file(path, metrics.as_bytes())?; + } else { + print!("{metrics}"); + } + if let Some(path) = args.out_status.as_ref() { + write_file(path, status.as_bytes())?; + } + return Ok(()); + } + + let shared = Arc::new(RwLock::new(scan_run_root(&args.run_root, &args.instance)?)); + let scanner = Arc::clone(&shared); + let run_root = args.run_root.clone(); + let instance = args.instance.clone(); + let poll_secs = args.poll_secs.max(1); + thread::spawn(move || { + loop { + thread::sleep(Duration::from_secs(poll_secs)); + let next = match scan_run_root(&run_root, &instance) { + Ok(snapshot) => snapshot, + Err(err) => { + let mut previous = scanner.write().expect("metrics lock poisoned"); + previous + .service + .parse_errors + .push(format!("scan failed: {err}")); + previous.service.last_reload_success = false; + previous.service.last_scan_timestamp_seconds = unix_now_seconds(); + continue; + } + }; + *scanner.write().expect("metrics lock poisoned") = next; + } + }); + + serve_http(&args.listen, shared) +} + +fn parse_args(argv: &[String]) -> Result { + let mut run_root = None; + let mut listen = "127.0.0.1:9556".to_string(); + let mut poll_secs = 10u64; + let mut instance = "ours-rp".to_string(); + let mut once = false; + let mut out_metrics = None; + let mut out_status = None; + let mut index = 1usize; + while index < argv.len() { + match argv[index].as_str() { + "--run-root" => { + index += 1; + run_root = Some(PathBuf::from(value_at(argv, index, "--run-root")?)); + } + "--listen" => { + index += 1; + listen = value_at(argv, index, "--listen")?.to_string(); + } + "--poll-secs" => { + index += 1; + let value = value_at(argv, index, "--poll-secs")?; + poll_secs = value + .parse::() + .map_err(|_| format!("invalid --poll-secs: {value}"))?; + } + "--instance" => { + index += 1; + instance = value_at(argv, index, "--instance")?.to_string(); + } + "--once" => once = true, + "--out-metrics" => { + index += 1; + out_metrics = Some(PathBuf::from(value_at(argv, index, "--out-metrics")?)); + } + "--out-status" => { + index += 1; + out_status = Some(PathBuf::from(value_at(argv, index, "--out-status")?)); + } + "-h" | "--help" => return Err(usage().to_string()), + other => return Err(format!("unknown argument: {other}\n{}", usage())), + } + index += 1; + } + Ok(Args { + run_root: run_root.ok_or_else(|| format!("--run-root is required\n{}", usage()))?, + listen, + poll_secs, + instance, + once, + out_metrics, + out_status, + }) +} + +fn value_at<'a>(argv: &'a [String], index: usize, flag: &str) -> Result<&'a str, String> { + argv.get(index) + .map(|s| s.as_str()) + .ok_or_else(|| format!("{flag} requires a value")) +} + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct MetricsSnapshot { + instance: String, + service: ServiceMetrics, + runs: RunScanSummary, + latest_run: Option, + cumulative: CumulativeMetrics, + repo_stats: Vec, + object_counts: BTreeMap<(String, String), u64>, + large_pp_counts: BTreeMap, + pp_sync_histograms: BTreeMap, + top_repos_by_sync_duration: Vec, + top_pp_by_object_count: Vec, + top_pp_by_sync_duration: Vec, + cir: Option, + ccr: Option, +} + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct ServiceMetrics { + last_scan_timestamp_seconds: f64, + last_scan_duration_seconds: f64, + last_reload_success: bool, + parse_errors: Vec, + run_root: String, + runs_root: String, +} + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct RunScanSummary { + known: u64, + success: u64, + failed: u64, + partial: u64, + consecutive_failures: u64, +} + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct CumulativeMetrics { + completed_success_total: u64, + completed_failed_total: u64, + observed_duration_seconds_sum: f64, + observed_duration_seconds_count: u64, + observed_download_bytes_total: u64, +} + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct LatestRunMetrics { + run_seq: u64, + run_id: String, + run_dir: String, + status: String, + sync_mode: String, + snapshot_reason: Option, + started_at: Option, + finished_at: Option, + start_timestamp_seconds: Option, + finish_timestamp_seconds: Option, + wall_seconds: f64, + user_cpu_seconds: Option, + system_cpu_seconds: Option, + cpu_percent: Option, + max_rss_bytes: Option, + exit_code: Option, + vrps: u64, + vaps: u64, + publication_points: u64, + warnings: u64, + tree_instances_processed: Option, + tree_instances_failed: Option, + stage_seconds: BTreeMap, + repo_sync_phase: BTreeMap, + repo_terminal_state: BTreeMap, + download_events: Option, + download_bytes: Option, + artifact_sizes: BTreeMap, + state_path_sizes: BTreeMap, +} + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct CountDuration { + count: u64, + duration_seconds_total: f64, +} + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct PathSize { + total_size_bytes: u64, + file_count: u64, + dir_count: u64, +} + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct RepoMetrics { + repo_id: String, + uri: String, + host: String, + transport: String, + publication_points: u64, + sync_success: bool, + download_bytes: u64, + duration_seconds_sum: f64, + duration_seconds_max: f64, + duration_seconds_avg: f64, + phase_counts: BTreeMap, + terminal_state_counts: BTreeMap, +} + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct TopRepo { + rank: usize, + repo_id: String, + uri: String, + host: String, + transport: String, + duration_ms_max: u64, + duration_ms_sum: u64, + publication_points: u64, +} + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct TopPublicationPoint { + rank: usize, + pp_id: String, + repo_id: String, + uri: String, + repo_uri: String, + host: String, + transport: String, + object_count: u64, + sync_duration_ms: u64, + terminal_state: String, + phase: String, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct Histogram { + buckets: Vec, + counts: Vec, + sum: f64, + count: u64, +} + +impl Default for Histogram { + fn default() -> Self { + Self { + buckets: Vec::new(), + counts: Vec::new(), + sum: 0.0, + count: 0, + } + } +} + +impl Histogram { + fn new(buckets: &[f64]) -> Self { + Self { + buckets: buckets.to_vec(), + counts: vec![0; buckets.len() + 1], + sum: 0.0, + count: 0, + } + } + + fn observe(&mut self, value: f64) { + self.sum += value; + self.count += 1; + let mut placed = false; + for (index, bucket) in self.buckets.iter().enumerate() { + if value <= *bucket { + self.counts[index] += 1; + placed = true; + break; + } + } + if !placed { + let last = self.counts.len() - 1; + self.counts[last] += 1; + } + } + + fn cumulative_counts(&self) -> Vec { + let mut out = Vec::with_capacity(self.counts.len()); + let mut running = 0u64; + for count in &self.counts { + running += *count; + out.push(running); + } + out + } +} + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct CirMetrics { + version: u32, + objects: u64, + trust_anchors: u64, + rejected_objects: u64, + reject_list_sha256: String, + objects_by_type: BTreeMap, + rejected_objects_by_type: BTreeMap, +} + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct CcrMetrics { + version: u32, + state_present: BTreeMap, + state_items: BTreeMap, + state_digests: BTreeMap, +} + +#[derive(Clone, Debug)] +struct RunRecord { + path: PathBuf, + status: String, + summary: Option, + meta: Option, +} + +fn scan_run_root(input_root: &Path, instance: &str) -> Result { + let started = Instant::now(); + let runs_root = resolve_runs_root(input_root); + let mut snapshot = MetricsSnapshot { + instance: instance.to_string(), + service: ServiceMetrics { + run_root: input_root.display().to_string(), + runs_root: runs_root.display().to_string(), + ..ServiceMetrics::default() + }, + ..MetricsSnapshot::default() + }; + + let records = collect_run_records(&runs_root, &mut snapshot.service.parse_errors)?; + snapshot.runs.known = records.len() as u64; + for record in &records { + match record.status.as_str() { + "success" => snapshot.runs.success += 1, + "failed" | "spawn_failed" => snapshot.runs.failed += 1, + _ => snapshot.runs.partial += 1, + } + } + snapshot.runs.consecutive_failures = consecutive_failures(&records); + snapshot.cumulative.completed_success_total = snapshot.runs.success; + snapshot.cumulative.completed_failed_total = snapshot.runs.failed; + + for record in records.iter().filter(|record| record.status == "success") { + if let Some(summary) = record.summary.as_ref() { + let wall_seconds = json_u64(summary, &["wallMs"]).unwrap_or(0) as f64 / 1000.0; + snapshot.cumulative.observed_duration_seconds_sum += wall_seconds; + snapshot.cumulative.observed_duration_seconds_count += 1; + if let Some(bytes) = json_u64(summary, &["stageTiming", "download_bytes_total"]) { + snapshot.cumulative.observed_download_bytes_total = snapshot + .cumulative + .observed_download_bytes_total + .saturating_add(bytes); + } + } + } + + if let Some(latest) = records + .iter() + .rev() + .find(|record| record.status == "success") + { + build_latest_metrics(latest, &mut snapshot); + } + + snapshot.service.last_scan_timestamp_seconds = unix_now_seconds(); + snapshot.service.last_scan_duration_seconds = started.elapsed().as_secs_f64(); + snapshot.service.last_reload_success = snapshot.service.parse_errors.is_empty(); + Ok(snapshot) +} + +fn resolve_runs_root(input_root: &Path) -> PathBuf { + let runs = input_root.join("runs"); + if runs.is_dir() { + runs + } else { + input_root.to_path_buf() + } +} + +fn collect_run_records( + runs_root: &Path, + errors: &mut Vec, +) -> Result, String> { + let mut records = Vec::new(); + if !runs_root.is_dir() { + return Err(format!( + "runs root is not a directory: {}", + runs_root.display() + )); + } + let entries = fs::read_dir(runs_root) + .map_err(|e| format!("read runs root failed: {}: {e}", runs_root.display()))?; + for entry in entries { + let entry = entry.map_err(|e| format!("read runs entry failed: {e}"))?; + let path = entry.path(); + if !path.is_dir() { + continue; + } + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + if !name.starts_with("run_") { + continue; + } + let summary = read_json_optional(&path.join("run-summary.json"), errors); + let meta = read_json_optional(&path.join("run-meta.json"), errors); + let status = classify_run_status(&summary, &meta, &path); + records.push(RunRecord { + path, + status, + summary, + meta, + }); + } + records.sort_by(|left, right| left.path.cmp(&right.path)); + Ok(records) +} + +fn classify_run_status(summary: &Option, meta: &Option, path: &Path) -> String { + let summary_status = summary.as_ref().and_then(|v| json_str(v, &["status"])); + let meta_status = meta.as_ref().and_then(|v| json_str(v, &["status"])); + if summary_status == Some("success") && meta_status == Some("success") { + return "success".to_string(); + } + if matches!(summary_status, Some("failed" | "spawn_failed")) + || matches!(meta_status, Some("failed" | "spawn_failed")) + { + return "failed".to_string(); + } + if path.join("run-summary.json").exists() || path.join("run-meta.json").exists() { + "partial".to_string() + } else { + "missing_metadata".to_string() + } +} + +fn consecutive_failures(records: &[RunRecord]) -> u64 { + let mut count = 0u64; + for record in records.iter().rev() { + if record.status == "success" { + break; + } + count += 1; + } + count +} + +fn read_json_optional(path: &Path, errors: &mut Vec) -> Option { + if !path.exists() { + return None; + } + match fs::read(path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + { + Some(value) => Some(value), + None => { + errors.push(format!("parse json failed: {}", path.display())); + None + } + } +} + +fn build_latest_metrics(record: &RunRecord, snapshot: &mut MetricsSnapshot) { + let summary = record.summary.as_ref(); + let meta = record.meta.as_ref(); + let run_seq = summary + .and_then(|v| json_u64(v, &["runSeq"])) + .or_else(|| meta.and_then(|v| json_u64(v, &["run_index"]))) + .unwrap_or_else(|| run_index_from_path(&record.path).unwrap_or(0)); + let run_id = summary + .and_then(|v| json_str(v, &["runId"])) + .or_else(|| meta.and_then(|v| json_str(v, &["run_id"]))) + .unwrap_or_else(|| { + record + .path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("unknown") + }) + .to_string(); + let sync_mode = meta + .and_then(|v| json_str(v, &["sync_mode"])) + .unwrap_or("unknown") + .to_string(); + let snapshot_reason = meta + .and_then(|v| json_str(v, &["snapshot_reason"])) + .map(|s| s.to_string()); + let started_at = summary + .and_then(|v| json_str(v, &["startedAtRfc3339Utc"])) + .or_else(|| meta.and_then(|v| json_str(v, &["started_at_rfc3339_utc"]))) + .map(|s| s.to_string()); + let finished_at = summary + .and_then(|v| json_str(v, &["finishedAtRfc3339Utc"])) + .or_else(|| meta.and_then(|v| json_str(v, &["completed_at_rfc3339_utc"]))) + .map(|s| s.to_string()); + let wall_seconds = summary.and_then(|v| json_u64(v, &["wallMs"])).unwrap_or(0) as f64 / 1000.0; + + let mut latest = LatestRunMetrics { + run_seq, + run_id, + run_dir: record.path.display().to_string(), + status: record.status.clone(), + sync_mode, + snapshot_reason, + started_at: started_at.clone(), + finished_at: finished_at.clone(), + start_timestamp_seconds: started_at.as_deref().and_then(parse_rfc3339_to_unix), + finish_timestamp_seconds: finished_at.as_deref().and_then(parse_rfc3339_to_unix), + wall_seconds, + user_cpu_seconds: summary.and_then(|v| json_f64(v, &["processMetrics", "userSeconds"])), + system_cpu_seconds: summary.and_then(|v| json_f64(v, &["processMetrics", "systemSeconds"])), + cpu_percent: summary.and_then(|v| json_f64(v, &["processMetrics", "cpuPercent"])), + max_rss_bytes: summary + .and_then(|v| json_u64(v, &["processMetrics", "maxRssKb"])) + .map(|kb| kb.saturating_mul(1024)), + exit_code: summary.and_then(|v| json_i64(v, &["exitCode"])), + ..LatestRunMetrics::default() + }; + + if let Some(summary) = summary { + latest.vrps = json_u64(summary, &["reportCounts", "vrps"]).unwrap_or(0); + latest.vaps = json_u64(summary, &["reportCounts", "aspas"]).unwrap_or(0); + latest.publication_points = + json_u64(summary, &["reportCounts", "publicationPoints"]).unwrap_or(0); + latest.warnings = json_u64(summary, &["reportCounts", "warnings"]).unwrap_or(0); + latest.tree_instances_processed = + json_u64(summary, &["reportCounts", "treeInstancesProcessed"]); + latest.tree_instances_failed = json_u64(summary, &["reportCounts", "treeInstancesFailed"]); + latest.stage_seconds = extract_stage_seconds(summary.get("stageTiming")); + latest.repo_sync_phase = + extract_count_duration_map(summary.pointer("/repoSyncStats/by_phase")); + latest.repo_terminal_state = + extract_count_duration_map(summary.pointer("/repoSyncStats/by_terminal_state")); + latest.download_events = json_u64(summary, &["stageTiming", "download_event_count"]); + latest.download_bytes = json_u64(summary, &["stageTiming", "download_bytes_total"]); + latest.artifact_sizes = extract_artifact_sizes(summary.get("artifacts")); + latest.state_path_sizes = extract_path_sizes(summary.get("pathStats")); + } + + parse_report(&record.path.join("report.json"), snapshot, &mut latest); + parse_cir(&record.path.join("input.cir"), snapshot); + parse_ccr(&record.path.join("result.ccr"), snapshot); + snapshot.latest_run = Some(latest); +} + +fn parse_report(path: &Path, snapshot: &mut MetricsSnapshot, latest: &mut LatestRunMetrics) { + if !path.exists() { + return; + } + let Ok(bytes) = fs::read(path) else { + snapshot + .service + .parse_errors + .push(format!("read report.json failed: {}", path.display())); + return; + }; + let Ok(report) = serde_json::from_slice::(&bytes) else { + snapshot + .service + .parse_errors + .push(format!("parse report.json failed: {}", path.display())); + return; + }; + if latest.vrps == 0 { + latest.vrps = report + .get("vrps") + .and_then(|v| v.as_array()) + .map(|a| a.len() as u64) + .unwrap_or(0); + } + if latest.vaps == 0 { + latest.vaps = report + .get("aspas") + .and_then(|v| v.as_array()) + .map(|a| a.len() as u64) + .unwrap_or(0); + } + latest.warnings = latest.warnings.max( + report + .pointer("/tree/warnings") + .and_then(|v| v.as_array()) + .map(|a| a.len() as u64) + .unwrap_or(0), + ); + if let Some(processed) = json_u64(&report, &["tree", "instances_processed"]) { + latest.tree_instances_processed = Some(processed); + } + if let Some(failed) = json_u64(&report, &["tree", "instances_failed"]) { + latest.tree_instances_failed = Some(failed); + } + if latest.repo_sync_phase.is_empty() { + latest.repo_sync_phase = + extract_count_duration_map(report.pointer("/repo_sync_stats/by_phase")); + } + if latest.repo_terminal_state.is_empty() { + latest.repo_terminal_state = + extract_count_duration_map(report.pointer("/repo_sync_stats/by_terminal_state")); + } + if let Some(pps) = report.get("publication_points").and_then(|v| v.as_array()) { + latest.publication_points = pps.len() as u64; + extract_publication_point_metrics(pps, report.get("downloads"), snapshot); + } +} + +fn extract_publication_point_metrics( + pps: &[Value], + downloads: Option<&Value>, + snapshot: &mut MetricsSnapshot, +) { + let mut repos: BTreeMap = BTreeMap::new(); + let mut pp_by_object_count = Vec::::new(); + let mut pp_by_sync_duration = Vec::::new(); + let mut large_pp_counts = BTreeMap::::new(); + let mut pp_sync_histograms = BTreeMap::::new(); + let mut object_counts = BTreeMap::<(String, String), u64>::new(); + + for pp in pps { + let pp_uri = json_str(pp, &["publication_point_rsync_uri"]) + .or_else(|| json_str(pp, &["manifest_rsync_uri"])) + .or_else(|| json_str(pp, &["rsync_base_uri"])) + .unwrap_or("unknown"); + let repo_uri = json_str(pp, &["rrdp_notification_uri"]) + .or_else(|| json_str(pp, &["rsync_base_uri"])) + .or_else(|| json_str(pp, &["publication_point_rsync_uri"])) + .unwrap_or(pp_uri); + let repo_id = short_sha256(repo_uri); + let pp_id = short_sha256(pp_uri); + let host = uri_host(repo_uri); + let transport = json_str(pp, &["repo_sync_source"]) + .or_else(|| json_str(pp, &["source"])) + .map(normalize_transport) + .unwrap_or_else(|| infer_transport(repo_uri)); + let duration_ms = json_u64(pp, &["repo_sync_duration_ms"]).unwrap_or(0); + let duration_seconds = duration_ms as f64 / 1000.0; + let phase = json_str(pp, &["repo_sync_phase"]) + .unwrap_or("unknown") + .to_string(); + let terminal_state = json_str(pp, &["repo_terminal_state"]) + .unwrap_or("unknown") + .to_string(); + let object_count = pp + .get("objects") + .and_then(|v| v.as_array()) + .map(|a| a.len() as u64) + .unwrap_or(0); + + let repo = repos.entry(repo_id.clone()).or_insert_with(|| RepoMetrics { + repo_id: repo_id.clone(), + uri: repo_uri.to_string(), + host: host.clone(), + transport: transport.clone(), + sync_success: true, + ..RepoMetrics::default() + }); + repo.publication_points += 1; + repo.duration_seconds_sum += duration_seconds; + repo.duration_seconds_max = repo.duration_seconds_max.max(duration_seconds); + if !is_success_terminal_state(&terminal_state) { + repo.sync_success = false; + } + *repo.phase_counts.entry(phase.clone()).or_default() += 1; + *repo + .terminal_state_counts + .entry(terminal_state.clone()) + .or_default() += 1; + + for threshold in LARGE_PP_OBJECT_THRESHOLDS { + if object_count > *threshold { + *large_pp_counts.entry(*threshold).or_default() += 1; + } + } + pp_sync_histograms + .entry(transport.clone()) + .or_insert_with(|| Histogram::new(PP_SYNC_SECONDS_BUCKETS)) + .observe(duration_seconds); + + if let Some(objects) = pp.get("objects").and_then(|v| v.as_array()) { + for object in objects { + let kind = json_str(object, &["kind"]).unwrap_or("unknown").to_string(); + let result = json_str(object, &["result"]) + .unwrap_or("unknown") + .to_string(); + *object_counts.entry((kind, result)).or_default() += 1; + } + } + + let top = TopPublicationPoint { + rank: 0, + pp_id, + repo_id, + uri: pp_uri.to_string(), + repo_uri: repo_uri.to_string(), + host, + transport, + object_count, + sync_duration_ms: duration_ms, + terminal_state, + phase, + }; + pp_by_object_count.push(top.clone()); + pp_by_sync_duration.push(top); + } + + let mut repo_stats = repos.into_values().collect::>(); + assign_download_bytes_to_repos(&mut repo_stats, downloads); + for repo in &mut repo_stats { + if repo.publication_points > 0 { + repo.duration_seconds_avg = repo.duration_seconds_sum / repo.publication_points as f64; + } + } + let mut top_repos = repo_stats + .iter() + .map(|repo| TopRepo { + rank: 0, + repo_id: repo.repo_id.clone(), + uri: repo.uri.clone(), + host: repo.host.clone(), + transport: repo.transport.clone(), + duration_ms_max: (repo.duration_seconds_max * 1000.0).round() as u64, + duration_ms_sum: (repo.duration_seconds_sum * 1000.0).round() as u64, + publication_points: repo.publication_points, + }) + .collect::>(); + top_repos.sort_by(|a, b| b.duration_ms_max.cmp(&a.duration_ms_max)); + top_repos.truncate(20); + for (index, item) in top_repos.iter_mut().enumerate() { + item.rank = index + 1; + } + + pp_by_object_count.sort_by(|a, b| b.object_count.cmp(&a.object_count)); + pp_by_object_count.truncate(20); + for (index, item) in pp_by_object_count.iter_mut().enumerate() { + item.rank = index + 1; + } + pp_by_sync_duration.sort_by(|a, b| b.sync_duration_ms.cmp(&a.sync_duration_ms)); + pp_by_sync_duration.truncate(20); + for (index, item) in pp_by_sync_duration.iter_mut().enumerate() { + item.rank = index + 1; + } + + snapshot.repo_stats = repo_stats; + snapshot.object_counts = object_counts; + snapshot.large_pp_counts = large_pp_counts; + snapshot.pp_sync_histograms = pp_sync_histograms; + snapshot.top_repos_by_sync_duration = top_repos; + snapshot.top_pp_by_object_count = pp_by_object_count; + snapshot.top_pp_by_sync_duration = pp_by_sync_duration; +} + +fn assign_download_bytes_to_repos(repos: &mut [RepoMetrics], downloads: Option<&Value>) { + let Some(downloads) = downloads.and_then(|v| v.as_array()) else { + return; + }; + for download in downloads { + let Some(uri) = json_str(download, &["uri"]) else { + continue; + }; + let bytes = json_u64(download, &["bytes"]).unwrap_or(0); + if bytes == 0 { + continue; + } + if let Some(index) = find_repo_for_download(repos, uri) { + repos[index].download_bytes = repos[index].download_bytes.saturating_add(bytes); + } + } +} + +fn find_repo_for_download(repos: &[RepoMetrics], uri: &str) -> Option { + if let Some(index) = repos.iter().position(|repo| repo.uri == uri) { + return Some(index); + } + if uri.starts_with("rsync://") { + return repos + .iter() + .enumerate() + .filter(|(_, repo)| uri.starts_with(&repo.uri)) + .max_by_key(|(_, repo)| repo.uri.len()) + .map(|(index, _)| index); + } + + let uri_host = uri_host(uri); + let mut candidates = repos + .iter() + .enumerate() + .filter(|(_, repo)| repo.host == uri_host && repo.uri.starts_with("http")) + .collect::>(); + if candidates.len() == 1 { + return Some(candidates[0].0); + } + candidates.sort_by_key(|(_, repo)| common_prefix_len(&repo.uri, uri)); + candidates + .last() + .and_then(|(index, repo)| (common_prefix_len(&repo.uri, uri) > 0).then_some(*index)) +} + +fn is_success_terminal_state(state: &str) -> bool { + matches!(state, "fresh" | "cached" | "reused" | "valid") +} + +fn parse_cir(path: &Path, snapshot: &mut MetricsSnapshot) { + if !path.exists() { + return; + } + match fs::read(path) + .map_err(|e| e.to_string()) + .and_then(|bytes| decode_cir(&bytes).map_err(|e| e.to_string())) + { + Ok(cir) => { + let mut objects_by_type = BTreeMap::new(); + for object in &cir.objects { + *objects_by_type + .entry(object_type_from_uri(&object.rsync_uri)) + .or_default() += 1; + } + let mut rejected_objects_by_type = BTreeMap::new(); + for object in &cir.rejected_objects { + *rejected_objects_by_type + .entry(object_type_from_uri(&object.object_uri)) + .or_default() += 1; + } + snapshot.cir = Some(CirMetrics { + version: cir.version, + objects: cir.objects.len() as u64, + trust_anchors: cir.trust_anchors.len() as u64, + rejected_objects: cir.rejected_objects.len() as u64, + reject_list_sha256: hex::encode(&cir.reject_list_sha256), + objects_by_type, + rejected_objects_by_type, + }); + } + Err(err) => snapshot + .service + .parse_errors + .push(format!("decode CIR failed: {}: {err}", path.display())), + } +} + +fn parse_ccr(path: &Path, snapshot: &mut MetricsSnapshot) { + if !path.exists() { + return; + } + match fs::read(path) + .map_err(|e| e.to_string()) + .and_then(|bytes| decode_content_info(&bytes).map_err(|e| e.to_string())) + { + Ok(ccr) => { + let content = ccr.content; + let mut state_present = BTreeMap::new(); + let mut state_items = BTreeMap::new(); + let mut state_digests = BTreeMap::new(); + if let Some(state) = content.mfts.as_ref() { + state_present.insert("mfts".to_string(), true); + state_items.insert("mfts".to_string(), state.mis.len() as u64); + state_digests.insert("mfts".to_string(), hex::encode(&state.hash)); + } else { + state_present.insert("mfts".to_string(), false); + } + if let Some(state) = content.vrps.as_ref() { + state_present.insert("vrps".to_string(), true); + state_items.insert("vrps".to_string(), state.rps.len() as u64); + state_digests.insert("vrps".to_string(), hex::encode(&state.hash)); + } else { + state_present.insert("vrps".to_string(), false); + } + if let Some(state) = content.vaps.as_ref() { + state_present.insert("vaps".to_string(), true); + state_items.insert("vaps".to_string(), state.aps.len() as u64); + state_digests.insert("vaps".to_string(), hex::encode(&state.hash)); + } else { + state_present.insert("vaps".to_string(), false); + } + if let Some(state) = content.tas.as_ref() { + state_present.insert("tas".to_string(), true); + state_items.insert("tas".to_string(), state.skis.len() as u64); + state_digests.insert("tas".to_string(), hex::encode(&state.hash)); + } else { + state_present.insert("tas".to_string(), false); + } + if let Some(state) = content.rks.as_ref() { + state_present.insert("rks".to_string(), true); + state_items.insert("rks".to_string(), state.rksets.len() as u64); + state_digests.insert("rks".to_string(), hex::encode(&state.hash)); + } else { + state_present.insert("rks".to_string(), false); + } + snapshot.ccr = Some(CcrMetrics { + version: content.version, + state_present, + state_items, + state_digests, + }); + } + Err(err) => snapshot + .service + .parse_errors + .push(format!("decode CCR failed: {}: {err}", path.display())), + } +} + +fn render_metrics(snapshot: &MetricsSnapshot) -> String { + let mut out = String::new(); + let mut writer = PromWriter::new(&mut out); + let instance = snapshot.instance.as_str(); + + writer.gauge( + "ours_rp_metrics_service_up", + "Artifact metrics service is up", + &[label("instance", instance)], + 1.0, + ); + writer.gauge( + "ours_rp_metrics_service_last_scan_timestamp_seconds", + "Unix timestamp of the last artifact scan", + &[label("instance", instance)], + snapshot.service.last_scan_timestamp_seconds, + ); + writer.gauge( + "ours_rp_metrics_service_last_scan_duration_seconds", + "Duration of the last artifact scan", + &[label("instance", instance)], + snapshot.service.last_scan_duration_seconds, + ); + writer.gauge( + "ours_rp_metrics_service_last_reload_success", + "Whether the last artifact reload had no parse errors", + &[label("instance", instance)], + bool_value(snapshot.service.last_reload_success), + ); + writer.gauge( + "ours_rp_metrics_service_parse_errors", + "Current parse error count", + &[label("instance", instance)], + snapshot.service.parse_errors.len() as f64, + ); + writer.gauge( + "ours_rp_metrics_service_known_runs", + "Known run directories by status", + &[label("instance", instance), label("status", "success")], + snapshot.runs.success as f64, + ); + writer.gauge( + "ours_rp_metrics_service_known_runs", + "Known run directories by status", + &[label("instance", instance), label("status", "failed")], + snapshot.runs.failed as f64, + ); + writer.gauge( + "ours_rp_metrics_service_known_runs", + "Known run directories by status", + &[label("instance", instance), label("status", "partial")], + snapshot.runs.partial as f64, + ); + + writer.counter( + "ours_rp_run_completed_total", + "Completed runs observed by the artifact metrics service", + &[label("instance", instance), label("status", "success")], + snapshot.cumulative.completed_success_total as f64, + ); + writer.counter( + "ours_rp_run_completed_total", + "Completed runs observed by the artifact metrics service", + &[label("instance", instance), label("status", "failed")], + snapshot.cumulative.completed_failed_total as f64, + ); + writer.counter( + "ours_rp_run_observed_duration_seconds_sum", + "Observed wall duration sum for successful runs", + &[label("instance", instance)], + snapshot.cumulative.observed_duration_seconds_sum, + ); + writer.counter( + "ours_rp_run_observed_duration_seconds_count", + "Observed wall duration count for successful runs", + &[label("instance", instance)], + snapshot.cumulative.observed_duration_seconds_count as f64, + ); + writer.counter( + "ours_rp_run_observed_download_bytes_total", + "Observed download bytes across successful runs", + &[label("instance", instance)], + snapshot.cumulative.observed_download_bytes_total as f64, + ); + writer.gauge( + "ours_rp_run_consecutive_failures", + "Consecutive non-success runs at the end of the run list", + &[label("instance", instance)], + snapshot.runs.consecutive_failures as f64, + ); + + if let Some(latest) = snapshot.latest_run.as_ref() { + render_latest_metrics(&mut writer, instance, latest); + } + render_repo_metrics(&mut writer, instance, &snapshot.repo_stats); + render_failed_repo_metrics(&mut writer, instance, &snapshot.repo_stats); + render_top_repo_metrics(&mut writer, instance, &snapshot.top_repos_by_sync_duration); + render_object_metrics(&mut writer, instance, &snapshot.object_counts); + render_large_pp_metrics(&mut writer, instance, &snapshot.large_pp_counts); + render_top_publication_point_metrics(&mut writer, instance, &snapshot.top_pp_by_object_count); + for (transport, histogram) in &snapshot.pp_sync_histograms { + writer.histogram( + "ours_rp_publication_point_sync_duration_seconds", + "Distribution of sync duration per publication point", + &[label("instance", instance), label("transport", transport)], + histogram, + ); + } + if let Some(cir) = snapshot.cir.as_ref() { + render_cir_metrics(&mut writer, instance, cir); + } + if let Some(ccr) = snapshot.ccr.as_ref() { + render_ccr_metrics(&mut writer, instance, ccr); + } + out +} + +fn render_latest_metrics(writer: &mut PromWriter<'_>, instance: &str, latest: &LatestRunMetrics) { + writer.gauge( + "ours_rp_run_sequence", + "Latest successful run sequence", + &[label("instance", instance)], + latest.run_seq as f64, + ); + writer.gauge( + "ours_rp_run_success", + "Whether the latest selected run is successful", + &[label("instance", instance)], + bool_value(latest.status == "success"), + ); + writer.gauge( + "ours_rp_run_sync_mode", + "Latest run sync mode state", + &[ + label("instance", instance), + label("sync_mode", &latest.sync_mode), + ], + 1.0, + ); + if let Some(ts) = latest.start_timestamp_seconds { + writer.gauge( + "ours_rp_run_start_timestamp_seconds", + "Latest run start timestamp", + &[label("instance", instance)], + ts, + ); + } + if let Some(ts) = latest.finish_timestamp_seconds { + writer.gauge( + "ours_rp_run_finish_timestamp_seconds", + "Latest run finish timestamp", + &[label("instance", instance)], + ts, + ); + } + writer.gauge( + "ours_rp_run_duration_seconds", + "Latest run wall duration", + &[label("instance", instance)], + latest.wall_seconds, + ); + if let Some(value) = latest.user_cpu_seconds { + writer.gauge( + "ours_rp_run_user_cpu_seconds", + "Latest run user CPU seconds", + &[label("instance", instance)], + value, + ); + } + if let Some(value) = latest.system_cpu_seconds { + writer.gauge( + "ours_rp_run_system_cpu_seconds", + "Latest run system CPU seconds", + &[label("instance", instance)], + value, + ); + } + if let Some(value) = latest.cpu_percent { + writer.gauge( + "ours_rp_run_cpu_percent", + "Latest run CPU percent from GNU time", + &[label("instance", instance)], + value, + ); + } + if let Some(value) = latest.max_rss_bytes { + writer.gauge( + "ours_rp_run_max_rss_bytes", + "Latest run maximum resident set size", + &[label("instance", instance)], + value as f64, + ); + } + if let Some(value) = latest.exit_code { + writer.gauge( + "ours_rp_run_exit_code", + "Latest run exit code", + &[label("instance", instance)], + value as f64, + ); + } + writer.gauge( + "ours_rp_vrps", + "Latest run VRP count", + &[label("instance", instance), label("kind", "total")], + latest.vrps as f64, + ); + writer.gauge( + "ours_rp_vaps", + "Latest run VAP/ASPA count", + &[label("instance", instance), label("kind", "total")], + latest.vaps as f64, + ); + writer.gauge( + "ours_rp_publication_points", + "Latest run publication point count", + &[label("instance", instance)], + latest.publication_points as f64, + ); + writer.gauge( + "ours_rp_warnings", + "Latest run warning count", + &[label("instance", instance)], + latest.warnings as f64, + ); + if let Some(value) = latest.tree_instances_processed { + writer.gauge( + "ours_rp_tree_instances", + "Latest run tree instances by state", + &[label("instance", instance), label("state", "processed")], + value as f64, + ); + } + if let Some(value) = latest.tree_instances_failed { + writer.gauge( + "ours_rp_tree_instances", + "Latest run tree instances by state", + &[label("instance", instance), label("state", "failed")], + value as f64, + ); + } + for (stage, value) in &latest.stage_seconds { + writer.gauge( + "ours_rp_run_stage_duration_seconds", + "Latest run stage duration", + &[label("instance", instance), label("stage", stage)], + *value, + ); + } + for (phase, stat) in &latest.repo_sync_phase { + writer.gauge( + "ours_rp_repo_sync_phase_count", + "Publication points by repo sync phase", + &[label("instance", instance), label("phase", phase)], + stat.count as f64, + ); + writer.gauge( + "ours_rp_repo_sync_phase_duration_seconds_total", + "Repo sync phase cumulative duration in latest run", + &[label("instance", instance), label("phase", phase)], + stat.duration_seconds_total, + ); + } + for (state, stat) in &latest.repo_terminal_state { + writer.gauge( + "ours_rp_repo_terminal_state_count", + "Publication points by terminal state", + &[label("instance", instance), label("terminal_state", state)], + stat.count as f64, + ); + writer.gauge( + "ours_rp_repo_terminal_state_duration_seconds_total", + "Terminal state cumulative duration in latest run", + &[label("instance", instance), label("terminal_state", state)], + stat.duration_seconds_total, + ); + } + if let Some(value) = latest.download_events { + writer.gauge( + "ours_rp_download_events", + "Latest run download event count", + &[label("instance", instance)], + value as f64, + ); + } + if let Some(value) = latest.download_bytes { + writer.gauge( + "ours_rp_download_bytes", + "Latest run download bytes", + &[label("instance", instance)], + value as f64, + ); + } + for (artifact, size) in &latest.artifact_sizes { + writer.gauge( + "ours_rp_artifact_size_bytes", + "Latest run artifact size", + &[label("instance", instance), label("artifact", artifact)], + *size as f64, + ); + } + for (path, stat) in &latest.state_path_sizes { + writer.gauge( + "ours_rp_state_path_size_bytes", + "State path size", + &[label("instance", instance), label("path", path)], + stat.total_size_bytes as f64, + ); + writer.gauge( + "ours_rp_state_path_files", + "State path file count", + &[label("instance", instance), label("path", path)], + stat.file_count as f64, + ); + } +} + +fn render_repo_metrics(writer: &mut PromWriter<'_>, instance: &str, repos: &[RepoMetrics]) { + for repo in repos { + let base = [ + label("instance", instance), + label("repo_id", &repo.repo_id), + label("host", &repo.host), + label("uri", &repo.uri), + label("transport", &repo.transport), + ]; + writer.gauge("ours_rp_repository_info", "Repository metadata", &base, 1.0); + writer.gauge( + "ours_rp_repository_publication_points", + "Publication points per repository", + &base, + repo.publication_points as f64, + ); + writer.gauge( + "ours_rp_repository_sync_success", + "Whether repository sync is successful in the latest run", + &base, + bool_value(repo.sync_success), + ); + writer.gauge( + "ours_rp_repository_download_bytes", + "Repository download bytes attributed from latest run download events", + &base, + repo.download_bytes as f64, + ); + for (stat, value) in [ + ("sum", repo.duration_seconds_sum), + ("max", repo.duration_seconds_max), + ("avg", repo.duration_seconds_avg), + ] { + let labels = [ + label("instance", instance), + label("repo_id", &repo.repo_id), + label("host", &repo.host), + label("transport", &repo.transport), + label("stat", stat), + ]; + writer.gauge( + "ours_rp_repository_sync_duration_seconds", + "Repository sync duration summary", + &labels, + value, + ); + } + for (phase, count) in &repo.phase_counts { + let labels = [ + label("instance", instance), + label("repo_id", &repo.repo_id), + label("host", &repo.host), + label("phase", phase), + ]; + writer.gauge( + "ours_rp_repository_sync_phase_publication_points", + "Repository publication points by sync phase", + &labels, + *count as f64, + ); + } + for (state, count) in &repo.terminal_state_counts { + let labels = [ + label("instance", instance), + label("repo_id", &repo.repo_id), + label("host", &repo.host), + label("terminal_state", state), + ]; + writer.gauge( + "ours_rp_repository_terminal_state_publication_points", + "Repository publication points by terminal state", + &labels, + *count as f64, + ); + } + } +} + +fn render_failed_repo_metrics(writer: &mut PromWriter<'_>, instance: &str, repos: &[RepoMetrics]) { + for repo in repos { + if repo.phase_counts.contains_key("rrdp_failed_rsync_failed") { + writer.gauge( + "ours_rp_rrdp_rsync_failed_repository_duration_seconds", + "Repositories whose RRDP and rsync sync both failed; value is max sync duration when available", + &[ + label("instance", instance), + label("repo_id", &repo.repo_id), + label("host", &repo.host), + label("phase", "rrdp_failed_rsync_failed"), + label("transport", &repo.transport), + label("uri", &repo.uri), + ], + repo.duration_seconds_max, + ); + } + } +} + +fn render_top_repo_metrics(writer: &mut PromWriter<'_>, instance: &str, repos: &[TopRepo]) { + for repo in repos { + writer.gauge( + "ours_rp_top_repository_sync_duration_seconds", + "Top repositories by max sync duration in latest run", + &[ + label("instance", instance), + label("rank", &repo.rank.to_string()), + label("repo_id", &repo.repo_id), + label("host", &repo.host), + label("transport", &repo.transport), + label("publication_points", &repo.publication_points.to_string()), + label("uri", &repo.uri), + ], + repo.duration_ms_max as f64 / 1000.0, + ); + } +} + +fn render_object_metrics( + writer: &mut PromWriter<'_>, + instance: &str, + counts: &BTreeMap<(String, String), u64>, +) { + for ((object_type, result), count) in counts { + writer.gauge( + "ours_rp_objects", + "Latest run audited objects by type and result", + &[ + label("instance", instance), + label("object_type", object_type), + label("result", result), + ], + *count as f64, + ); + } +} + +fn render_top_publication_point_metrics( + writer: &mut PromWriter<'_>, + instance: &str, + publication_points: &[TopPublicationPoint], +) { + for publication_point in publication_points { + writer.gauge( + "ours_rp_top_publication_point_object_count", + "Top publication points by object count in latest run", + &[ + label("instance", instance), + label("rank", &publication_point.rank.to_string()), + label("pp_id", &publication_point.pp_id), + label("repo_id", &publication_point.repo_id), + label("host", &publication_point.host), + label("transport", &publication_point.transport), + label("terminal_state", &publication_point.terminal_state), + label("phase", &publication_point.phase), + label("uri", &publication_point.uri), + ], + publication_point.object_count as f64, + ); + } +} + +fn render_large_pp_metrics( + writer: &mut PromWriter<'_>, + instance: &str, + counts: &BTreeMap, +) { + for threshold in LARGE_PP_OBJECT_THRESHOLDS { + writer.gauge( + "ours_rp_large_publication_points", + "Publication points with object count greater than threshold", + &[ + label("instance", instance), + label("object_count_gt", &threshold.to_string()), + ], + counts.get(threshold).copied().unwrap_or(0) as f64, + ); + } +} + +fn render_cir_metrics(writer: &mut PromWriter<'_>, instance: &str, cir: &CirMetrics) { + writer.gauge( + "ours_rp_cir_version", + "CIR version", + &[label("instance", instance)], + cir.version as f64, + ); + writer.gauge( + "ours_rp_cir_objects", + "CIR object count", + &[label("instance", instance)], + cir.objects as f64, + ); + writer.gauge( + "ours_rp_cir_trust_anchors", + "CIR trust anchor count", + &[label("instance", instance)], + cir.trust_anchors as f64, + ); + writer.gauge( + "ours_rp_cir_rejected_objects", + "CIR rejected object count", + &[label("instance", instance)], + cir.rejected_objects as f64, + ); + writer.gauge( + "ours_rp_cir_reject_list_digest_present", + "CIR reject list digest is present", + &[label("instance", instance)], + if cir.reject_list_sha256.len() == 64 { + 1.0 + } else { + 0.0 + }, + ); + for (object_type, count) in &cir.objects_by_type { + writer.gauge( + "ours_rp_cir_objects_by_type", + "CIR object count by file type", + &[ + label("instance", instance), + label("object_type", object_type), + ], + *count as f64, + ); + } + for (object_type, count) in &cir.rejected_objects_by_type { + writer.gauge( + "ours_rp_cir_rejected_objects_by_type", + "CIR rejected object count by file type", + &[ + label("instance", instance), + label("object_type", object_type), + ], + *count as f64, + ); + } +} + +fn render_ccr_metrics(writer: &mut PromWriter<'_>, instance: &str, ccr: &CcrMetrics) { + writer.gauge( + "ours_rp_ccr_version", + "CCR version", + &[label("instance", instance)], + ccr.version as f64, + ); + for (state, present) in &ccr.state_present { + writer.gauge( + "ours_rp_ccr_state_present", + "CCR state presence", + &[label("instance", instance), label("state", state)], + bool_value(*present), + ); + } + for (state, count) in &ccr.state_items { + writer.gauge( + "ours_rp_ccr_state_items", + "CCR state item count", + &[label("instance", instance), label("state", state)], + *count as f64, + ); + } + for state in ccr.state_digests.keys() { + writer.gauge( + "ours_rp_ccr_state_digest_present", + "CCR state digest presence", + &[label("instance", instance), label("state", state)], + 1.0, + ); + } +} + +fn render_status_json(snapshot: &MetricsSnapshot) -> Result { + serde_json::to_string_pretty(&json!({ + "schemaVersion": 1, + "generatedBy": "rpki_artifact_metrics", + "instance": snapshot.instance, + "service": snapshot.service, + "runs": snapshot.runs, + "latestRun": snapshot.latest_run, + "cir": snapshot.cir, + "ccr": snapshot.ccr, + "topRepositoriesBySyncDuration": snapshot.top_repos_by_sync_duration, + "topPublicationPointsByObjectCount": snapshot.top_pp_by_object_count, + "topPublicationPointsBySyncDuration": snapshot.top_pp_by_sync_duration, + })) + .map_err(|e| e.to_string()) +} + +struct PromWriter<'a> { + out: &'a mut String, + emitted_headers: BTreeSet, +} + +#[derive(Clone, Debug)] +struct Label<'a> { + key: &'a str, + value: &'a str, +} + +fn label<'a>(key: &'a str, value: &'a str) -> Label<'a> { + Label { key, value } +} + +impl<'a> PromWriter<'a> { + fn new(out: &'a mut String) -> Self { + Self { + out, + emitted_headers: BTreeSet::new(), + } + } + + fn gauge(&mut self, name: &str, help: &str, labels: &[Label<'_>], value: f64) { + self.metric("gauge", name, help, labels, value); + } + + fn counter(&mut self, name: &str, help: &str, labels: &[Label<'_>], value: f64) { + self.metric("counter", name, help, labels, value); + } + + fn metric( + &mut self, + metric_type: &str, + name: &str, + help: &str, + labels: &[Label<'_>], + value: f64, + ) { + self.header(name, help, metric_type); + self.out.push_str(name); + write_labels(self.out, labels); + self.out.push(' '); + self.out.push_str(&format_prom_value(value)); + self.out.push('\n'); + } + + fn histogram( + &mut self, + name: &str, + help: &str, + base_labels: &[Label<'_>], + histogram: &Histogram, + ) { + self.header(name, help, "histogram"); + let cumulative = histogram.cumulative_counts(); + for (index, count) in cumulative.iter().enumerate() { + let le = if index < histogram.buckets.len() { + format_prom_value(histogram.buckets[index]) + } else { + "+Inf".to_string() + }; + let mut labels = base_labels.to_vec(); + labels.push(label("le", &le)); + self.out.push_str(name); + self.out.push_str("_bucket"); + write_labels(self.out, &labels); + self.out.push(' '); + self.out.push_str(&count.to_string()); + self.out.push('\n'); + } + self.out.push_str(name); + self.out.push_str("_sum"); + write_labels(self.out, base_labels); + self.out.push(' '); + self.out.push_str(&format_prom_value(histogram.sum)); + self.out.push('\n'); + self.out.push_str(name); + self.out.push_str("_count"); + write_labels(self.out, base_labels); + self.out.push(' '); + self.out.push_str(&histogram.count.to_string()); + self.out.push('\n'); + } + + fn header(&mut self, name: &str, help: &str, metric_type: &str) { + if self.emitted_headers.insert(name.to_string()) { + self.out.push_str("# HELP "); + self.out.push_str(name); + self.out.push(' '); + self.out.push_str(&escape_help(help)); + self.out.push('\n'); + self.out.push_str("# TYPE "); + self.out.push_str(name); + self.out.push(' '); + self.out.push_str(metric_type); + self.out.push('\n'); + } + } +} + +fn write_labels(out: &mut String, labels: &[Label<'_>]) { + if labels.is_empty() { + return; + } + out.push('{'); + for (index, label) in labels.iter().enumerate() { + if index > 0 { + out.push(','); + } + out.push_str(label.key); + out.push_str("=\""); + out.push_str(&escape_label(label.value)); + out.push('"'); + } + out.push('}'); +} + +fn serve_http(listen: &str, shared: Arc>) -> Result<(), String> { + let listener = TcpListener::bind(listen).map_err(|e| format!("bind failed: {listen}: {e}"))?; + for stream in listener.incoming() { + match stream { + Ok(mut stream) => { + let snapshot = shared.read().expect("metrics lock poisoned").clone(); + if let Err(err) = handle_http_stream(&mut stream, &snapshot) { + eprintln!("http request failed: {err}"); + } + } + Err(err) => eprintln!("accept failed: {err}"), + } + } + Ok(()) +} + +fn handle_http_stream(stream: &mut TcpStream, snapshot: &MetricsSnapshot) -> Result<(), String> { + let mut buf = [0u8; 4096]; + let len = stream.read(&mut buf).map_err(|e| e.to_string())?; + let req = String::from_utf8_lossy(&buf[..len]); + let path = req + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .unwrap_or("/"); + match path { + "/metrics" => write_http_response( + stream, + "200 OK", + "text/plain; version=0.0.4", + &render_metrics(snapshot), + ), + "/status" => write_http_response( + stream, + "200 OK", + "application/json", + &render_status_json(snapshot)?, + ), + "/healthz" => write_http_response(stream, "200 OK", "text/plain", "ok\n"), + _ => write_http_response(stream, "404 Not Found", "text/plain", "not found\n"), + } +} + +fn write_http_response( + stream: &mut TcpStream, + status: &str, + content_type: &str, + body: &str, +) -> Result<(), String> { + let header = format!( + "HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.as_bytes().len() + ); + stream + .write_all(header.as_bytes()) + .map_err(|e| e.to_string())?; + stream.write_all(body.as_bytes()).map_err(|e| e.to_string()) +} + +fn write_file(path: &Path, bytes: &[u8]) -> Result<(), String> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .map_err(|e| format!("create parent failed: {}: {e}", parent.display()))?; + } + fs::write(path, bytes).map_err(|e| format!("write failed: {}: {e}", path.display())) +} + +fn extract_stage_seconds(value: Option<&Value>) -> BTreeMap { + let mut out = BTreeMap::new(); + let Some(value) = value else { + return out; + }; + let mapping = [ + ("validation_ms", "validation"), + ("report_build_ms", "report_build"), + ("report_write_ms", "report_write"), + ("ccr_build_ms", "ccr_build"), + ("ccr_write_ms", "ccr_write"), + ("compare_view_build_ms", "compare_view_build"), + ("compare_view_write_ms", "compare_view_write"), + ("cir_build_cir_ms", "cir_build"), + ("cir_write_cir_ms", "cir_write"), + ("cir_total_ms", "cir_total"), + ("total_ms", "total"), + ("repo_sync_ms_total", "repo_sync_total"), + ("rrdp_download_ms_total", "rrdp_download_total"), + ("rsync_download_ms_total", "rsync_download_total"), + ]; + for (field, stage) in mapping { + if let Some(ms) = json_u64(value, &[field]) { + out.insert(stage.to_string(), ms as f64 / 1000.0); + } + } + out +} + +fn extract_count_duration_map(value: Option<&Value>) -> BTreeMap { + let mut out = BTreeMap::new(); + let Some(object) = value.and_then(|v| v.as_object()) else { + return out; + }; + for (key, value) in object { + out.insert( + key.clone(), + CountDuration { + count: json_u64(value, &["count"]).unwrap_or(0), + duration_seconds_total: json_u64(value, &["duration_ms_total"]).unwrap_or(0) as f64 + / 1000.0, + }, + ); + } + out +} + +fn extract_artifact_sizes(value: Option<&Value>) -> BTreeMap { + let mut out = BTreeMap::new(); + for item in value.and_then(|v| v.as_array()).into_iter().flatten() { + let artifact = json_str(item, &["type"]) + .or_else(|| { + json_str(item, &["path"]) + .and_then(|path| Path::new(path).file_name().and_then(|name| name.to_str())) + }) + .unwrap_or("unknown"); + let size = json_u64(item, &["sizeBytes"]) + .or_else(|| json_u64(item, &["size"])) + .unwrap_or(0); + *out.entry(artifact.to_string()).or_default() += size; + } + out +} + +fn extract_path_sizes(value: Option<&Value>) -> BTreeMap { + let mut out = BTreeMap::new(); + for item in value.and_then(|v| v.as_array()).into_iter().flatten() { + let label = json_str(item, &["label"]).unwrap_or("unknown").to_string(); + out.insert( + label, + PathSize { + total_size_bytes: json_u64(item, &["totalSizeBytes"]).unwrap_or(0), + file_count: json_u64(item, &["fileCount"]).unwrap_or(0), + dir_count: json_u64(item, &["dirCount"]).unwrap_or(0), + }, + ); + } + out +} + +fn run_index_from_path(path: &Path) -> Option { + path.file_name() + .and_then(|name| name.to_str()) + .and_then(|name| name.strip_prefix("run_")) + .and_then(|value| value.parse::().ok()) +} + +fn json_str<'a>(value: &'a Value, path: &[&str]) -> Option<&'a str> { + let mut current = value; + for key in path { + current = current.get(*key)?; + } + current.as_str() +} + +fn json_u64(value: &Value, path: &[&str]) -> Option { + let mut current = value; + for key in path { + current = current.get(*key)?; + } + current.as_u64() +} + +fn json_i64(value: &Value, path: &[&str]) -> Option { + let mut current = value; + for key in path { + current = current.get(*key)?; + } + current.as_i64() +} + +fn json_f64(value: &Value, path: &[&str]) -> Option { + let mut current = value; + for key in path { + current = current.get(*key)?; + } + current + .as_f64() + .or_else(|| current.as_u64().map(|v| v as f64)) +} + +fn parse_rfc3339_to_unix(value: &str) -> Option { + time::OffsetDateTime::parse(value, &time::format_description::well_known::Rfc3339) + .ok() + .map(|dt| dt.unix_timestamp() as f64) +} + +fn unix_now_seconds() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0) +} + +fn bool_value(value: bool) -> f64 { + if value { 1.0 } else { 0.0 } +} + +fn normalize_transport(value: &str) -> String { + let lower = value.to_ascii_lowercase(); + if lower.contains("rrdp") || lower.contains("https") { + "rrdp".to_string() + } else if lower.contains("rsync") { + "rsync".to_string() + } else { + lower + } +} + +fn infer_transport(uri: &str) -> String { + if uri.starts_with("http://") || uri.starts_with("https://") { + "rrdp".to_string() + } else if uri.starts_with("rsync://") { + "rsync".to_string() + } else { + "unknown".to_string() + } +} + +fn uri_host(uri: &str) -> String { + let without_scheme = uri.split_once("://").map(|(_, rest)| rest).unwrap_or(uri); + without_scheme + .split('/') + .next() + .filter(|s| !s.is_empty()) + .unwrap_or("unknown") + .to_string() +} + +fn object_type_from_uri(uri: &str) -> String { + let lower = uri.to_ascii_lowercase(); + for (suffix, kind) in [ + (".mft", "manifest"), + (".crl", "crl"), + (".cer", "certificate"), + (".roa", "roa"), + (".asa", "aspa"), + (".gbr", "gbr"), + ] { + if lower.ends_with(suffix) { + return kind.to_string(); + } + } + "other".to_string() +} + +fn short_sha256(value: &str) -> String { + let digest = Sha256::digest(value.as_bytes()); + hex::encode(&digest[..6]) +} + +fn common_prefix_len(left: &str, right: &str) -> usize { + left.bytes() + .zip(right.bytes()) + .take_while(|(l, r)| l == r) + .count() +} + +fn format_prom_value(value: f64) -> String { + if value.is_infinite() && value.is_sign_positive() { + "+Inf".to_string() + } else if value.fract() == 0.0 { + format!("{value:.0}") + } else { + format!("{value:.6}") + .trim_end_matches('0') + .trim_end_matches('.') + .to_string() + } +} + +fn escape_label(value: &str) -> String { + value + .replace('\\', "\\\\") + .replace('\n', "\\n") + .replace('"', "\\\"") +} + +fn escape_help(value: &str) -> String { + value.replace('\\', "\\\\").replace('\n', "\\n") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ccr::model::CCR_VERSION_V0; + use crate::ccr::{ + CcrContentInfo, CcrDigestAlgorithm, RpkiCanonicalCacheRepresentation, TrustAnchorState, + encode_content_info, + }; + use crate::cir::{ + CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirRejectedObject, + CirTrustAnchor, compute_reject_list_sha256, encode_cir, sha256, + }; + use tempfile::TempDir; + + #[test] + fn parse_args_accepts_once_outputs() { + let args = parse_args(&[ + "rpki_artifact_metrics".to_string(), + "--run-root".to_string(), + "root".to_string(), + "--once".to_string(), + "--out-metrics".to_string(), + "metrics.prom".to_string(), + "--out-status".to_string(), + "status.json".to_string(), + ]) + .expect("parse"); + assert!(args.once); + assert_eq!(args.run_root, PathBuf::from("root")); + assert_eq!(args.out_metrics.as_deref(), Some(Path::new("metrics.prom"))); + } + + #[test] + fn scan_fixture_exports_repo_pp_cir_and_ccr_metrics() { + let td = TempDir::new().expect("tempdir"); + let run = td.path().join("runs/run_0001"); + fs::create_dir_all(&run).expect("create run"); + fs::write( + run.join("run-meta.json"), + r#"{"status":"success","run_index":1,"run_id":"run_0001","sync_mode":"snapshot","snapshot_reason":"first_run","started_at_rfc3339_utc":"2026-05-25T00:00:00Z","completed_at_rfc3339_utc":"2026-05-25T00:00:10Z"}"#, + ) + .expect("meta"); + fs::write( + run.join("run-summary.json"), + r#"{"runSeq":1,"runId":"run_0001","runDir":"RUN","startedAtRfc3339Utc":"2026-05-25T00:00:00Z","finishedAtRfc3339Utc":"2026-05-25T00:00:10Z","wallMs":10000,"status":"success","exitCode":0,"processMetrics":{"userSeconds":2.5,"systemSeconds":1.5,"cpuPercent":40,"maxRssKb":1000},"stageTiming":{"validation_ms":7000,"total_ms":9000,"download_event_count":2,"download_bytes_total":1234},"reportCounts":{"vrps":2,"aspas":1,"publicationPoints":2,"warnings":0},"repoSyncStats":{"by_phase":{"rrdp_delta":{"count":2,"duration_ms_total":3000}},"by_terminal_state":{"fresh":{"count":2,"duration_ms_total":3000}}},"pathStats":[{"label":"work-db","totalSizeBytes":99,"fileCount":2,"dirCount":1}],"artifacts":[{"path":"report.json","sizeBytes":10}]}"#, + ) + .expect("summary"); + fs::write(run.join("process-time.txt"), "time").expect("time"); + fs::write(run.join("stage-timing.json"), "{}").expect("stage"); + fs::write( + run.join("report.json"), + r#"{"tree":{"instances_processed":2,"instances_failed":0,"warnings":[]},"vrps":[{},{}],"aspas":[{}],"downloads":[{"kind":"rrdp_notification","uri":"https://repo.example/notify.xml","success":true,"duration_ms":100,"bytes":111},{"kind":"rrdp_delta","uri":"https://repo.example/session/1/delta.xml","success":true,"duration_ms":200,"bytes":222}],"publication_points":[{"rsync_base_uri":"rsync://repo.example/a/","manifest_rsync_uri":"rsync://repo.example/a/a.mft","publication_point_rsync_uri":"rsync://repo.example/a/","rrdp_notification_uri":"https://repo.example/notify.xml","repo_sync_source":"rrdp","repo_sync_phase":"rrdp_delta","repo_sync_duration_ms":1000,"repo_terminal_state":"fresh","objects":[{"kind":"roa","result":"ok"},{"kind":"manifest","result":"ok"}]},{"rsync_base_uri":"rsync://repo.example/b/","manifest_rsync_uri":"rsync://repo.example/b/b.mft","publication_point_rsync_uri":"rsync://repo.example/b/","rrdp_notification_uri":"https://repo.example/notify.xml","repo_sync_source":"rrdp","repo_sync_phase":"rrdp_delta","repo_sync_duration_ms":2000,"repo_terminal_state":"fresh","objects":[{"kind":"roa","result":"ok"}]}],"repo_sync_stats":{"publication_points_total":2,"by_phase":{"rrdp_delta":{"count":2,"duration_ms_total":3000}},"by_terminal_state":{"fresh":{"count":2,"duration_ms_total":3000}}}}"#, + ) + .expect("report"); + fs::write(run.join("input.cir"), sample_cir()).expect("cir"); + fs::write(run.join("result.ccr"), sample_ccr()).expect("ccr"); + + let snapshot = scan_run_root(td.path(), "test").expect("scan"); + assert_eq!(snapshot.runs.success, 1); + assert_eq!(snapshot.repo_stats.len(), 1); + assert!(snapshot.repo_stats[0].sync_success); + assert_eq!(snapshot.repo_stats[0].download_bytes, 333); + assert_eq!(snapshot.top_pp_by_object_count[0].object_count, 2); + assert_eq!(snapshot.cir.as_ref().unwrap().objects, 1); + assert_eq!(snapshot.ccr.as_ref().unwrap().state_items["tas"], 1); + let metrics = render_metrics(&snapshot); + assert!(metrics.contains("ours_rp_repository_info")); + assert!(metrics.contains("ours_rp_repository_sync_success")); + assert!(metrics.contains("ours_rp_repository_download_bytes")); + assert!(metrics.contains("ours_rp_large_publication_points")); + assert!(metrics.contains("ours_rp_cir_objects")); + assert!(metrics.contains("ours_rp_ccr_state_items")); + let status = render_status_json(&snapshot).expect("status"); + assert!(status.contains("topPublicationPointsByObjectCount")); + } + + #[test] + fn partial_run_does_not_become_latest_success() { + let td = TempDir::new().expect("tempdir"); + let run = td.path().join("runs/run_0001"); + fs::create_dir_all(&run).expect("create run"); + fs::write(run.join("run-meta.json"), r#"{"status":"running"}"#).expect("meta"); + let snapshot = scan_run_root(td.path(), "test").expect("scan"); + assert_eq!(snapshot.runs.partial, 1); + assert!(snapshot.latest_run.is_none()); + } + + fn sample_cir() -> Vec { + let rejected = vec![CirRejectedObject { + object_uri: "rsync://repo.example/a/bad.roa".to_string(), + reason: Some("bad".to_string()), + }]; + let cir = CanonicalInputRepresentation { + version: crate::cir::CIR_VERSION_V3, + hash_alg: CirHashAlgorithm::Sha256, + validation_time: time::OffsetDateTime::parse( + "2026-05-25T00:00:00Z", + &time::format_description::well_known::Rfc3339, + ) + .unwrap(), + objects: vec![CirObject { + rsync_uri: "rsync://repo.example/a/a.roa".to_string(), + sha256: vec![1; 32], + }], + trust_anchors: vec![CirTrustAnchor { + ta_rsync_uri: "rsync://repo.example/ta.cer".to_string(), + tal_uri: "https://tal.example/tal.tal".to_string(), + tal_bytes: b"rsync://repo.example/ta.cer\n\nAQID\n".to_vec(), + ta_certificate_der: b"ta".to_vec(), + ta_certificate_sha256: sha256(b"ta"), + }], + reject_list_sha256: compute_reject_list_sha256( + rejected.iter().map(|item| item.object_uri.as_str()), + ), + rejected_objects: rejected, + }; + encode_cir(&cir).expect("encode cir") + } + + fn sample_ccr() -> Vec { + let ci = CcrContentInfo::new(RpkiCanonicalCacheRepresentation { + version: CCR_VERSION_V0, + hash_alg: CcrDigestAlgorithm::Sha256, + produced_at: time::OffsetDateTime::parse( + "2026-05-25T00:00:00Z", + &time::format_description::well_known::Rfc3339, + ) + .unwrap(), + mfts: None, + vrps: None, + vaps: None, + tas: Some(TrustAnchorState { + skis: vec![vec![1; 20]], + hash: vec![2; 32], + }), + rks: None, + }); + encode_content_info(&ci).expect("encode ccr") + } +} diff --git a/src/tools/rpki_daemon.rs b/src/tools/rpki_daemon.rs new file mode 100644 index 0000000..3d2d8b8 --- /dev/null +++ b/src/tools/rpki_daemon.rs @@ -0,0 +1,1784 @@ +use serde::Serialize; +use std::collections::BTreeMap; +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; + +#[derive(Clone, Debug, PartialEq, Eq)] +struct Args { + state_root: PathBuf, + rpki_bin: PathBuf, + interval_secs: u64, + max_runs: Option, + retain_runs: usize, + status_json: Option, + summary_jsonl: Option, + work_db: PathBuf, + repo_bytes_db: Option, + raw_store_db: Option, + db_stats_bin: Option, + db_stats_exact_every: Option, + time_bin: Option, + child_args: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct RunContext { + seq: u64, + run_id: String, + run_dir: PathBuf, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum DaemonState { + Starting, + Idle, + Running, + Collecting, + Sleeping, + Exited, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct DaemonStatus { + state: DaemonState, + updated_at_rfc3339_utc: String, + runs_completed: u64, + max_runs: Option, + current_run_seq: Option, + current_run_id: Option, + last_run_id: Option, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum RunStatus { + Success, + Failed, + SpawnFailed, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct ArtifactInfo { + path: String, + size_bytes: u64, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct ProcessMetrics { + time_wrapper_used: bool, + time_output_path: Option, + user_seconds: Option, + system_seconds: Option, + cpu_percent: Option, + elapsed_raw: Option, + max_rss_kb: Option, + exit_status_from_time: Option, + parse_error: Option, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct ReportCounts { + vrps: usize, + aspas: usize, + publication_points: usize, + rrdp_repos_unique: Option, + tree_instances_processed: Option, + tree_instances_failed: Option, + warnings: usize, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct PathFileStats { + label: String, + path: String, + exists: bool, + is_dir: bool, + total_size_bytes: u64, + file_count: u64, + dir_count: u64, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct DbStatsSummary { + mode: String, + db_path: String, + output_path: Option, + stderr_path: Option, + status: String, + exit_code: Option, + error: Option, + metrics: BTreeMap, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct RunSummary { + run_seq: u64, + run_id: String, + run_dir: String, + started_at_rfc3339_utc: String, + finished_at_rfc3339_utc: String, + wall_ms: u64, + status: RunStatus, + exit_code: Option, + exit_status: Option, + error: Option, + rpki_bin: String, + child_args: Vec, + stdout_path: String, + stderr_path: String, + process_metrics: Option, + stage_timing: Option, + report_counts: Option, + repo_sync_stats: Option, + path_stats: Vec, + db_stats: Vec, + retention_deleted_runs: Vec, + artifacts: Vec, +} + +fn usage() -> String { + let bin = "rpki_daemon"; + format!( + "\ +Usage: + {bin} --state-root --rpki-bin [options] -- + +Options: + --state-root Persistent daemon root containing state/, runs/, status, and JSONL summary + --rpki-bin rpki child binary to execute for each run + --interval-secs Sleep seconds between runs (default: 60) + --max-runs Stop after n runs (default: run forever) + --retain-runs Keep only the latest n run directories (default: 10) + --status-json Override status JSON path (default: /daemon-status.json) + --summary-jsonl Override summary JSONL path (default: /daemon-runs.jsonl) + --work-db Work DB path for metrics (default: /state/work-db) + --repo-bytes-db Repo bytes DB path for file metrics (default: /state/repo-bytes.db) + --raw-store-db Raw store DB path for file metrics (optional) + --db-stats-bin db_stats binary path (default: sibling db_stats next to this executable when present) + --db-stats-exact-every + Run db_stats --exact every n runs (default: disabled) + --time-bin GNU time binary for child process metrics (default: /usr/bin/time when present) + --no-time-wrapper Disable GNU time wrapper + --help Show this help + +Child argument placeholders: + {{state_root}} Daemon state root + {{run_out}} Current run output directory + {{run_id}} Current run id, e.g. 000001-20260428T090000Z + {{run_seq}} Current run sequence number +" + ) +} + +fn default_time_bin() -> Option { + let path = PathBuf::from("/usr/bin/time"); + if path.is_file() { Some(path) } else { None } +} + +fn default_db_stats_bin() -> Option { + let mut path = std::env::current_exe().ok()?; + path.set_file_name("db_stats"); + if path.is_file() { Some(path) } else { None } +} + +fn parse_args(argv: &[String]) -> Result { + if argv.iter().any(|arg| arg == "--help" || arg == "-h") { + return Err(usage()); + } + + let mut state_root: Option = None; + let mut rpki_bin: Option = None; + let mut interval_secs = 60u64; + let mut max_runs = None; + let mut retain_runs = 10usize; + let mut status_json: Option = None; + let mut summary_jsonl: Option = None; + let mut work_db: Option = None; + let mut repo_bytes_db: Option = None; + let mut raw_store_db: Option = None; + let mut db_stats_bin: Option = None; + let mut db_stats_exact_every = None; + let mut time_bin = default_time_bin(); + let mut no_time_wrapper = false; + + let mut i = 1usize; + while i < argv.len() { + match argv[i].as_str() { + "--" => { + let child_args = argv[i + 1..].to_vec(); + let state_root = + state_root.ok_or_else(|| format!("--state-root is required\n\n{}", usage()))?; + let work_db = work_db.unwrap_or_else(|| state_root.join("state").join("work-db")); + let repo_bytes_db = + repo_bytes_db.or_else(|| Some(state_root.join("state").join("repo-bytes.db"))); + if no_time_wrapper { + time_bin = None; + } + let args = Args { + state_root, + rpki_bin: rpki_bin + .ok_or_else(|| format!("--rpki-bin is required\n\n{}", usage()))?, + interval_secs, + max_runs, + retain_runs, + status_json, + summary_jsonl, + work_db, + repo_bytes_db, + raw_store_db, + db_stats_bin, + db_stats_exact_every, + time_bin, + child_args, + }; + return validate_args(args); + } + "--state-root" => { + i += 1; + state_root = Some(PathBuf::from(value_at(argv, i, "--state-root")?)); + } + "--rpki-bin" => { + i += 1; + rpki_bin = Some(PathBuf::from(value_at(argv, i, "--rpki-bin")?)); + } + "--interval-secs" => { + i += 1; + interval_secs = + parse_u64(value_at(argv, i, "--interval-secs")?, "--interval-secs")?; + } + "--max-runs" => { + i += 1; + let parsed = parse_u64(value_at(argv, i, "--max-runs")?, "--max-runs")?; + if parsed == 0 { + return Err("--max-runs must be > 0".to_string()); + } + max_runs = Some(parsed); + } + "--retain-runs" => { + i += 1; + let parsed = parse_usize(value_at(argv, i, "--retain-runs")?, "--retain-runs")?; + if parsed == 0 { + return Err("--retain-runs must be > 0".to_string()); + } + retain_runs = parsed; + } + "--status-json" => { + i += 1; + status_json = Some(PathBuf::from(value_at(argv, i, "--status-json")?)); + } + "--summary-jsonl" => { + i += 1; + summary_jsonl = Some(PathBuf::from(value_at(argv, i, "--summary-jsonl")?)); + } + "--work-db" => { + i += 1; + work_db = Some(PathBuf::from(value_at(argv, i, "--work-db")?)); + } + "--repo-bytes-db" => { + i += 1; + repo_bytes_db = Some(PathBuf::from(value_at(argv, i, "--repo-bytes-db")?)); + } + "--raw-store-db" => { + i += 1; + raw_store_db = Some(PathBuf::from(value_at(argv, i, "--raw-store-db")?)); + } + "--db-stats-bin" => { + i += 1; + db_stats_bin = Some(PathBuf::from(value_at(argv, i, "--db-stats-bin")?)); + } + "--db-stats-exact-every" => { + i += 1; + let parsed = parse_u64( + value_at(argv, i, "--db-stats-exact-every")?, + "--db-stats-exact-every", + )?; + if parsed == 0 { + return Err("--db-stats-exact-every must be > 0".to_string()); + } + db_stats_exact_every = Some(parsed); + } + "--time-bin" => { + i += 1; + time_bin = Some(PathBuf::from(value_at(argv, i, "--time-bin")?)); + } + "--no-time-wrapper" => { + no_time_wrapper = true; + } + other => return Err(format!("unknown argument: {other}\n\n{}", usage())), + } + i += 1; + } + + Err(format!("missing -- before child rpki args\n\n{}", usage())) +} + +fn validate_args(args: Args) -> Result { + if args.child_args.is_empty() { + return Err(format!( + "child rpki args are required after --\n\n{}", + usage() + )); + } + Ok(args) +} + +fn value_at<'a>(argv: &'a [String], index: usize, flag: &str) -> Result<&'a str, String> { + argv.get(index) + .map(String::as_str) + .ok_or_else(|| format!("{flag} requires a value")) +} + +fn parse_u64(raw: &str, flag: &str) -> Result { + raw.parse::() + .map_err(|_| format!("invalid {flag}: {raw}")) +} + +fn parse_usize(raw: &str, flag: &str) -> Result { + raw.parse::() + .map_err(|_| format!("invalid {flag}: {raw}")) +} + +fn status_path(args: &Args) -> PathBuf { + args.status_json + .clone() + .unwrap_or_else(|| args.state_root.join("daemon-status.json")) +} + +fn summary_jsonl_path(args: &Args) -> PathBuf { + args.summary_jsonl + .clone() + .unwrap_or_else(|| args.state_root.join("daemon-runs.jsonl")) +} + +fn utc_now() -> time::OffsetDateTime { + time::OffsetDateTime::now_utc().to_offset(time::UtcOffset::UTC) +} + +fn format_rfc3339(t: time::OffsetDateTime) -> Result { + t.format(&time::format_description::well_known::Rfc3339) + .map_err(|e| format!("format RFC3339 failed: {e}")) +} + +fn format_compact_utc(t: time::OffsetDateTime) -> String { + format!( + "{:04}{:02}{:02}T{:02}{:02}{:02}Z", + t.year(), + u8::from(t.month()), + t.day(), + t.hour(), + t.minute(), + t.second() + ) +} + +fn write_json_pretty(path: &Path, value: &T) -> Result<(), String> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .map_err(|e| format!("create parent dir failed: {}: {e}", parent.display()))?; + } + let file = + File::create(path).map_err(|e| format!("create json failed: {}: {e}", path.display()))?; + serde_json::to_writer_pretty(file, value) + .map_err(|e| format!("write json failed: {}: {e}", path.display())) +} + +fn append_json_line(path: &Path, value: &T) -> Result<(), String> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .map_err(|e| format!("create parent dir failed: {}: {e}", parent.display()))?; + } + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(path) + .map_err(|e| format!("open jsonl failed: {}: {e}", path.display()))?; + serde_json::to_writer(&mut file, value) + .map_err(|e| format!("write jsonl failed: {}: {e}", path.display()))?; + file.write_all(b"\n") + .map_err(|e| format!("flush jsonl failed: {}: {e}", path.display())) +} + +fn write_status( + args: &Args, + state: DaemonState, + runs_completed: u64, + current: Option<&RunContext>, + last_run_id: Option, +) -> Result<(), String> { + let updated_at_rfc3339_utc = format_rfc3339(utc_now())?; + let status = DaemonStatus { + state, + updated_at_rfc3339_utc, + runs_completed, + max_runs: args.max_runs, + current_run_seq: current.map(|ctx| ctx.seq), + current_run_id: current.map(|ctx| ctx.run_id.clone()), + last_run_id, + }; + write_json_pretty(&status_path(args), &status) +} + +fn render_child_args(args: &[String], daemon_args: &Args, ctx: &RunContext) -> Vec { + args.iter() + .map(|arg| { + arg.replace("{state_root}", &path_string(&daemon_args.state_root)) + .replace("{run_out}", &path_string(&ctx.run_dir)) + .replace("{run_id}", &ctx.run_id) + .replace("{run_seq}", &ctx.seq.to_string()) + }) + .collect() +} + +fn render_path_template(path: &Path, daemon_args: &Args, ctx: &RunContext) -> PathBuf { + PathBuf::from( + path_string(path) + .replace("{state_root}", &path_string(&daemon_args.state_root)) + .replace("{run_out}", &path_string(&ctx.run_dir)) + .replace("{run_id}", &ctx.run_id) + .replace("{run_seq}", &ctx.seq.to_string()), + ) +} + +fn path_string(path: &Path) -> String { + path.to_string_lossy().into_owned() +} + +fn make_run_context(args: &Args, seq: u64, now: time::OffsetDateTime) -> RunContext { + let run_id = format!("{seq:06}-{}", format_compact_utc(now)); + let run_dir = args.state_root.join("runs").join(&run_id); + RunContext { + seq, + run_id, + run_dir, + } +} + +fn collect_artifacts(run_dir: &Path) -> Result, String> { + let mut artifacts = Vec::new(); + for entry in fs::read_dir(run_dir) + .map_err(|e| format!("read run dir failed: {}: {e}", run_dir.display()))? + { + let entry = entry.map_err(|e| format!("read run dir entry failed: {e}"))?; + if !entry + .file_type() + .map_err(|e| format!("read file type failed: {}: {e}", entry.path().display()))? + .is_file() + { + continue; + } + let metadata = entry + .metadata() + .map_err(|e| format!("read metadata failed: {}: {e}", entry.path().display()))?; + artifacts.push(ArtifactInfo { + path: path_string(&entry.path()), + size_bytes: metadata.len(), + }); + } + artifacts.sort_by(|a, b| a.path.cmp(&b.path)); + Ok(artifacts) +} + +fn collect_process_metrics(time_wrapper_used: bool, time_output_path: &Path) -> ProcessMetrics { + if !time_wrapper_used { + return ProcessMetrics { + time_wrapper_used, + time_output_path: None, + user_seconds: None, + system_seconds: None, + cpu_percent: None, + elapsed_raw: None, + max_rss_kb: None, + exit_status_from_time: None, + parse_error: None, + }; + } + + let mut metrics = ProcessMetrics { + time_wrapper_used, + time_output_path: Some(path_string(time_output_path)), + user_seconds: None, + system_seconds: None, + cpu_percent: None, + elapsed_raw: None, + max_rss_kb: None, + exit_status_from_time: None, + parse_error: None, + }; + + let text = match fs::read_to_string(time_output_path) { + Ok(text) => text, + Err(err) => { + metrics.parse_error = Some(format!( + "read process time output failed: {}: {err}", + time_output_path.display() + )); + return metrics; + } + }; + + for line in text.lines() { + let line = line.trim(); + if let Some(value) = line.strip_prefix("User time (seconds):") { + metrics.user_seconds = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("System time (seconds):") { + metrics.system_seconds = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("Percent of CPU this job got:") { + metrics.cpu_percent = value.trim().trim_end_matches('%').parse::().ok(); + } else if let Some(value) = + line.strip_prefix("Elapsed (wall clock) time (h:mm:ss or m:ss):") + { + metrics.elapsed_raw = Some(value.trim().to_string()); + } else if let Some(value) = line.strip_prefix("Maximum resident set size (kbytes):") { + metrics.max_rss_kb = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("Exit status:") { + metrics.exit_status_from_time = value.trim().parse::().ok(); + } + } + metrics +} + +fn run_child_once(args: &Args, ctx: &RunContext) -> Result { + fs::create_dir_all(&ctx.run_dir) + .map_err(|e| format!("create run dir failed: {}: {e}", ctx.run_dir.display()))?; + + let started_at = utc_now(); + let started_at_rfc3339_utc = format_rfc3339(started_at)?; + let stdout_path = ctx.run_dir.join("stdout.log"); + let stderr_path = ctx.run_dir.join("stderr.log"); + let stdout = File::create(&stdout_path) + .map_err(|e| format!("create stdout log failed: {}: {e}", stdout_path.display()))?; + let stderr = File::create(&stderr_path) + .map_err(|e| format!("create stderr log failed: {}: {e}", stderr_path.display()))?; + let child_args = render_child_args(&args.child_args, args, ctx); + let time_output_path = ctx.run_dir.join("process-time.txt"); + + let mut command = if let Some(time_bin) = args.time_bin.as_ref() { + let mut command = Command::new(time_bin); + command + .arg("-v") + .arg("-o") + .arg(&time_output_path) + .arg("--") + .arg(&args.rpki_bin) + .args(&child_args); + command + } else { + let mut command = Command::new(&args.rpki_bin); + command.args(&child_args); + command + }; + command + .stdout(Stdio::from(stdout)) + .stderr(Stdio::from(stderr)); + + let (status, exit_code, exit_status, error) = match command.status() { + Ok(status) if status.success() => ( + RunStatus::Success, + status.code(), + Some(status.to_string()), + None, + ), + Ok(status) => ( + RunStatus::Failed, + status.code(), + Some(status.to_string()), + None, + ), + Err(err) => ( + RunStatus::SpawnFailed, + None, + None, + Some(format!("spawn child failed: {err}")), + ), + }; + + let finished_at = utc_now(); + let finished_at_rfc3339_utc = format_rfc3339(finished_at)?; + let wall_ms = (finished_at - started_at).whole_milliseconds().max(0) as u64; + let process_metrics = Some(collect_process_metrics( + args.time_bin.is_some(), + &time_output_path, + )); + let artifacts = collect_artifacts(&ctx.run_dir)?; + let summary = RunSummary { + run_seq: ctx.seq, + run_id: ctx.run_id.clone(), + run_dir: path_string(&ctx.run_dir), + started_at_rfc3339_utc, + finished_at_rfc3339_utc, + wall_ms, + status, + exit_code, + exit_status, + error, + rpki_bin: path_string(&args.rpki_bin), + child_args, + stdout_path: path_string(&stdout_path), + stderr_path: path_string(&stderr_path), + process_metrics, + stage_timing: None, + report_counts: None, + repo_sync_stats: None, + path_stats: Vec::new(), + db_stats: Vec::new(), + retention_deleted_runs: Vec::new(), + artifacts, + }; + Ok(summary) +} + +fn apply_retention(runs_root: &Path, retain_runs: usize) -> Result, String> { + if !runs_root.exists() { + return Ok(Vec::new()); + } + let mut dirs = Vec::new(); + for entry in fs::read_dir(runs_root) + .map_err(|e| format!("read runs dir failed: {}: {e}", runs_root.display()))? + { + let entry = entry.map_err(|e| format!("read runs dir entry failed: {e}"))?; + if entry + .file_type() + .map_err(|e| format!("read file type failed: {}: {e}", entry.path().display()))? + .is_dir() + { + dirs.push(entry.path()); + } + } + dirs.sort(); + let remove_count = dirs.len().saturating_sub(retain_runs); + let mut removed = Vec::new(); + for dir in dirs.into_iter().take(remove_count) { + fs::remove_dir_all(&dir) + .map_err(|e| format!("remove old run dir failed: {}: {e}", dir.display()))?; + removed.push(dir); + } + Ok(removed) +} + +fn find_named_file(root: &Path, name: &str) -> Option { + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + let entries = fs::read_dir(&dir).ok()?; + for entry in entries.flatten() { + let path = entry.path(); + let file_type = entry.file_type().ok()?; + if file_type.is_file() && entry.file_name().to_string_lossy() == name { + return Some(path); + } + if file_type.is_dir() { + stack.push(path); + } + } + } + None +} + +fn read_json_value_if_exists(path: &Path) -> Option { + let bytes = fs::read(path).ok()?; + serde_json::from_slice(&bytes).ok() +} + +fn json_array_len(value: &serde_json::Value, key: &str) -> usize { + value + .get(key) + .and_then(serde_json::Value::as_array) + .map(Vec::len) + .unwrap_or(0) +} + +fn parse_stdout_summary(run_dir: &Path) -> Option { + let stdout_path = run_dir.join("stdout.log"); + let text = fs::read_to_string(stdout_path).ok()?; + let mut vrps = None; + let mut aspas = None; + let mut publication_points = None; + let mut rrdp_repos_unique = None; + let mut tree_instances_processed = None; + let mut tree_instances_failed = None; + let mut warnings = None; + + for line in text.lines() { + if let Some(value) = line.strip_prefix("vrps=") { + vrps = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("aspas=") { + aspas = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("audit_publication_points=") { + publication_points = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("rrdp_repos_unique=") { + rrdp_repos_unique = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("warnings_total=") { + warnings = value.trim().parse::().ok(); + } else if let Some(rest) = line.strip_prefix("publication_points_processed=") { + for token in rest.split_whitespace() { + if let Some(value) = token.strip_prefix("publication_points_failed=") { + tree_instances_failed = value.parse::().ok(); + } else if tree_instances_processed.is_none() { + tree_instances_processed = token.parse::().ok(); + } + } + } + } + + Some(ReportCounts { + vrps: vrps?, + aspas: aspas?, + publication_points: publication_points?, + rrdp_repos_unique, + tree_instances_processed, + tree_instances_failed, + warnings: warnings.unwrap_or(0), + }) +} + +fn parse_report_counts_fallback(report: &serde_json::Value) -> ReportCounts { + let tree = report.get("tree"); + let tree_warnings = tree + .and_then(|tree| tree.get("warnings")) + .and_then(serde_json::Value::as_array) + .map(Vec::len) + .unwrap_or(0); + let pp_warnings = report + .get("publication_points") + .and_then(serde_json::Value::as_array) + .map(|items| { + items + .iter() + .map(|pp| { + pp.get("warnings") + .and_then(serde_json::Value::as_array) + .map(Vec::len) + .unwrap_or(0) + }) + .sum() + }) + .unwrap_or(0); + ReportCounts { + vrps: json_array_len(report, "vrps"), + aspas: json_array_len(report, "aspas"), + publication_points: json_array_len(report, "publication_points"), + rrdp_repos_unique: None, + tree_instances_processed: tree + .and_then(|tree| tree.get("instances_processed")) + .and_then(serde_json::Value::as_u64), + tree_instances_failed: tree + .and_then(|tree| tree.get("instances_failed")) + .and_then(serde_json::Value::as_u64), + warnings: tree_warnings + pp_warnings, + } +} + +fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +fn extract_json_object_field(path: &Path, field_name: &str) -> Option { + let bytes = fs::read(path).ok()?; + let needle = format!("\"{field_name}\":"); + let pos = find_subslice(&bytes, needle.as_bytes())?; + let mut i = pos + needle.len(); + while i < bytes.len() && bytes[i].is_ascii_whitespace() { + i += 1; + } + if bytes.get(i).copied()? != b'{' { + return None; + } + let start = i; + let mut depth = 0u32; + let mut in_string = false; + let mut escaped = false; + for (offset, &b) in bytes[start..].iter().enumerate() { + if in_string { + if escaped { + escaped = false; + } else if b == b'\\' { + escaped = true; + } else if b == b'"' { + in_string = false; + } + continue; + } + match b { + b'"' => in_string = true, + b'{' => depth = depth.saturating_add(1), + b'}' => { + depth = depth.saturating_sub(1); + if depth == 0 { + let end = start + offset + 1; + return serde_json::from_slice(&bytes[start..end]).ok(); + } + } + _ => {} + } + } + None +} + +fn parse_report_metadata(run_dir: &Path) -> (Option, Option) { + let Some(report_path) = find_named_file(run_dir, "report.json") else { + return (parse_stdout_summary(run_dir), None); + }; + let counts = parse_stdout_summary(run_dir).or_else(|| { + read_json_value_if_exists(&report_path).map(|report| parse_report_counts_fallback(&report)) + }); + let repo_sync_stats = extract_json_object_field(&report_path, "repo_sync_stats"); + (counts, repo_sync_stats) +} + +fn collect_path_file_stats(label: &str, path: &Path) -> PathFileStats { + let mut stats = PathFileStats { + label: label.to_string(), + path: path_string(path), + exists: path.exists(), + is_dir: path.is_dir(), + total_size_bytes: 0, + file_count: 0, + dir_count: 0, + }; + if !stats.exists { + return stats; + } + if path.is_file() { + if let Ok(metadata) = path.metadata() { + stats.total_size_bytes = metadata.len(); + stats.file_count = 1; + } + return stats; + } + + let mut stack = vec![path.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let Ok(file_type) = entry.file_type() else { + continue; + }; + if file_type.is_dir() { + stats.dir_count = stats.dir_count.saturating_add(1); + stack.push(entry.path()); + } else if file_type.is_file() { + stats.file_count = stats.file_count.saturating_add(1); + if let Ok(metadata) = entry.metadata() { + stats.total_size_bytes = stats.total_size_bytes.saturating_add(metadata.len()); + } + } + } + } + stats +} + +fn collect_state_path_stats(args: &Args, ctx: &RunContext) -> Vec { + let mut stats = Vec::new(); + stats.push(collect_path_file_stats( + "work_db", + &render_path_template(&args.work_db, args, ctx), + )); + if let Some(path) = args.repo_bytes_db.as_ref() { + stats.push(collect_path_file_stats( + "repo_bytes_db", + &render_path_template(path, args, ctx), + )); + } + if let Some(path) = args.raw_store_db.as_ref() { + stats.push(collect_path_file_stats( + "raw_store_db", + &render_path_template(path, args, ctx), + )); + } + stats +} + +fn parse_key_value_metrics(text: &str) -> BTreeMap { + let mut metrics = BTreeMap::new(); + for line in text.lines() { + let Some((key, value)) = line.split_once('=') else { + continue; + }; + metrics.insert(key.trim().to_string(), value.trim().to_string()); + } + metrics +} + +fn run_db_stats_command( + db_stats_bin: &Path, + db_path: &Path, + run_dir: &Path, + mode: &str, +) -> DbStatsSummary { + let output_path = run_dir.join(format!("db-stats-{mode}.txt")); + let stderr_path = run_dir.join(format!("db-stats-{mode}.stderr.txt")); + let mut summary = DbStatsSummary { + mode: mode.to_string(), + db_path: path_string(db_path), + output_path: Some(path_string(&output_path)), + stderr_path: None, + status: "success".to_string(), + exit_code: None, + error: None, + metrics: BTreeMap::new(), + }; + + if !db_path.exists() { + summary.status = "skipped".to_string(); + summary.error = Some(format!("db path does not exist: {}", db_path.display())); + summary.output_path = None; + return summary; + } + + let mut command = Command::new(db_stats_bin); + command.arg("--db").arg(db_path); + if mode == "exact" { + command.arg("--exact"); + } + match command.output() { + Ok(output) => { + summary.exit_code = output.status.code(); + if !output.status.success() { + summary.status = "failed".to_string(); + } + let stdout_text = String::from_utf8_lossy(&output.stdout).into_owned(); + if let Err(err) = fs::write(&output_path, stdout_text.as_bytes()) { + summary.status = "failed".to_string(); + summary.error = Some(format!( + "write db_stats output failed: {}: {err}", + output_path.display() + )); + } + summary.metrics = parse_key_value_metrics(&stdout_text); + if !output.stderr.is_empty() { + if fs::write(&stderr_path, &output.stderr).is_ok() { + summary.stderr_path = Some(path_string(&stderr_path)); + } + } + if !output.status.success() && summary.error.is_none() { + summary.error = Some(String::from_utf8_lossy(&output.stderr).into_owned()); + } + } + Err(err) => { + summary.status = "spawn_failed".to_string(); + summary.exit_code = None; + summary.output_path = None; + summary.error = Some(format!("spawn db_stats failed: {err}")); + } + } + summary +} + +fn collect_db_stats(args: &Args, ctx: &RunContext) -> Vec { + let work_db = render_path_template(&args.work_db, args, ctx); + let Some(db_stats_bin) = args + .db_stats_bin + .as_ref() + .cloned() + .or_else(default_db_stats_bin) + else { + return vec![DbStatsSummary { + mode: "estimate".to_string(), + db_path: path_string(&work_db), + output_path: None, + stderr_path: None, + status: "skipped".to_string(), + exit_code: None, + error: Some( + "db_stats binary not configured and sibling db_stats was not found".to_string(), + ), + metrics: BTreeMap::new(), + }]; + }; + + let mut stats = Vec::new(); + stats.push(run_db_stats_command( + &db_stats_bin, + &work_db, + &ctx.run_dir, + "estimate", + )); + if args + .db_stats_exact_every + .is_some_and(|every| ctx.seq % every == 0) + { + stats.push(run_db_stats_command( + &db_stats_bin, + &work_db, + &ctx.run_dir, + "exact", + )); + } + stats +} + +fn collect_post_run_metrics(args: &Args, ctx: &RunContext, summary: &mut RunSummary) { + if let Some(path) = find_named_file(&ctx.run_dir, "stage-timing.json") { + summary.stage_timing = read_json_value_if_exists(&path); + } + let (report_counts, repo_sync_stats) = parse_report_metadata(&ctx.run_dir); + summary.report_counts = report_counts; + summary.repo_sync_stats = repo_sync_stats; + summary.path_stats = collect_state_path_stats(args, ctx); + summary.db_stats = collect_db_stats(args, ctx); + summary.artifacts = collect_artifacts(&ctx.run_dir).unwrap_or_default(); +} + +fn run_daemon(args: &Args) -> Result<(), String> { + fs::create_dir_all(args.state_root.join("state")).map_err(|e| { + format!( + "create daemon state dir failed: {}: {e}", + args.state_root.join("state").display() + ) + })?; + fs::create_dir_all(args.state_root.join("runs")).map_err(|e| { + format!( + "create daemon runs dir failed: {}: {e}", + args.state_root.join("runs").display() + ) + })?; + + let mut runs_completed = 0u64; + let mut next_seq = 1u64; + let mut last_run_id = None; + write_status( + args, + DaemonState::Starting, + runs_completed, + None, + last_run_id.clone(), + )?; + + loop { + if args.max_runs.is_some_and(|max| runs_completed >= max) { + break; + } + + write_status( + args, + DaemonState::Idle, + runs_completed, + None, + last_run_id.clone(), + )?; + let ctx = make_run_context(args, next_seq, utc_now()); + write_status( + args, + DaemonState::Running, + runs_completed, + Some(&ctx), + last_run_id.clone(), + )?; + let mut summary = run_child_once(args, &ctx)?; + write_status( + args, + DaemonState::Collecting, + runs_completed, + Some(&ctx), + last_run_id.clone(), + )?; + collect_post_run_metrics(args, &ctx, &mut summary); + let removed = apply_retention(&args.state_root.join("runs"), args.retain_runs)?; + summary.retention_deleted_runs = removed.iter().map(|p| path_string(p)).collect(); + summary.artifacts = collect_artifacts(&ctx.run_dir).unwrap_or_default(); + write_json_pretty(&ctx.run_dir.join("run-summary.json"), &summary)?; + append_json_line(&summary_jsonl_path(args), &summary)?; + runs_completed += 1; + next_seq += 1; + last_run_id = Some(ctx.run_id); + + if args.max_runs.is_some_and(|max| runs_completed >= max) { + break; + } + write_status( + args, + DaemonState::Sleeping, + runs_completed, + None, + last_run_id.clone(), + )?; + if args.interval_secs > 0 { + std::thread::sleep(Duration::from_secs(args.interval_secs)); + } + } + + write_status(args, DaemonState::Exited, runs_completed, None, last_run_id) +} + +pub fn main_entry() -> i32 { + let argv: Vec = std::env::args().collect(); + match parse_args(&argv) { + Ok(args) => match run_daemon(&args) { + Ok(()) => 0, + Err(err) => { + eprintln!("{err}"); + 2 + } + }, + Err(err) => { + if argv.iter().any(|a| a == "--help" || a == "-h") { + println!("{err}"); + return 0; + } + eprintln!("{err}"); + 2 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_args(state_root: PathBuf) -> Args { + Args { + work_db: state_root.join("state/work-db"), + repo_bytes_db: Some(state_root.join("state/repo-bytes.db")), + state_root, + rpki_bin: PathBuf::from("/bin/true"), + interval_secs: 0, + max_runs: Some(1), + retain_runs: 10, + status_json: None, + summary_jsonl: None, + raw_store_db: None, + db_stats_bin: None, + db_stats_exact_every: None, + time_bin: None, + child_args: vec!["--version".to_string()], + } + } + + #[test] + fn parse_args_accepts_required_flags_and_child_args() { + let argv = vec![ + "rpki_daemon".to_string(), + "--state-root".to_string(), + "/tmp/daemon".to_string(), + "--rpki-bin".to_string(), + "/bin/echo".to_string(), + "--interval-secs".to_string(), + "0".to_string(), + "--max-runs".to_string(), + "2".to_string(), + "--retain-runs".to_string(), + "3".to_string(), + "--".to_string(), + "--db".to_string(), + "{state_root}/state/work-db".to_string(), + "--report-json".to_string(), + "{run_out}/report.json".to_string(), + ]; + + let args = parse_args(&argv).expect("parse args"); + assert_eq!(args.state_root, PathBuf::from("/tmp/daemon")); + assert_eq!(args.rpki_bin, PathBuf::from("/bin/echo")); + assert_eq!(args.interval_secs, 0); + assert_eq!(args.max_runs, Some(2)); + assert_eq!(args.retain_runs, 3); + assert_eq!(args.work_db, PathBuf::from("/tmp/daemon/state/work-db")); + assert_eq!( + args.repo_bytes_db, + Some(PathBuf::from("/tmp/daemon/state/repo-bytes.db")) + ); + assert_eq!( + args.child_args, + vec![ + "--db", + "{state_root}/state/work-db", + "--report-json", + "{run_out}/report.json" + ] + ); + } + + #[test] + fn usage_and_parse_args_cover_optional_flags_and_errors() { + let help = usage(); + assert!(help.contains("--db-stats-exact-every")); + assert!(help.contains("{run_seq}")); + + let argv = vec![ + "rpki_daemon".to_string(), + "--state-root".to_string(), + "/tmp/daemon".to_string(), + "--rpki-bin".to_string(), + "/bin/echo".to_string(), + "--status-json".to_string(), + "/tmp/status.json".to_string(), + "--summary-jsonl".to_string(), + "/tmp/runs.jsonl".to_string(), + "--work-db".to_string(), + "{state_root}/work".to_string(), + "--repo-bytes-db".to_string(), + "{state_root}/repo-bytes".to_string(), + "--raw-store-db".to_string(), + "{state_root}/raw".to_string(), + "--db-stats-bin".to_string(), + "/bin/echo".to_string(), + "--db-stats-exact-every".to_string(), + "2".to_string(), + "--time-bin".to_string(), + "/usr/bin/time".to_string(), + "--no-time-wrapper".to_string(), + "--".to_string(), + "child".to_string(), + ]; + let args = parse_args(&argv).expect("optional args"); + assert_eq!(args.status_json, Some(PathBuf::from("/tmp/status.json"))); + assert_eq!(args.summary_jsonl, Some(PathBuf::from("/tmp/runs.jsonl"))); + assert_eq!(args.work_db, PathBuf::from("{state_root}/work")); + assert_eq!( + args.repo_bytes_db, + Some(PathBuf::from("{state_root}/repo-bytes")) + ); + assert_eq!(args.raw_store_db, Some(PathBuf::from("{state_root}/raw"))); + assert_eq!(args.db_stats_bin, Some(PathBuf::from("/bin/echo"))); + assert_eq!(args.db_stats_exact_every, Some(2)); + assert_eq!(args.time_bin, None); + + for (argv, expected) in [ + (vec!["rpki_daemon", "--help"], "Usage:"), + ( + vec![ + "rpki_daemon", + "--state-root", + "/tmp/x", + "--rpki-bin", + "/bin/echo", + "--max-runs", + "0", + "--", + "child", + ], + "--max-runs must be > 0", + ), + ( + vec![ + "rpki_daemon", + "--state-root", + "/tmp/x", + "--rpki-bin", + "/bin/echo", + "--retain-runs", + "0", + "--", + "child", + ], + "--retain-runs must be > 0", + ), + ( + vec![ + "rpki_daemon", + "--state-root", + "/tmp/x", + "--rpki-bin", + "/bin/echo", + "--db-stats-exact-every", + "0", + "--", + "child", + ], + "--db-stats-exact-every must be > 0", + ), + (vec!["rpki_daemon", "--unknown"], "unknown argument"), + ( + vec!["rpki_daemon", "--state-root"], + "--state-root requires a value", + ), + (vec!["rpki_daemon"], "missing -- before child rpki args"), + ( + vec![ + "rpki_daemon", + "--state-root", + "/tmp/x", + "--rpki-bin", + "/bin/echo", + "--", + ], + "child rpki args are required", + ), + ] { + let owned: Vec = argv.into_iter().map(str::to_string).collect(); + let err = parse_args(&owned).expect_err("parse should fail"); + assert!(err.contains(expected), "{err}"); + } + } + + #[test] + fn render_child_args_replaces_placeholders() { + let args = Args { + state_root: PathBuf::from("/tmp/root"), + rpki_bin: PathBuf::from("/bin/echo"), + interval_secs: 0, + max_runs: Some(1), + retain_runs: 10, + status_json: None, + summary_jsonl: None, + work_db: PathBuf::from("/tmp/root/state/work-db"), + repo_bytes_db: Some(PathBuf::from("/tmp/root/state/repo-bytes.db")), + raw_store_db: None, + db_stats_bin: None, + db_stats_exact_every: None, + time_bin: None, + child_args: vec![ + "{state_root}/state/work-db".to_string(), + "{run_out}/result.ccr".to_string(), + "{run_id}".to_string(), + "{run_seq}".to_string(), + ], + }; + let ctx = RunContext { + seq: 7, + run_id: "000007-20260428T090000Z".to_string(), + run_dir: PathBuf::from("/tmp/root/runs/000007-20260428T090000Z"), + }; + + assert_eq!( + render_child_args(&args.child_args, &args, &ctx), + vec![ + "/tmp/root/state/work-db", + "/tmp/root/runs/000007-20260428T090000Z/result.ccr", + "000007-20260428T090000Z", + "7", + ] + ); + } + + #[test] + fn path_json_and_report_helpers_cover_fallbacks_and_nested_stats() { + let td = tempfile::tempdir().expect("tempdir"); + let state_root = td.path().join("daemon"); + let mut args = test_args(state_root.clone()); + args.work_db = PathBuf::from("{state_root}/state/work-db"); + args.repo_bytes_db = Some(PathBuf::from("{state_root}/state/repo-bytes.db")); + args.raw_store_db = Some(PathBuf::from("{state_root}/state/raw-store.db")); + + let now = time::Date::from_calendar_date(2026, time::Month::April, 28) + .expect("date") + .with_hms(9, 0, 0) + .expect("time") + .assume_utc(); + let ctx = make_run_context(&args, 42, now); + assert_eq!(ctx.run_id, "000042-20260428T090000Z"); + + let rendered = render_path_template(Path::new("{run_out}/{run_id}/{run_seq}"), &args, &ctx); + assert!(rendered.ends_with("000042-20260428T090000Z/000042-20260428T090000Z/42")); + + let nested_json = td.path().join("nested/out/status.json"); + write_json_pretty(&nested_json, &serde_json::json!({"ok": true})).expect("write json"); + append_json_line( + &td.path().join("nested/out/runs.jsonl"), + &serde_json::json!({"n": 1}), + ) + .expect("append jsonl"); + + fs::create_dir_all(ctx.run_dir.join("subdir")).expect("subdir"); + fs::write(ctx.run_dir.join("a.txt"), "aaa").expect("file"); + fs::write(ctx.run_dir.join("subdir/ignored.txt"), "bbb").expect("nested file"); + let artifacts = collect_artifacts(&ctx.run_dir).expect("artifacts"); + assert_eq!(artifacts.len(), 1); + assert!(artifacts[0].path.ends_with("a.txt")); + + fs::create_dir_all(state_root.join("state/work-db/nested")).expect("work db"); + fs::write(state_root.join("state/work-db/file.sst"), "abc").expect("sst"); + fs::write(state_root.join("state/work-db/nested/inner.sst"), "def").expect("inner"); + fs::write(state_root.join("state/raw-store.db"), "raw").expect("raw file"); + let file_stats = + collect_path_file_stats("raw_store_db", &state_root.join("state/raw-store.db")); + assert!(file_stats.exists); + assert!(!file_stats.is_dir); + assert_eq!(file_stats.file_count, 1); + let missing_stats = collect_path_file_stats("missing", &state_root.join("missing")); + assert!(!missing_stats.exists); + let state_stats = collect_state_path_stats(&args, &ctx); + assert!( + state_stats + .iter() + .any(|s| s.label == "raw_store_db" && s.exists) + ); + assert!( + state_stats + .iter() + .any(|s| s.label == "work_db" && s.file_count == 2) + ); + + let report_path = td.path().join("report.json"); + fs::write( + &report_path, + r#"{"repo_sync_stats": { "nested": {"text": "a\"b"} }, "after": 1}"#, + ) + .expect("report"); + assert_eq!( + extract_json_object_field(&report_path, "repo_sync_stats").expect("repo stats")["nested"] + ["text"], + "a\"b" + ); + fs::write(&report_path, r#"{"repo_sync_stats": []}"#).expect("report"); + assert!(extract_json_object_field(&report_path, "repo_sync_stats").is_none()); + assert!(extract_json_object_field(&report_path, "missing").is_none()); + + let counts = parse_report_counts_fallback(&serde_json::json!({ + "vrps": [{}, {}], + "aspas": [{}], + "publication_points": [{"warnings": [{}, {}]}, {"warnings": [{}]}], + "tree": {"instances_processed": 2, "instances_failed": 1, "warnings": [{}]} + })); + assert_eq!(counts.vrps, 2); + assert_eq!(counts.aspas, 1); + assert_eq!(counts.publication_points, 2); + assert_eq!(counts.warnings, 4); + assert_eq!(counts.tree_instances_processed, Some(2)); + assert_eq!(counts.tree_instances_failed, Some(1)); + } + + #[test] + fn retention_removes_oldest_run_directories() { + let td = tempfile::tempdir().expect("tempdir"); + let runs = td.path().join("runs"); + fs::create_dir_all(&runs).expect("runs dir"); + for name in [ + "000001-20260428T000001Z", + "000002-20260428T000002Z", + "000003-20260428T000003Z", + ] { + fs::create_dir_all(runs.join(name)).expect("run dir"); + } + + let removed = apply_retention(&runs, 2).expect("retention"); + assert_eq!(removed.len(), 1); + assert!(!runs.join("000001-20260428T000001Z").exists()); + assert!(runs.join("000002-20260428T000002Z").exists()); + assert!(runs.join("000003-20260428T000003Z").exists()); + } + + #[test] + fn retention_empty_root_and_parse_metrics_error_paths_are_reported() { + let td = tempfile::tempdir().expect("tempdir"); + let missing_runs = td.path().join("missing-runs"); + assert!( + apply_retention(&missing_runs, 2) + .expect("empty retention") + .is_empty() + ); + + let disabled = collect_process_metrics(false, &td.path().join("missing-time.txt")); + assert!(!disabled.time_wrapper_used); + assert!(disabled.time_output_path.is_none()); + + let missing = collect_process_metrics(true, &td.path().join("missing-time.txt")); + assert!(missing.time_wrapper_used); + assert!( + missing + .parse_error + .expect("parse error") + .contains("read process time output failed") + ); + } + + #[test] + fn process_metrics_parses_gnu_time_elapsed_line() { + let td = tempfile::tempdir().expect("tempdir"); + let path = td.path().join("time.txt"); + fs::write( + &path, + "User time (seconds): 1.25\nSystem time (seconds): 0.50\nPercent of CPU this job got: 175%\nElapsed (wall clock) time (h:mm:ss or m:ss): 0:01.00\nMaximum resident set size (kbytes): 12345\nExit status: 0\n", + ) + .expect("write time"); + + let metrics = collect_process_metrics(true, &path); + assert_eq!(metrics.user_seconds, Some(1.25)); + assert_eq!(metrics.system_seconds, Some(0.50)); + assert_eq!(metrics.cpu_percent, Some(175.0)); + assert_eq!(metrics.elapsed_raw.as_deref(), Some("0:01.00")); + assert_eq!(metrics.max_rss_kb, Some(12345)); + assert_eq!(metrics.exit_status_from_time, Some(0)); + } + + #[test] + #[cfg(unix)] + fn child_and_db_stats_error_paths_are_reported() { + use std::os::unix::fs::PermissionsExt; + + let td = tempfile::tempdir().expect("tempdir"); + let run_dir = td.path().join("run"); + fs::create_dir_all(&run_dir).expect("run dir"); + let missing_db = td.path().join("missing-db"); + let skipped = + run_db_stats_command(Path::new("/bin/echo"), &missing_db, &run_dir, "estimate"); + assert_eq!(skipped.status, "skipped"); + assert!(skipped.output_path.is_none()); + + let db_path = td.path().join("db"); + fs::create_dir_all(&db_path).expect("db"); + let failing_bin = td.path().join("failing_db_stats.sh"); + fs::write( + &failing_bin, + "#!/bin/sh\necho partial=1\necho db-stats failed >&2\nexit 7\n", + ) + .expect("script"); + let mut permissions = fs::metadata(&failing_bin).expect("metadata").permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&failing_bin, permissions).expect("chmod"); + let failed = run_db_stats_command(&failing_bin, &db_path, &run_dir, "exact"); + assert_eq!(failed.status, "failed"); + assert_eq!(failed.exit_code, Some(7)); + assert_eq!(failed.metrics.get("partial").map(String::as_str), Some("1")); + assert!(failed.stderr_path.is_some()); + assert!(failed.error.expect("stderr").contains("db-stats failed")); + + let spawn_failed = run_db_stats_command( + Path::new("/definitely/not/db_stats"), + &db_path, + &run_dir, + "estimate", + ); + assert_eq!(spawn_failed.status, "spawn_failed"); + assert!( + spawn_failed + .error + .expect("spawn err") + .contains("spawn db_stats failed") + ); + + let metrics = parse_key_value_metrics("alpha=1\nnot-a-pair\nbeta = two\n"); + assert_eq!(metrics.get("alpha").map(String::as_str), Some("1")); + assert_eq!(metrics.get("beta").map(String::as_str), Some("two")); + + let mut args = test_args(td.path().join("daemon")); + args.rpki_bin = PathBuf::from("/bin/sh"); + args.time_bin = Some(PathBuf::from("/usr/bin/time")); + args.child_args = vec![ + "-c".to_string(), + "echo child-out; echo child-err >&2; exit 7".to_string(), + ]; + let ctx = RunContext { + seq: 1, + run_id: "000001-20260428T000000Z".to_string(), + run_dir: args.state_root.join("runs/000001-20260428T000000Z"), + }; + let summary = run_child_once(&args, &ctx).expect("run failing child"); + assert_eq!(summary.status, RunStatus::Failed); + assert_eq!(summary.exit_code, Some(7)); + let process_metrics = summary.process_metrics.expect("process metrics"); + assert!(process_metrics.time_wrapper_used); + assert_eq!(process_metrics.exit_status_from_time, Some(7)); + + let mut spawn_args = test_args(td.path().join("spawn")); + spawn_args.rpki_bin = PathBuf::from("/definitely/not/rpki"); + spawn_args.child_args = vec!["--help".to_string()]; + let spawn_ctx = RunContext { + seq: 1, + run_id: "000001-20260428T000001Z".to_string(), + run_dir: spawn_args.state_root.join("runs/000001-20260428T000001Z"), + }; + let spawn_summary = run_child_once(&spawn_args, &spawn_ctx).expect("spawn summary"); + assert_eq!(spawn_summary.status, RunStatus::SpawnFailed); + assert!( + spawn_summary + .error + .expect("spawn error") + .contains("spawn child failed") + ); + } + + #[test] + fn report_metadata_prefers_stdout_summary_and_extracts_repo_sync_stats() { + let td = tempfile::tempdir().expect("tempdir"); + fs::write( + td.path().join("stdout.log"), + "RPKI stage2 serial run summary\npublication_points_processed=7 publication_points_failed=1\nrrdp_repos_unique=3\nvrps=11\naspas=2\naudit_publication_points=7\nwarnings_total=5\n", + ) + .expect("stdout"); + fs::write( + td.path().join("report.json"), + "{\"large\":[{\"ignored\":\"{}\"}],\"repo_sync_stats\":{\"by_phase\":{\"rrdp_ok\":{\"count\":7}},\"by_terminal_state\":{},\"publication_points_total\":7}}", + ) + .expect("report"); + + let (counts, repo_sync_stats) = parse_report_metadata(td.path()); + let counts = counts.expect("counts"); + assert_eq!(counts.vrps, 11); + assert_eq!(counts.aspas, 2); + assert_eq!(counts.publication_points, 7); + assert_eq!(counts.rrdp_repos_unique, Some(3)); + assert_eq!(counts.tree_instances_processed, Some(7)); + assert_eq!(counts.tree_instances_failed, Some(1)); + assert_eq!(counts.warnings, 5); + assert_eq!( + repo_sync_stats.expect("repo sync")["by_phase"]["rrdp_ok"]["count"].as_u64(), + Some(7) + ); + } + + #[test] + fn daemon_exits_immediately_when_max_runs_already_reached() { + let td = tempfile::tempdir().expect("tempdir"); + let mut args = test_args(td.path().join("daemon")); + args.max_runs = Some(0); + + run_daemon(&args).expect("run daemon"); + + let status_text = + fs::read_to_string(args.state_root.join("daemon-status.json")).expect("status"); + assert!(status_text.contains("\"state\": \"exited\"")); + assert!(status_text.contains("\"runsCompleted\": 0")); + assert!(args.state_root.join("runs").exists()); + } + + #[test] + fn daemon_runs_fake_child_twice_and_writes_summaries() { + let td = tempfile::tempdir().expect("tempdir"); + let args = Args { + state_root: td.path().join("daemon"), + rpki_bin: PathBuf::from("/bin/sh"), + interval_secs: 0, + max_runs: Some(2), + retain_runs: 10, + status_json: None, + summary_jsonl: None, + work_db: td.path().join("daemon/state/work-db"), + repo_bytes_db: Some(td.path().join("daemon/state/repo-bytes.db")), + raw_store_db: None, + db_stats_bin: None, + db_stats_exact_every: None, + time_bin: None, + child_args: vec![ + "-c".to_string(), + "echo stdout-{run_seq}; echo stderr-{run_seq} >&2; echo marker > {run_out}/marker.txt".to_string(), + ], + }; + + run_daemon(&args).expect("run daemon"); + + let status_text = + fs::read_to_string(args.state_root.join("daemon-status.json")).expect("status"); + assert!(status_text.contains("\"state\": \"exited\"")); + assert!(status_text.contains("\"runsCompleted\": 2")); + + let jsonl = + fs::read_to_string(args.state_root.join("daemon-runs.jsonl")).expect("summary jsonl"); + assert_eq!(jsonl.lines().count(), 2); + + let run_dirs = fs::read_dir(args.state_root.join("runs")) + .expect("runs dir") + .collect::, _>>() + .expect("entries"); + assert_eq!(run_dirs.len(), 2); + for entry in run_dirs { + assert!(entry.path().join("run-summary.json").exists()); + assert!(entry.path().join("marker.txt").exists()); + } + } + + #[test] + fn daemon_sleep_and_retention_deletion_are_recorded() { + let td = tempfile::tempdir().expect("tempdir"); + let mut args = test_args(td.path().join("daemon")); + args.rpki_bin = PathBuf::from("/bin/sh"); + args.interval_secs = 1; + args.max_runs = Some(2); + args.retain_runs = 1; + args.child_args = vec![ + "-c".to_string(), + "echo run-{run_seq}; printf marker > {run_out}/marker.txt".to_string(), + ]; + + run_daemon(&args).expect("run daemon"); + + let jsonl = fs::read_to_string(args.state_root.join("daemon-runs.jsonl")).expect("jsonl"); + assert_eq!(jsonl.lines().count(), 2); + let last: serde_json::Value = + serde_json::from_str(jsonl.lines().last().expect("last line")).expect("summary"); + assert_eq!( + last["retentionDeletedRuns"] + .as_array() + .expect("deleted") + .len(), + 1 + ); + let run_dirs = fs::read_dir(args.state_root.join("runs")) + .expect("runs") + .collect::, _>>() + .expect("entries"); + assert_eq!(run_dirs.len(), 1); + } + + #[test] + #[cfg(unix)] + fn daemon_collects_stage_report_db_and_file_metrics() { + use std::os::unix::fs::PermissionsExt; + + let td = tempfile::tempdir().expect("tempdir"); + let db_stats_bin = td.path().join("fake_db_stats.sh"); + fs::write( + &db_stats_bin, + "#!/bin/sh\nif [ \"$3\" = \"--exact\" ]; then echo mode=exact; else echo mode=estimate; fi\necho total=42\necho db.files.total_size_bytes=123\n", + ) + .expect("fake db_stats"); + let mut permissions = fs::metadata(&db_stats_bin).expect("metadata").permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&db_stats_bin, permissions).expect("chmod"); + + let args = Args { + state_root: td.path().join("daemon"), + rpki_bin: PathBuf::from("/bin/sh"), + interval_secs: 0, + max_runs: Some(1), + retain_runs: 10, + status_json: None, + summary_jsonl: None, + work_db: PathBuf::from("{state_root}/state/work-db"), + repo_bytes_db: Some(PathBuf::from("{state_root}/state/repo-bytes.db")), + raw_store_db: None, + db_stats_bin: Some(db_stats_bin), + db_stats_exact_every: Some(1), + time_bin: None, + child_args: vec![ + "-c".to_string(), + "mkdir -p {state_root}/state/work-db {state_root}/state/repo-bytes.db; \ + printf x > {state_root}/state/work-db/000001.sst; \ + printf '{\"validation_ms\":7,\"download_event_count\":2}' > {run_out}/stage-timing.json; \ + printf '{\"tree\":{\"instances_processed\":3,\"instances_failed\":1,\"warnings\":[{}]},\"publication_points\":[{},{}],\"vrps\":[{},{}],\"aspas\":[{}],\"repo_sync_stats\":{\"by_phase\":{\"snapshot\":{\"count\":1}}}}' > {run_out}/report.json" + .to_string(), + ], + }; + + run_daemon(&args).expect("run daemon"); + + let run_summary = find_named_file(&args.state_root.join("runs"), "run-summary.json") + .expect("run summary"); + let summary: serde_json::Value = + serde_json::from_slice(&fs::read(run_summary).expect("read run summary")) + .expect("parse run summary"); + + assert_eq!(summary["stageTiming"]["validation_ms"].as_u64(), Some(7)); + assert_eq!(summary["reportCounts"]["vrps"].as_u64(), Some(2)); + assert_eq!(summary["reportCounts"]["aspas"].as_u64(), Some(1)); + assert_eq!( + summary["reportCounts"]["publicationPoints"].as_u64(), + Some(2) + ); + assert_eq!( + summary["repoSyncStats"]["by_phase"]["snapshot"]["count"].as_u64(), + Some(1) + ); + assert_eq!(summary["dbStats"].as_array().expect("db stats").len(), 2); + assert!( + summary["pathStats"] + .as_array() + .expect("path stats") + .iter() + .any(|item| item["label"] == "work_db" && item["exists"] == true) + ); + } +} diff --git a/src/tools/sequence_triage_ccr_cir.rs b/src/tools/sequence_triage_ccr_cir.rs new file mode 100644 index 0000000..8de9adf --- /dev/null +++ b/src/tools/sequence_triage_ccr_cir.rs @@ -0,0 +1,2711 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use crate::ccr::{decode_ccr_compare_views, decode_content_info}; +use crate::cir::decode_cir; +use serde::Deserialize; +use serde_json::{Value, json}; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; + +#[derive(Debug, PartialEq, Eq)] +struct Args { + left_sequence: PathBuf, + right_sequence: PathBuf, + out_dir: PathBuf, + align_window_runs: u32, + align_window_secs: i64, + sample_limit: usize, + warmup_samples: usize, + cooldown_samples: usize, + timeline_sample_limit: usize, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct SequenceItemRaw { + schema_version: Option, + rp_id: String, + side: Option, + seq: u32, + run_id: String, + sync_mode: Option, + status: Option, + start_time: Option, + finish_time: Option, + validation_time: Option, + ccr_path: PathBuf, + cir_path: PathBuf, + ccr_sha256: Option, + cir_sha256: Option, + wall_ms: Option, + max_rss_kb: Option, + vrps: Option, + vaps: Option, +} + +#[derive(Clone, Debug)] +struct SequenceSample { + raw: SequenceItemRaw, + validation_time: OffsetDateTime, + ccr_path: PathBuf, + cir_path: PathBuf, + objects: BTreeMap, + object_uris: BTreeSet, + object_hashes: BTreeSet, + rejects: BTreeSet, + trust_anchors: BTreeSet, + vrps: BTreeSet, + vaps: BTreeSet, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Side { + Left, + Right, +} + +impl Side { + fn as_str(self) -> &'static str { + match self { + Side::Left => "left", + Side::Right => "right", + } + } +} + +#[derive(Clone, Debug)] +struct EventOccurrence { + side: Side, + seq: u32, + run_id: String, +} + +#[derive(Clone, Debug)] +struct SampleRecord { + classification: &'static str, + event_type: &'static str, + key: String, + source_side: Side, + source_seq: u32, + source_run_id: String, + matched_seq: Option, + matched_run_id: Option, + note: String, +} + +#[derive(Clone, Debug, Default)] +struct ClassStats { + total: usize, + samples: Vec, +} + +#[derive(Clone, Debug, Default)] +struct AnalysisResult { + stats: BTreeMap<&'static str, ClassStats>, +} + +#[derive(Clone, Debug)] +struct DiffEvent { + event_type: &'static str, + raw_class: &'static str, + key: String, + source_side: Side, + source_seq: u32, + source_run_id: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum EdgePosition { + Leading, + Stable, + Trailing, +} + +#[derive(Clone, Debug)] +struct AdjustedRecord { + classification: &'static str, + event_type: &'static str, + key: String, + source_side: Side, + source_seq: u32, + source_run_id: String, + note: String, +} + +#[derive(Clone, Debug, Default)] +struct AdjustedClassStats { + total: usize, + unique_keys: BTreeSet, + samples: Vec, +} + +#[derive(Clone, Debug, Default)] +struct AdjustedAnalysis { + raw_persistent_occurrences: usize, + raw_persistent_unique_keys: usize, + edge_filtered_occurrences: usize, + edge_filtered_unique_keys: usize, + adjusted_stable_occurrences: usize, + adjusted_stable_unique_keys: usize, + stats: BTreeMap<&'static str, AdjustedClassStats>, + uri_timeline_samples: Vec, + stable_object_groups: Vec, +} + +#[derive(Clone, Debug)] +struct SandwichRecord { + classification: &'static str, + set_type: &'static str, + key: String, + source_side: Side, + source_start_seq: u32, + source_start_run_id: String, + source_end_seq: u32, + source_end_run_id: String, + peer_seq: u32, + peer_run_id: String, + source_value: Option, + peer_value: Option, + source_start_time: String, + peer_time: String, + source_end_time: String, + note: String, +} + +#[derive(Clone, Debug, Default)] +struct SandwichClassStats { + total: usize, + unique_keys: BTreeSet, + samples: Vec, +} + +#[derive(Clone, Debug, Default)] +struct SandwichAnalysis { + total_occurrences: usize, + unique_keys: BTreeSet, + by_set_type: BTreeMap<&'static str, usize>, + stats: BTreeMap<&'static str, SandwichClassStats>, +} + +fn usage() -> &'static str { + "Usage: sequence_triage_ccr_cir --left-sequence --right-sequence --out-dir [--align-window-runs ] [--align-window-secs ] [--sample-limit ] [--warmup-samples ] [--cooldown-samples ] [--timeline-sample-limit ]" +} + +pub fn main_entry() -> Result<(), String> { + real_main() +} + +fn real_main() -> Result<(), String> { + let args = parse_args(&std::env::args().collect::>())?; + run(args) +} + +fn parse_args(argv: &[String]) -> Result { + let mut left_sequence = None; + let mut right_sequence = None; + let mut out_dir = None; + let mut align_window_runs = 2u32; + let mut align_window_secs = 1800i64; + let mut sample_limit = 200usize; + let mut warmup_samples = 1usize; + let mut cooldown_samples = 1usize; + let mut timeline_sample_limit = 0usize; + let mut index = 1usize; + while index < argv.len() { + match argv[index].as_str() { + "--left-sequence" => { + index += 1; + left_sequence = Some(PathBuf::from( + argv.get(index).ok_or("--left-sequence requires a value")?, + )); + } + "--right-sequence" => { + index += 1; + right_sequence = Some(PathBuf::from( + argv.get(index).ok_or("--right-sequence requires a value")?, + )); + } + "--out-dir" => { + index += 1; + out_dir = Some(PathBuf::from( + argv.get(index).ok_or("--out-dir requires a value")?, + )); + } + "--align-window-runs" => { + index += 1; + let value = argv + .get(index) + .ok_or("--align-window-runs requires a value")?; + align_window_runs = value + .parse::() + .map_err(|_| format!("invalid --align-window-runs: {value}"))?; + } + "--align-window-secs" => { + index += 1; + let value = argv + .get(index) + .ok_or("--align-window-secs requires a value")?; + align_window_secs = value + .parse::() + .map_err(|_| format!("invalid --align-window-secs: {value}"))?; + } + "--sample-limit" => { + index += 1; + let value = argv.get(index).ok_or("--sample-limit requires a value")?; + sample_limit = value + .parse::() + .map_err(|_| format!("invalid --sample-limit: {value}"))?; + } + "--warmup-samples" => { + index += 1; + let value = argv.get(index).ok_or("--warmup-samples requires a value")?; + warmup_samples = value + .parse::() + .map_err(|_| format!("invalid --warmup-samples: {value}"))?; + } + "--cooldown-samples" => { + index += 1; + let value = argv + .get(index) + .ok_or("--cooldown-samples requires a value")?; + cooldown_samples = value + .parse::() + .map_err(|_| format!("invalid --cooldown-samples: {value}"))?; + } + "--timeline-sample-limit" => { + index += 1; + let value = argv + .get(index) + .ok_or("--timeline-sample-limit requires a value")?; + timeline_sample_limit = value + .parse::() + .map_err(|_| format!("invalid --timeline-sample-limit: {value}"))?; + } + "-h" | "--help" => return Err(usage().to_string()), + other => return Err(format!("unknown argument: {other}\n{}", usage())), + } + index += 1; + } + Ok(Args { + left_sequence: left_sequence + .ok_or_else(|| format!("--left-sequence is required\n{}", usage()))?, + right_sequence: right_sequence + .ok_or_else(|| format!("--right-sequence is required\n{}", usage()))?, + out_dir: out_dir.ok_or_else(|| format!("--out-dir is required\n{}", usage()))?, + align_window_runs, + align_window_secs, + sample_limit, + warmup_samples, + cooldown_samples, + timeline_sample_limit, + }) +} + +fn run(args: Args) -> Result<(), String> { + std::fs::create_dir_all(&args.out_dir) + .map_err(|e| format!("create out-dir failed: {}: {e}", args.out_dir.display()))?; + let left = load_sequence(&args.left_sequence, Side::Left)?; + let right = load_sequence(&args.right_sequence, Side::Right)?; + if left.is_empty() || right.is_empty() { + return Err("left and right sequences must both contain at least one sample".into()); + } + + let mut result = AnalysisResult::default(); + analyze_set( + &mut result, + "object_uri", + &left, + &right, + |sample| &sample.object_uris, + "TEMPORAL_LAG_RESOLVED", + "PERSISTENT_OBJECT_SET_DIVERGENCE", + &args, + ); + analyze_set( + &mut result, + "object_hash", + &left, + &right, + |sample| &sample.object_hashes, + "TEMPORAL_LAG_RESOLVED", + "PERSISTENT_CONTENT_DIVERGENCE", + &args, + ); + analyze_hash_rollover(&mut result, &left, &right, &args); + analyze_set( + &mut result, + "reject_uri", + &left, + &right, + |sample| &sample.rejects, + "TEMPORAL_LAG_RESOLVED", + "PERSISTENT_REJECT_DIVERGENCE", + &args, + ); + analyze_set( + &mut result, + "trust_anchor", + &left, + &right, + |sample| &sample.trust_anchors, + "TEMPORAL_LAG_RESOLVED", + "PERSISTENT_TA_DIFFERENCE", + &args, + ); + analyze_set( + &mut result, + "vrp_output", + &left, + &right, + |sample| &sample.vrps, + "TEMPORAL_LAG_RESOLVED", + "PERSISTENT_OUTPUT_DIVERGENCE", + &args, + ); + analyze_set( + &mut result, + "vap_output", + &left, + &right, + |sample| &sample.vaps, + "TEMPORAL_LAG_RESOLVED", + "PERSISTENT_OUTPUT_DIVERGENCE", + &args, + ); + + let persistent_events = collect_persistent_events(&left, &right, &args); + let adjusted = build_adjusted_analysis(&args, &left, &right, &persistent_events); + let sandwich = build_sandwich_analysis(&args, &left, &right); + let output = build_output(&args, &left, &right, &result, &adjusted, &sandwich); + write_json(&args.out_dir.join("sequence-triage.json"), &output)?; + write_markdown(&args.out_dir.join("sequence-triage.md"), &output)?; + write_samples_jsonl(&args.out_dir.join("sequence-diff-samples.jsonl"), &result)?; + println!("{}", args.out_dir.display()); + Ok(()) +} + +fn load_sequence(path: &Path, side: Side) -> Result, String> { + let base_dir = path.parent().unwrap_or_else(|| Path::new(".")); + let text = std::fs::read_to_string(path) + .map_err(|e| format!("read sequence failed: {}: {e}", path.display()))?; + let mut samples = Vec::new(); + let mut seen_seq = BTreeSet::new(); + for (line_index, line) in text.lines().enumerate() { + let line = line.trim(); + if line.is_empty() { + continue; + } + let raw: SequenceItemRaw = serde_json::from_str(line).map_err(|e| { + format!( + "parse sequence JSONL failed: {}:{}: {e}", + path.display(), + line_index + 1 + ) + })?; + if raw.schema_version.unwrap_or(1) != 1 { + return Err(format!( + "unsupported sequence item schemaVersion in {}:{}", + path.display(), + line_index + 1 + )); + } + if !seen_seq.insert(raw.seq) { + return Err(format!("duplicate seq {} in {}", raw.seq, path.display())); + } + if let Some(status) = &raw.status + && status != "success" + { + return Err(format!( + "sequence sample {} has non-success status: {status}", + raw.run_id + )); + } + let cir_path = resolve_path(base_dir, &raw.cir_path); + let ccr_path = resolve_path(base_dir, &raw.ccr_path); + let cir = decode_cir(&read_file(&cir_path)?).map_err(|e| { + format!( + "decode CIR failed for sample {} ({}): {e}", + raw.run_id, + cir_path.display() + ) + })?; + let ccr = decode_content_info(&read_file(&ccr_path)?).map_err(|e| { + format!( + "decode CCR failed for sample {} ({}): {e}", + raw.run_id, + ccr_path.display() + ) + })?; + let validation_time = raw + .validation_time + .as_deref() + .map(parse_rfc3339) + .transpose()? + .unwrap_or(cir.validation_time); + let objects = cir + .objects + .iter() + .map(|item| (item.rsync_uri.clone(), hex::encode(&item.sha256))) + .collect::>(); + let object_uris = objects.keys().cloned().collect::>(); + let object_hashes = objects + .iter() + .map(|(uri, hash)| object_hash_key(uri, hash)) + .collect::>(); + let rejects = cir + .rejected_objects + .iter() + .map(|item| item.object_uri.clone()) + .collect::>(); + let trust_anchors = cir + .trust_anchors + .iter() + .map(|item| { + format!( + "{}|{}|{}|{}", + item.ta_rsync_uri, + item.tal_uri, + hex::encode(crate::cir::sha256(&item.tal_bytes)), + hex::encode(&item.ta_certificate_sha256) + ) + }) + .collect::>(); + let (vrps, vaps) = decode_ccr_compare_views(&ccr).map_err(|e| { + format!( + "decode CCR compare views failed for sample {} ({}): {e}", + raw.run_id, + ccr_path.display() + ) + })?; + let vrps = vrps + .into_iter() + .map(|row| format!("{}|{}|{}", row.asn, row.ip_prefix, row.max_length)) + .collect::>(); + let vaps = vaps + .into_iter() + .map(|row| format!("{}|{}", row.customer_asn, row.providers)) + .collect::>(); + samples.push(SequenceSample { + raw, + validation_time, + ccr_path, + cir_path, + objects, + object_uris, + object_hashes, + rejects, + trust_anchors, + vrps, + vaps, + }); + } + samples.sort_by_key(|sample| sample.raw.seq); + for pair in samples.windows(2) { + if pair[0].raw.seq >= pair[1].raw.seq { + return Err("sequence must be sorted by increasing seq".into()); + } + } + if samples.iter().any(|sample| { + sample + .raw + .side + .as_deref() + .is_some_and(|item| item != side.as_str()) + }) { + return Err(format!( + "sequence side field does not match expected side: {}", + side.as_str() + )); + } + Ok(samples) +} + +fn analyze_set( + result: &mut AnalysisResult, + event_type: &'static str, + left: &[SequenceSample], + right: &[SequenceSample], + extract: F, + resolved_class: &'static str, + persistent_class: &'static str, + args: &Args, +) where + F: for<'a> Fn(&'a SequenceSample) -> &'a BTreeSet, +{ + analyze_direction( + result, + event_type, + Side::Left, + left, + right, + &extract, + resolved_class, + persistent_class, + args, + ); + analyze_direction( + result, + event_type, + Side::Right, + right, + left, + &extract, + resolved_class, + persistent_class, + args, + ); +} + +fn analyze_direction( + result: &mut AnalysisResult, + event_type: &'static str, + source_side: Side, + source: &[SequenceSample], + peer: &[SequenceSample], + extract: &F, + resolved_class: &'static str, + persistent_class: &'static str, + args: &Args, +) where + F: for<'a> Fn(&'a SequenceSample) -> &'a BTreeSet, +{ + for sample in source { + let source_set = extract(sample); + for key in source_set { + if peer_sample_at_seq(peer, sample.raw.seq) + .is_some_and(|peer_sample| extract(peer_sample).contains(key)) + { + continue; + } + if let Some(matched) = find_future_match(peer, sample, &key, extract, args) { + result.add( + resolved_class, + SampleRecord { + classification: resolved_class, + event_type, + key: key.clone(), + source_side, + source_seq: sample.raw.seq, + source_run_id: sample.raw.run_id.clone(), + matched_seq: Some(matched.seq), + matched_run_id: Some(matched.run_id), + note: format!( + "matched in {} sequence within alignment window", + matched.side.as_str() + ), + }, + args.sample_limit, + ); + } else { + result.add( + persistent_class, + SampleRecord { + classification: persistent_class, + event_type, + key: key.clone(), + source_side, + source_seq: sample.raw.seq, + source_run_id: sample.raw.run_id.clone(), + matched_seq: None, + matched_run_id: None, + note: "no matching event in peer sequence alignment window".to_string(), + }, + args.sample_limit, + ); + } + } + } +} + +fn analyze_hash_rollover( + result: &mut AnalysisResult, + left: &[SequenceSample], + right: &[SequenceSample], + args: &Args, +) { + for (source_side, source, peer) in [(Side::Left, left, right), (Side::Right, right, left)] { + for sample in source { + for (uri, hash) in &sample.objects { + if peer_sample_at_seq(peer, sample.raw.seq) + .and_then(|peer_sample| peer_sample.objects.get(uri)) + .is_some_and(|peer_hash| peer_hash == hash) + { + continue; + } + if let Some(peer_sample) = peer_sample_at_seq(peer, sample.raw.seq) + && peer_sample.objects.contains_key(uri) + && find_future_hash_match(peer, sample, uri, hash, args).is_some() + { + let matched = + find_future_hash_match(peer, sample, uri, hash, args).expect("match"); + result.add( + "CONTENT_ROLLOVER_RESOLVED", + SampleRecord { + classification: "CONTENT_ROLLOVER_RESOLVED", + event_type: "object_content_rollover", + key: object_hash_key(uri, hash), + source_side, + source_seq: sample.raw.seq, + source_run_id: sample.raw.run_id.clone(), + matched_seq: Some(matched.seq), + matched_run_id: Some(matched.run_id), + note: "same URI hash appeared in peer sequence later".to_string(), + }, + args.sample_limit, + ); + } + } + } + } +} + +fn collect_persistent_events( + left: &[SequenceSample], + right: &[SequenceSample], + args: &Args, +) -> Vec { + let mut events = Vec::new(); + collect_persistent_set( + &mut events, + "object_uri", + "PERSISTENT_OBJECT_SET_DIVERGENCE", + left, + right, + |sample| &sample.object_uris, + args, + ); + collect_persistent_set( + &mut events, + "object_hash", + "PERSISTENT_CONTENT_DIVERGENCE", + left, + right, + |sample| &sample.object_hashes, + args, + ); + collect_persistent_set( + &mut events, + "reject_uri", + "PERSISTENT_REJECT_DIVERGENCE", + left, + right, + |sample| &sample.rejects, + args, + ); + collect_persistent_set( + &mut events, + "trust_anchor", + "PERSISTENT_TA_DIFFERENCE", + left, + right, + |sample| &sample.trust_anchors, + args, + ); + collect_persistent_set( + &mut events, + "vrp_output", + "PERSISTENT_OUTPUT_DIVERGENCE", + left, + right, + |sample| &sample.vrps, + args, + ); + collect_persistent_set( + &mut events, + "vap_output", + "PERSISTENT_OUTPUT_DIVERGENCE", + left, + right, + |sample| &sample.vaps, + args, + ); + events +} + +fn collect_persistent_set( + events: &mut Vec, + event_type: &'static str, + raw_class: &'static str, + left: &[SequenceSample], + right: &[SequenceSample], + extract: F, + args: &Args, +) where + F: for<'a> Fn(&'a SequenceSample) -> &'a BTreeSet, +{ + collect_persistent_direction( + events, + event_type, + raw_class, + Side::Left, + left, + right, + &extract, + args, + ); + collect_persistent_direction( + events, + event_type, + raw_class, + Side::Right, + right, + left, + &extract, + args, + ); +} + +fn collect_persistent_direction( + events: &mut Vec, + event_type: &'static str, + raw_class: &'static str, + source_side: Side, + source: &[SequenceSample], + peer: &[SequenceSample], + extract: &F, + args: &Args, +) where + F: for<'a> Fn(&'a SequenceSample) -> &'a BTreeSet, +{ + for sample in source { + for key in extract(sample) { + if peer_sample_at_seq(peer, sample.raw.seq) + .is_some_and(|peer_sample| extract(peer_sample).contains(key)) + { + continue; + } + if find_future_match(peer, sample, key, extract, args).is_some() { + continue; + } + events.push(DiffEvent { + event_type, + raw_class, + key: key.clone(), + source_side, + source_seq: sample.raw.seq, + source_run_id: sample.raw.run_id.clone(), + }); + } + } +} + +fn build_adjusted_analysis( + args: &Args, + left: &[SequenceSample], + right: &[SequenceSample], + events: &[DiffEvent], +) -> AdjustedAnalysis { + let mut analysis = AdjustedAnalysis { + raw_persistent_occurrences: events.len(), + raw_persistent_unique_keys: unique_event_count(events), + ..Default::default() + }; + let mut edge_filtered_unique = BTreeSet::new(); + + for event in events { + let source_samples = samples_for_side(event.source_side, left, right); + let peer_samples = samples_for_side(opposite_side(event.source_side), left, right); + let edge = edge_position(source_samples, event.source_seq, args); + if edge == EdgePosition::Stable { + analysis.edge_filtered_occurrences += 1; + edge_filtered_unique.insert(event_identity(event)); + } + let (classification, note) = + adjusted_classification(event, edge, source_samples, peer_samples); + analysis.add( + classification, + AdjustedRecord { + classification, + event_type: event.event_type, + key: event.key.clone(), + source_side: event.source_side, + source_seq: event.source_seq, + source_run_id: event.source_run_id.clone(), + note, + }, + args.sample_limit, + ); + } + + analysis.edge_filtered_unique_keys = edge_filtered_unique.len(); + let mut stable_unique = BTreeSet::new(); + for (class, stats) in &analysis.stats { + if class.starts_with("STABLE_") { + analysis.adjusted_stable_occurrences += stats.total; + stable_unique.extend(stats.unique_keys.iter().cloned()); + } + } + analysis.adjusted_stable_unique_keys = stable_unique.len(); + let timeline_limit = if args.timeline_sample_limit == 0 { + args.sample_limit + } else { + args.timeline_sample_limit + }; + analysis.uri_timeline_samples = + build_uri_timeline_samples(left, right, &analysis, timeline_limit); + analysis.stable_object_groups = + build_stable_object_groups(&analysis, left, right, timeline_limit); + analysis +} + +fn adjusted_classification( + event: &DiffEvent, + edge: EdgePosition, + source: &[SequenceSample], + peer: &[SequenceSample], +) -> (&'static str, String) { + if event.event_type == "trust_anchor" { + if peer_has_same_ta_identity(peer, &event.key) { + return ( + "TA_PROJECTION_FORMAT_DIFFERENCE", + "same TAL hash and TA certificate hash exist on peer side; full TA projection string differs".to_string(), + ); + } + if edge == EdgePosition::Stable { + return ( + "STABLE_TA_DIVERGENCE", + "trust-anchor identity is not aligned in a non-boundary sample".to_string(), + ); + } + return edge_unresolved_class(edge); + } + + if event.event_type == "object_hash" { + if let Some((uri, hash)) = split_object_hash_key(&event.key) { + let peer_has_uri = peer_has_uri_any(peer, uri); + let peer_has_different_hash = peer_has_uri_different_hash_any(peer, uri, hash); + let peer_hash_at_source_seq = peer_sample_at_seq(peer, event.source_seq) + .and_then(|peer_sample| peer_sample.objects.get(uri)); + let source_later_matches_peer_hash = peer_hash_at_source_seq.is_some_and(|peer_hash| { + source_future_has_hash(source, event.source_seq, uri, peer_hash) + }); + if edge == EdgePosition::Leading && peer_has_different_hash { + return ( + "EDGE_LEADING_CONTENT_ROLLOVER", + "source leading-edge hash is absent, while peer already has the same URI with another hash".to_string(), + ); + } + if edge == EdgePosition::Trailing && peer_has_different_hash { + return ( + "EDGE_TRAILING_CONTENT_ROLLOVER", + "source trailing-edge hash is absent, while peer has the same URI with another hash and no later source observation exists".to_string(), + ); + } + if edge == EdgePosition::Stable && source_later_matches_peer_hash { + return ( + "MID_SEQUENCE_CONTENT_ROLLOVER_RESOLVED", + "peer already has a newer same-URI hash at this seq and source catches up later in the observed sequence".to_string(), + ); + } + if edge == EdgePosition::Stable && peer_has_different_hash { + return ( + "STABLE_CONTENT_DIVERGENCE", + "same URI exists on peer side with a different hash in a non-boundary sample" + .to_string(), + ); + } + if edge == EdgePosition::Stable && !peer_has_uri { + return ( + "STABLE_OBJECT_SET_DIVERGENCE", + "content key is derived from a non-boundary URI that is absent on peer side" + .to_string(), + ); + } + } + if edge == EdgePosition::Stable { + return ( + "STABLE_CONTENT_DIVERGENCE", + "object hash key remains unaligned in a non-boundary sample".to_string(), + ); + } + return edge_unresolved_class(edge); + } + + if edge != EdgePosition::Stable { + return edge_unresolved_class(edge); + } + + let stable_class = match event.raw_class { + "PERSISTENT_OBJECT_SET_DIVERGENCE" => "STABLE_OBJECT_SET_DIVERGENCE", + "PERSISTENT_REJECT_DIVERGENCE" => "STABLE_REJECT_DIVERGENCE", + "PERSISTENT_OUTPUT_DIVERGENCE" => "STABLE_OUTPUT_DIVERGENCE", + "PERSISTENT_CONTENT_DIVERGENCE" => "STABLE_CONTENT_DIVERGENCE", + "PERSISTENT_TA_DIFFERENCE" => "STABLE_TA_DIVERGENCE", + _ => "STABLE_UNCLASSIFIED_DIVERGENCE", + }; + ( + stable_class, + "persistent event appears in a non-boundary sample after sequence edge filtering" + .to_string(), + ) +} + +fn edge_unresolved_class(edge: EdgePosition) -> (&'static str, String) { + match edge { + EdgePosition::Leading => ( + "EDGE_LEADING_UNRESOLVED", + "event appears only from the warmup edge and may predate the observed sequence" + .to_string(), + ), + EdgePosition::Trailing => ( + "EDGE_TRAILING_UNRESOLVED", + "event appears at the cooldown edge and lacks later peer observations".to_string(), + ), + EdgePosition::Stable => ( + "STABLE_UNCLASSIFIED_DIVERGENCE", + "event is not on an edge but no specific stable category matched".to_string(), + ), + } +} + +impl AdjustedAnalysis { + fn add(&mut self, class: &'static str, record: AdjustedRecord, sample_limit: usize) { + let stats = self.stats.entry(class).or_default(); + stats.total += 1; + stats.unique_keys.insert(format!( + "{}|{}|{}", + record.event_type, + record.source_side.as_str(), + record.key + )); + if stats.samples.len() < sample_limit { + stats.samples.push(record); + } + } +} + +fn build_sandwich_analysis( + args: &Args, + left: &[SequenceSample], + right: &[SequenceSample], +) -> SandwichAnalysis { + let mut analysis = SandwichAnalysis::default(); + analyze_sandwich_objects(&mut analysis, Side::Left, left, right, args); + analyze_sandwich_objects(&mut analysis, Side::Right, right, left, args); + analyze_sandwich_sets( + &mut analysis, + "reject_uri", + "PEER_MISSING_STABLE_REJECT", + Side::Left, + left, + right, + |sample| &sample.rejects, + args, + ); + analyze_sandwich_sets( + &mut analysis, + "reject_uri", + "PEER_MISSING_STABLE_REJECT", + Side::Right, + right, + left, + |sample| &sample.rejects, + args, + ); + analyze_sandwich_sets( + &mut analysis, + "vrp_output", + "PEER_MISSING_STABLE_OUTPUT", + Side::Left, + left, + right, + |sample| &sample.vrps, + args, + ); + analyze_sandwich_sets( + &mut analysis, + "vrp_output", + "PEER_MISSING_STABLE_OUTPUT", + Side::Right, + right, + left, + |sample| &sample.vrps, + args, + ); + analyze_sandwich_sets( + &mut analysis, + "vap_output", + "PEER_MISSING_STABLE_OUTPUT", + Side::Left, + left, + right, + |sample| &sample.vaps, + args, + ); + analyze_sandwich_sets( + &mut analysis, + "vap_output", + "PEER_MISSING_STABLE_OUTPUT", + Side::Right, + right, + left, + |sample| &sample.vaps, + args, + ); + analysis +} + +fn analyze_sandwich_objects( + analysis: &mut SandwichAnalysis, + source_side: Side, + source: &[SequenceSample], + peer: &[SequenceSample], + args: &Args, +) { + for pair in source.windows(2) { + let source_start = &pair[0]; + let source_end = &pair[1]; + if source_start.validation_time >= source_end.validation_time { + continue; + } + let peers = peer_samples_between(peer, source_start, source_end); + if peers.is_empty() { + continue; + } + for (uri, source_hash) in &source_start.objects { + if source_end.objects.get(uri) != Some(source_hash) { + continue; + } + for peer_sample in &peers { + match peer_sample.objects.get(uri) { + Some(peer_hash) if peer_hash == source_hash => {} + Some(peer_hash) => analysis.add( + "PEER_HASH_MISMATCH_STABLE_OBJECT", + sandwich_record( + "PEER_HASH_MISMATCH_STABLE_OBJECT", + "object", + uri.clone(), + source_side, + source_start, + source_end, + peer_sample, + Some(source_hash.clone()), + Some(peer_hash.clone()), + "source interval has stable object hash; peer sample has same URI with another hash", + ), + args.sample_limit, + ), + None => analysis.add( + "PEER_MISSING_STABLE_OBJECT", + sandwich_record( + "PEER_MISSING_STABLE_OBJECT", + "object", + uri.clone(), + source_side, + source_start, + source_end, + peer_sample, + Some(source_hash.clone()), + None, + "source interval has stable object hash; peer sample misses the URI", + ), + args.sample_limit, + ), + } + } + } + } +} + +fn analyze_sandwich_sets( + analysis: &mut SandwichAnalysis, + set_type: &'static str, + classification: &'static str, + source_side: Side, + source: &[SequenceSample], + peer: &[SequenceSample], + extract: F, + args: &Args, +) where + F: for<'a> Fn(&'a SequenceSample) -> &'a BTreeSet, +{ + for pair in source.windows(2) { + let source_start = &pair[0]; + let source_end = &pair[1]; + if source_start.validation_time >= source_end.validation_time { + continue; + } + let peers = peer_samples_between(peer, source_start, source_end); + if peers.is_empty() { + continue; + } + let start_set = extract(source_start); + let end_set = extract(source_end); + for key in start_set { + if !end_set.contains(key) { + continue; + } + for peer_sample in &peers { + if extract(peer_sample).contains(key) { + continue; + } + analysis.add( + classification, + sandwich_record( + classification, + set_type, + key.clone(), + source_side, + source_start, + source_end, + peer_sample, + Some(key.clone()), + None, + "source interval has a stable key; peer sample misses the key", + ), + args.sample_limit, + ); + } + } + } +} + +fn peer_samples_between<'a>( + peer: &'a [SequenceSample], + source_start: &SequenceSample, + source_end: &SequenceSample, +) -> Vec<&'a SequenceSample> { + peer.iter() + .filter(|sample| { + source_start.validation_time < sample.validation_time + && sample.validation_time < source_end.validation_time + }) + .collect() +} + +#[allow(clippy::too_many_arguments)] +fn sandwich_record( + classification: &'static str, + set_type: &'static str, + key: String, + source_side: Side, + source_start: &SequenceSample, + source_end: &SequenceSample, + peer_sample: &SequenceSample, + source_value: Option, + peer_value: Option, + note: &str, +) -> SandwichRecord { + SandwichRecord { + classification, + set_type, + key, + source_side, + source_start_seq: source_start.raw.seq, + source_start_run_id: source_start.raw.run_id.clone(), + source_end_seq: source_end.raw.seq, + source_end_run_id: source_end.raw.run_id.clone(), + peer_seq: peer_sample.raw.seq, + peer_run_id: peer_sample.raw.run_id.clone(), + source_value, + peer_value, + source_start_time: format_time(source_start.validation_time), + peer_time: format_time(peer_sample.validation_time), + source_end_time: format_time(source_end.validation_time), + note: note.to_string(), + } +} + +impl SandwichAnalysis { + fn add(&mut self, class: &'static str, record: SandwichRecord, sample_limit: usize) { + self.total_occurrences += 1; + self.unique_keys.insert(sandwich_unique_key(&record)); + *self.by_set_type.entry(record.set_type).or_default() += 1; + let stats = self.stats.entry(class).or_default(); + stats.total += 1; + stats.unique_keys.insert(sandwich_unique_key(&record)); + if stats.samples.len() < sample_limit { + stats.samples.push(record); + } + } +} + +fn sandwich_unique_key(record: &SandwichRecord) -> String { + format!( + "{}|{}|{}|{}", + record.classification, + record.set_type, + record.source_side.as_str(), + record.key + ) +} + +fn unique_event_count(events: &[DiffEvent]) -> usize { + events + .iter() + .map(event_identity) + .collect::>() + .len() +} + +fn event_identity(event: &DiffEvent) -> String { + format!( + "{}|{}|{}", + event.event_type, + event.source_side.as_str(), + event.key + ) +} + +fn samples_for_side<'a>( + side: Side, + left: &'a [SequenceSample], + right: &'a [SequenceSample], +) -> &'a [SequenceSample] { + match side { + Side::Left => left, + Side::Right => right, + } +} + +fn opposite_side(side: Side) -> Side { + match side { + Side::Left => Side::Right, + Side::Right => Side::Left, + } +} + +fn edge_position(samples: &[SequenceSample], seq: u32, args: &Args) -> EdgePosition { + let Some(index) = samples.iter().position(|sample| sample.raw.seq == seq) else { + return EdgePosition::Stable; + }; + if index < args.warmup_samples { + return EdgePosition::Leading; + } + if samples.len().saturating_sub(index) <= args.cooldown_samples { + return EdgePosition::Trailing; + } + EdgePosition::Stable +} + +fn peer_has_uri_any(peer: &[SequenceSample], uri: &str) -> bool { + peer.iter().any(|sample| sample.objects.contains_key(uri)) +} + +fn peer_has_uri_different_hash_any(peer: &[SequenceSample], uri: &str, hash: &str) -> bool { + peer.iter() + .filter_map(|sample| sample.objects.get(uri)) + .any(|peer_hash| peer_hash != hash) +} + +fn source_future_has_hash( + source: &[SequenceSample], + seq: u32, + uri: &str, + expected_hash: &str, +) -> bool { + source + .iter() + .filter(|sample| sample.raw.seq > seq) + .any(|sample| { + sample + .objects + .get(uri) + .is_some_and(|hash| hash == expected_hash) + }) +} + +fn peer_has_same_ta_identity(peer: &[SequenceSample], key: &str) -> bool { + let Some(identity) = trust_anchor_identity(key) else { + return false; + }; + peer.iter().any(|sample| { + sample + .trust_anchors + .iter() + .filter_map(|peer_key| trust_anchor_identity(peer_key)) + .any(|peer_identity| peer_identity == identity) + }) +} + +fn trust_anchor_identity(key: &str) -> Option { + let parts = key.split('|').collect::>(); + if parts.len() != 4 { + return None; + } + Some(format!("{}|{}", parts[2], parts[3])) +} + +fn split_object_hash_key(key: &str) -> Option<(&str, &str)> { + key.rsplit_once('|') +} + +fn build_uri_timeline_samples( + left: &[SequenceSample], + right: &[SequenceSample], + adjusted: &AdjustedAnalysis, + limit: usize, +) -> Vec { + let mut uris = BTreeSet::new(); + for stats in adjusted.stats.values() { + for sample in &stats.samples { + if sample.event_type == "object_hash" + && let Some((uri, _)) = split_object_hash_key(&sample.key) + { + uris.insert(uri.to_string()); + } + if sample.event_type == "object_uri" { + uris.insert(sample.key.clone()); + } + } + } + uris.into_iter() + .take(limit) + .map(|uri| { + json!({ + "uri": uri, + "left": timeline_for_uri(left, &uri), + "right": timeline_for_uri(right, &uri), + }) + }) + .collect() +} + +fn timeline_for_uri(samples: &[SequenceSample], uri: &str) -> Vec { + samples + .iter() + .filter_map(|sample| { + sample.objects.get(uri).map(|hash| { + json!({ + "seq": sample.raw.seq, + "runId": sample.raw.run_id, + "validationTime": format_time(sample.validation_time), + "hash": hash, + }) + }) + }) + .collect() +} + +fn build_stable_object_groups( + adjusted: &AdjustedAnalysis, + left: &[SequenceSample], + right: &[SequenceSample], + limit: usize, +) -> Vec { + let mut groups: BTreeMap = BTreeMap::new(); + let Some(stats) = adjusted.stats.get("STABLE_OBJECT_SET_DIVERGENCE") else { + return Vec::new(); + }; + for sample in &stats.samples { + let physical_uri = physical_object_uri(sample); + let group_key = format!( + "{}|{}|{}|{}", + sample.source_side.as_str(), + sample.source_seq, + sample.source_run_id, + publication_point_prefix(&physical_uri) + ); + let group = groups + .entry(group_key) + .or_insert_with(|| StableObjectGroup::new(sample, &physical_uri)); + if group.source_cir_path.is_none() + && let Some(source_sample) = sample_by_side_seq_run( + left, + right, + sample.source_side, + sample.source_seq, + &sample.source_run_id, + ) + { + group.source_cir_path = Some(path_string(&source_sample.cir_path)); + } + group.add(sample, physical_uri); + } + groups + .into_values() + .take(limit) + .map(StableObjectGroup::to_json) + .collect() +} + +#[derive(Clone, Debug)] +struct StableObjectGroup { + source_side: Side, + source_seq: u32, + source_run_id: String, + source_cir_path: Option, + publication_point: String, + event_count: usize, + event_types: BTreeMap<&'static str, usize>, + physical_objects: BTreeMap, +} + +impl StableObjectGroup { + fn new(sample: &AdjustedRecord, physical_uri: &str) -> Self { + Self { + source_side: sample.source_side, + source_seq: sample.source_seq, + source_run_id: sample.source_run_id.clone(), + source_cir_path: None, + publication_point: publication_point_prefix(physical_uri), + event_count: 0, + event_types: BTreeMap::new(), + physical_objects: BTreeMap::new(), + } + } + + fn add(&mut self, sample: &AdjustedRecord, physical_uri: String) { + self.event_count += 1; + *self.event_types.entry(sample.event_type).or_default() += 1; + let object = self + .physical_objects + .entry(physical_uri.clone()) + .or_insert_with(|| StablePhysicalObject { + extension: object_extension(&physical_uri).to_string(), + uri: physical_uri, + event_types: BTreeSet::new(), + hashes: BTreeSet::new(), + }); + object.event_types.insert(sample.event_type); + if let Some(hash) = event_hash(sample) { + object.hashes.insert(hash.to_string()); + } + } + + fn to_json(self) -> Value { + json!({ + "sourceSide": self.source_side.as_str(), + "sourceSeq": self.source_seq, + "sourceRunId": self.source_run_id, + "sourceCirPath": self.source_cir_path, + "publicationPoint": self.publication_point, + "eventCount": self.event_count, + "eventTypes": self.event_types, + "physicalObjectCount": self.physical_objects.len(), + "physicalObjects": self.physical_objects.into_values().map(StablePhysicalObject::to_json).collect::>(), + }) + } +} + +#[derive(Clone, Debug)] +struct StablePhysicalObject { + uri: String, + extension: String, + event_types: BTreeSet<&'static str>, + hashes: BTreeSet, +} + +impl StablePhysicalObject { + fn to_json(self) -> Value { + json!({ + "uri": self.uri, + "extension": self.extension, + "eventTypes": self.event_types, + "hashes": self.hashes, + }) + } +} + +fn physical_object_uri(sample: &AdjustedRecord) -> String { + if sample.event_type == "object_hash" + && let Some((uri, _)) = split_object_hash_key(&sample.key) + { + return uri.to_string(); + } + sample.key.clone() +} + +fn event_hash(sample: &AdjustedRecord) -> Option<&str> { + if sample.event_type == "object_hash" { + split_object_hash_key(&sample.key).map(|(_, hash)| hash) + } else { + None + } +} + +fn publication_point_prefix(uri: &str) -> String { + uri.rsplit_once('/') + .map(|(prefix, _)| format!("{prefix}/")) + .unwrap_or_else(|| uri.to_string()) +} + +fn object_extension(uri: &str) -> &str { + uri.rsplit_once('.') + .map(|(_, extension)| extension) + .unwrap_or("") +} + +fn sample_by_side_seq_run<'a>( + left: &'a [SequenceSample], + right: &'a [SequenceSample], + side: Side, + seq: u32, + run_id: &str, +) -> Option<&'a SequenceSample> { + samples_for_side(side, left, right) + .iter() + .find(|sample| sample.raw.seq == seq && sample.raw.run_id == run_id) +} + +impl AnalysisResult { + fn add(&mut self, class: &'static str, record: SampleRecord, sample_limit: usize) { + let stats = self.stats.entry(class).or_default(); + stats.total += 1; + if stats.samples.len() < sample_limit { + stats.samples.push(record); + } + } +} + +fn peer_sample_at_seq(peer: &[SequenceSample], seq: u32) -> Option<&SequenceSample> { + peer.iter().find(|sample| sample.raw.seq == seq) +} + +fn find_future_match( + peer: &[SequenceSample], + source: &SequenceSample, + key: &str, + extract: &F, + args: &Args, +) -> Option +where + F: for<'a> Fn(&'a SequenceSample) -> &'a BTreeSet, +{ + peer.iter() + .filter(|candidate| is_in_alignment_window(source, candidate, args)) + .find(|candidate| extract(candidate).contains(key)) + .map(|candidate| { + occurrence( + candidate, + if candidate.raw.side.as_deref() == Some("left") { + Side::Left + } else { + Side::Right + }, + ) + }) +} + +fn find_future_hash_match( + peer: &[SequenceSample], + source: &SequenceSample, + uri: &str, + hash: &str, + args: &Args, +) -> Option { + peer.iter() + .filter(|candidate| is_in_alignment_window(source, candidate, args)) + .find(|candidate| { + candidate + .objects + .get(uri) + .is_some_and(|peer_hash| peer_hash == hash) + }) + .map(|candidate| { + occurrence( + candidate, + if candidate.raw.side.as_deref() == Some("left") { + Side::Left + } else { + Side::Right + }, + ) + }) +} + +fn is_in_alignment_window( + source: &SequenceSample, + candidate: &SequenceSample, + args: &Args, +) -> bool { + if candidate.raw.seq < source.raw.seq { + return false; + } + let run_delta = candidate.raw.seq.saturating_sub(source.raw.seq); + let time_delta = candidate.validation_time - source.validation_time; + let secs = time_delta.whole_seconds().abs(); + run_delta <= args.align_window_runs || secs <= args.align_window_secs +} + +fn occurrence(sample: &SequenceSample, side: Side) -> EventOccurrence { + EventOccurrence { + side, + seq: sample.raw.seq, + run_id: sample.raw.run_id.clone(), + } +} + +fn build_output( + args: &Args, + left: &[SequenceSample], + right: &[SequenceSample], + result: &AnalysisResult, + adjusted: &AdjustedAnalysis, + sandwich: &SandwichAnalysis, +) -> Value { + let classifications = result + .stats + .iter() + .map(|(class, stats)| { + json!({ + "classification": class, + "count": stats.total, + "samples": stats.samples.iter().map(sample_to_json).collect::>(), + }) + }) + .collect::>(); + let adjusted_classifications = adjusted + .stats + .iter() + .map(|(class, stats)| { + json!({ + "classification": class, + "occurrences": stats.total, + "uniqueKeys": stats.unique_keys.len(), + "samples": stats.samples.iter().map(adjusted_sample_to_json).collect::>(), + }) + }) + .collect::>(); + let sandwich_classifications = sandwich + .stats + .iter() + .map(|(class, stats)| { + json!({ + "classification": class, + "occurrences": stats.total, + "uniqueKeys": stats.unique_keys.len(), + "samples": stats.samples.iter().map(sandwich_sample_to_json).collect::>(), + }) + }) + .collect::>(); + json!({ + "schemaVersion": 1, + "generatedBy": "sequence_triage_ccr_cir", + "inputContract": "left-right-sequence-jsonl-with-ccr-cir-artifacts", + "parameters": { + "leftSequence": path_string(&args.left_sequence), + "rightSequence": path_string(&args.right_sequence), + "alignWindowRuns": args.align_window_runs, + "alignWindowSecs": args.align_window_secs, + "sampleLimit": args.sample_limit, + "warmupSamples": args.warmup_samples, + "cooldownSamples": args.cooldown_samples, + "timelineSampleLimit": if args.timeline_sample_limit == 0 { args.sample_limit } else { args.timeline_sample_limit }, + }, + "left": sequence_summary(left), + "right": sequence_summary(right), + "classificationCounts": classifications, + "totals": { + "resolvedTemporalLike": count_class(result, "TEMPORAL_LAG_RESOLVED") + count_class(result, "CONTENT_ROLLOVER_RESOLVED"), + "persistent": count_prefix(result, "PERSISTENT_"), + "unclassified": count_class(result, "UNCLASSIFIED_INSUFFICIENT_WINDOW"), + }, + "adjusted": { + "warmupSamples": args.warmup_samples, + "cooldownSamples": args.cooldown_samples, + "rawPersistent": { + "occurrences": adjusted.raw_persistent_occurrences, + "uniqueKeys": adjusted.raw_persistent_unique_keys, + }, + "edgeFilteredPersistent": { + "occurrences": adjusted.edge_filtered_occurrences, + "uniqueKeys": adjusted.edge_filtered_unique_keys, + }, + "adjustedStablePersistent": { + "occurrences": adjusted.adjusted_stable_occurrences, + "uniqueKeys": adjusted.adjusted_stable_unique_keys, + }, + "classificationCounts": adjusted_classifications, + "uriTimelineSamples": adjusted.uri_timeline_samples, + "stableObjectGroups": adjusted.stable_object_groups, + "interpretation": { + "rawPersistentMeaning": "raw persistent keeps the original #043 single-event matching semantics.", + "edgeFilteredMeaning": "edge-filtered persistent keeps only non-warmup and non-cooldown source occurrences.", + "adjustedStableMeaning": "adjusted stable persistent keeps non-edge findings after URI-level rollover and TA projection filtering.", + "stableObjectGroupsMeaning": "STABLE_OBJECT_SET_DIVERGENCE events are additionally collapsed by source CIR, publication point, and physical object URI so object_uri/object_hash duplicate event views do not inflate physical-object counts.", + } + }, + "sandwich": { + "strictTimeWindow": true, + "method": "For each side, use two adjacent source samples as a stable interval. If source_start.time < peer.time < source_end.time and the source value is identical at both interval endpoints, the peer sample is expected to contain the same value.", + "totals": { + "occurrences": sandwich.total_occurrences, + "uniqueKeys": sandwich.unique_keys.len(), + }, + "bySetType": sandwich.by_set_type, + "classificationCounts": sandwich_classifications, + "interpretation": { + "missingStableObject": "The source side proves a URI/hash is stable across an interval that contains the peer sample, but the peer sample has no such URI.", + "hashMismatchStableObject": "The source side proves a URI/hash is stable across an interval that contains the peer sample, but the peer sample has the same URI with a different hash.", + "missingStableReject": "The source side consistently rejects the URI across an interval that contains the peer sample, but the peer sample does not reject it.", + "missingStableOutput": "The source side consistently outputs a VRP/VAP key across an interval that contains the peer sample, but the peer sample does not output it." + } + }, + "interpretation": { + "temporalResolvedMeaning": "Event appeared only on one side at one sample but aligned in the peer sequence later.", + "persistentMeaning": "Event did not align within the configured run/time window; inspect as a candidate RP behavior difference or persistent sync/input difference.", + "limits": [ + "Sequence triage only reads sequence JSONL plus referenced CCR/CIR files.", + "It does not read report.json, logs, repo-bytes DB, cache, mirror, or raw objects for root cause proof.", + "Resolved temporal findings reduce false positives but do not prove the exact external repository update time." + ] + } + }) +} + +fn sequence_summary(samples: &[SequenceSample]) -> Value { + json!({ + "sampleCount": samples.len(), + "rpIds": samples.iter().map(|sample| sample.raw.rp_id.clone()).collect::>(), + "firstSeq": samples.first().map(|sample| sample.raw.seq), + "lastSeq": samples.last().map(|sample| sample.raw.seq), + "firstValidationTime": samples.first().map(|sample| format_time(sample.validation_time)), + "lastValidationTime": samples.last().map(|sample| format_time(sample.validation_time)), + "samples": samples.iter().map(|sample| json!({ + "seq": sample.raw.seq, + "runId": sample.raw.run_id, + "syncMode": sample.raw.sync_mode, + "startTime": sample.raw.start_time, + "finishTime": sample.raw.finish_time, + "validationTime": format_time(sample.validation_time), + "status": sample.raw.status, + "ccrPath": path_string(&sample.ccr_path), + "cirPath": path_string(&sample.cir_path), + "ccrSha256": sample.raw.ccr_sha256, + "cirSha256": sample.raw.cir_sha256, + "wallMs": sample.raw.wall_ms, + "maxRssKb": sample.raw.max_rss_kb, + "vrps": sample.raw.vrps.or(Some(sample.vrps.len() as u64)), + "vaps": sample.raw.vaps.or(Some(sample.vaps.len() as u64)), + "objectCount": sample.object_uris.len(), + "rejectCount": sample.rejects.len(), + "trustAnchorCount": sample.trust_anchors.len(), + })).collect::>(), + }) +} + +fn sample_to_json(sample: &SampleRecord) -> Value { + json!({ + "classification": sample.classification, + "eventType": sample.event_type, + "key": sample.key, + "sourceSide": sample.source_side.as_str(), + "sourceSeq": sample.source_seq, + "sourceRunId": sample.source_run_id, + "matchedSeq": sample.matched_seq, + "matchedRunId": sample.matched_run_id, + "note": sample.note, + }) +} + +fn adjusted_sample_to_json(sample: &AdjustedRecord) -> Value { + json!({ + "classification": sample.classification, + "eventType": sample.event_type, + "key": sample.key, + "sourceSide": sample.source_side.as_str(), + "sourceSeq": sample.source_seq, + "sourceRunId": sample.source_run_id, + "note": sample.note, + }) +} + +fn sandwich_sample_to_json(sample: &SandwichRecord) -> Value { + json!({ + "classification": sample.classification, + "setType": sample.set_type, + "key": sample.key, + "sourceSide": sample.source_side.as_str(), + "sourceStartSeq": sample.source_start_seq, + "sourceStartRunId": sample.source_start_run_id, + "sourceEndSeq": sample.source_end_seq, + "sourceEndRunId": sample.source_end_run_id, + "peerSeq": sample.peer_seq, + "peerRunId": sample.peer_run_id, + "sourceValue": sample.source_value, + "peerValue": sample.peer_value, + "sourceStartTime": sample.source_start_time, + "peerTime": sample.peer_time, + "sourceEndTime": sample.source_end_time, + "note": sample.note, + }) +} + +fn count_class(result: &AnalysisResult, class: &'static str) -> usize { + result + .stats + .get(class) + .map(|stats| stats.total) + .unwrap_or(0) +} + +fn count_prefix(result: &AnalysisResult, prefix: &str) -> usize { + result + .stats + .iter() + .filter(|(class, _)| class.starts_with(prefix)) + .map(|(_, stats)| stats.total) + .sum() +} + +fn write_json(path: &Path, value: &Value) -> Result<(), String> { + std::fs::write( + path, + serde_json::to_string_pretty(value).map_err(|e| e.to_string())? + "\n", + ) + .map_err(|e| format!("write JSON failed: {}: {e}", path.display())) +} + +fn write_markdown(path: &Path, output: &Value) -> Result<(), String> { + let mut lines = vec![ + "# CCR/CIR Sequence Triage Summary".to_string(), + "".to_string(), + format!( + "- `generatedBy`: `{}`", + output["generatedBy"].as_str().unwrap_or("") + ), + format!( + "- `leftSamples`: `{}`", + output["left"]["sampleCount"].as_u64().unwrap_or(0) + ), + format!( + "- `rightSamples`: `{}`", + output["right"]["sampleCount"].as_u64().unwrap_or(0) + ), + format!( + "- `alignWindowRuns`: `{}`", + output["parameters"]["alignWindowRuns"] + .as_u64() + .unwrap_or(0) + ), + format!( + "- `alignWindowSecs`: `{}`", + output["parameters"]["alignWindowSecs"] + .as_i64() + .unwrap_or(0) + ), + format!( + "- `warmupSamples`: `{}`", + output["adjusted"]["warmupSamples"].as_u64().unwrap_or(0) + ), + format!( + "- `cooldownSamples`: `{}`", + output["adjusted"]["cooldownSamples"].as_u64().unwrap_or(0) + ), + "".to_string(), + "## Classification Counts".to_string(), + "".to_string(), + "| Classification | Count |".to_string(), + "|---|---:|".to_string(), + ]; + if let Some(classes) = output["classificationCounts"].as_array() { + for item in classes { + lines.push(format!( + "| `{}` | {} |", + item["classification"].as_str().unwrap_or(""), + item["count"].as_u64().unwrap_or(0) + )); + } + } + lines.extend([ + "".to_string(), + "## Adjusted Boundary / Rollover Classification".to_string(), + "".to_string(), + format!( + "- `rawPersistent`: `{}` occurrences / `{}` unique keys", + output["adjusted"]["rawPersistent"]["occurrences"] + .as_u64() + .unwrap_or(0), + output["adjusted"]["rawPersistent"]["uniqueKeys"] + .as_u64() + .unwrap_or(0) + ), + format!( + "- `edgeFilteredPersistent`: `{}` occurrences / `{}` unique keys", + output["adjusted"]["edgeFilteredPersistent"]["occurrences"] + .as_u64() + .unwrap_or(0), + output["adjusted"]["edgeFilteredPersistent"]["uniqueKeys"] + .as_u64() + .unwrap_or(0) + ), + format!( + "- `adjustedStablePersistent`: `{}` occurrences / `{}` unique keys", + output["adjusted"]["adjustedStablePersistent"]["occurrences"] + .as_u64() + .unwrap_or(0), + output["adjusted"]["adjustedStablePersistent"]["uniqueKeys"] + .as_u64() + .unwrap_or(0) + ), + "".to_string(), + "| Adjusted Classification | Occurrences | Unique Keys |".to_string(), + "|---|---:|---:|".to_string(), + ]); + if let Some(classes) = output["adjusted"]["classificationCounts"].as_array() { + for item in classes { + lines.push(format!( + "| `{}` | {} | {} |", + item["classification"].as_str().unwrap_or(""), + item["occurrences"].as_u64().unwrap_or(0), + item["uniqueKeys"].as_u64().unwrap_or(0) + )); + } + } + if let Some(groups) = output["adjusted"]["stableObjectGroups"].as_array() + && !groups.is_empty() + { + lines.extend([ + "".to_string(), + "## Stable Object Groups".to_string(), + "".to_string(), + "| Source | CIR | Publication Point | Events | Physical Objects |".to_string(), + "|---|---|---|---:|---:|".to_string(), + ]); + for group in groups { + lines.push(format!( + "| `{}/seq{}/{}` | `{}` | `{}` | {} | {} |", + group["sourceSide"].as_str().unwrap_or(""), + group["sourceSeq"].as_u64().unwrap_or(0), + group["sourceRunId"].as_str().unwrap_or(""), + group["sourceCirPath"].as_str().unwrap_or(""), + group["publicationPoint"].as_str().unwrap_or(""), + group["eventCount"].as_u64().unwrap_or(0), + group["physicalObjectCount"].as_u64().unwrap_or(0), + )); + } + } + lines.extend([ + "".to_string(), + "## Sandwich Anomaly Check".to_string(), + "".to_string(), + format!( + "- `occurrences`: `{}`", + output["sandwich"]["totals"]["occurrences"] + .as_u64() + .unwrap_or(0) + ), + format!( + "- `uniqueKeys`: `{}`", + output["sandwich"]["totals"]["uniqueKeys"] + .as_u64() + .unwrap_or(0) + ), + "- Rule: source side has two adjacent stable samples, and peer sample timestamp is strictly between them.".to_string(), + "".to_string(), + "| Sandwich Classification | Occurrences | Unique Keys |".to_string(), + "|---|---:|---:|".to_string(), + ]); + if let Some(classes) = output["sandwich"]["classificationCounts"].as_array() { + for item in classes { + lines.push(format!( + "| `{}` | {} | {} |", + item["classification"].as_str().unwrap_or(""), + item["occurrences"].as_u64().unwrap_or(0), + item["uniqueKeys"].as_u64().unwrap_or(0) + )); + } + } + lines.extend([ + "".to_string(), + "## Interpretation".to_string(), + "".to_string(), + "- `TEMPORAL_LAG_RESOLVED` / `CONTENT_ROLLOVER_RESOLVED` 表示差异在后续采样中对齐，优先视为采样时刻或仓库滚动窗口差异。".to_string(), + "- `PERSISTENT_*` 表示配置窗口内仍未对齐，才是后续人工排查的高价值候选。".to_string(), + "- `EDGE_*` 表示差异落在序列首尾，缺少前置或后续观察，不能直接当作实现差异。".to_string(), + "- `STABLE_*` 表示经过首尾边界过滤和 URI-level rollover 过滤后仍存在的稳定候选。".to_string(), + ]); + std::fs::write(path, lines.join("\n") + "\n") + .map_err(|e| format!("write markdown failed: {}: {e}", path.display())) +} + +fn write_samples_jsonl(path: &Path, result: &AnalysisResult) -> Result<(), String> { + let mut body = String::new(); + for stats in result.stats.values() { + for sample in &stats.samples { + body.push_str( + &serde_json::to_string(&sample_to_json(sample)).map_err(|e| e.to_string())?, + ); + body.push('\n'); + } + } + std::fs::write(path, body).map_err(|e| format!("write samples failed: {}: {e}", path.display())) +} + +fn read_file(path: &Path) -> Result, String> { + std::fs::read(path).map_err(|e| format!("read file failed: {}: {e}", path.display())) +} + +fn resolve_path(base_dir: &Path, path: &Path) -> PathBuf { + if path.is_absolute() { + path.to_path_buf() + } else { + base_dir.join(path) + } +} + +fn parse_rfc3339(value: &str) -> Result { + OffsetDateTime::parse(value, &Rfc3339) + .map_err(|e| format!("parse RFC3339 failed: {value}: {e}")) +} + +fn format_time(value: OffsetDateTime) -> String { + value.format(&Rfc3339).unwrap_or_else(|_| value.to_string()) +} + +fn path_string(path: &Path) -> String { + path.to_string_lossy().into_owned() +} + +fn object_hash_key(uri: &str, hash: &str) -> String { + format!("{uri}|{hash}") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ccr::{ + CcrContentInfo, CcrDigestAlgorithm, RpkiCanonicalCacheRepresentation, + build_aspa_payload_state, build_roa_payload_state, encode_content_info, + }; + use crate::cir::{ + CIR_VERSION_V3, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, + CirRejectedObject, CirTrustAnchor, compute_reject_list_sha256, encode_cir, sha256, + }; + use crate::data_model::roa::{IpPrefix, RoaAfi}; + use crate::validation::objects::{AspaAttestation, Vrp}; + + #[test] + fn parse_args_accepts_required_flags() { + let argv = vec![ + "sequence_triage_ccr_cir".to_string(), + "--left-sequence".to_string(), + "left.jsonl".to_string(), + "--right-sequence".to_string(), + "right.jsonl".to_string(), + "--out-dir".to_string(), + "out".to_string(), + "--align-window-runs".to_string(), + "3".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.align_window_runs, 3); + assert_eq!(args.align_window_secs, 1800); + assert_eq!(args.warmup_samples, 1); + assert_eq!(args.cooldown_samples, 1); + assert_eq!(args.timeline_sample_limit, 0); + } + + #[test] + fn run_classifies_temporal_and_persistent_differences() { + let temp = tempfile::tempdir().expect("tempdir"); + let root = temp.path(); + write_sample( + root, + "left1", + "left", + 1, + &[object("rsync://example.net/a.roa", 0x11)], + &[], + 64496, + ); + write_sample( + root, + "left2", + "left", + 2, + &[ + object("rsync://example.net/a.roa", 0x11), + object("rsync://example.net/persistent.roa", 0x44), + ], + &[], + 64496, + ); + write_sample(root, "right1", "right", 1, &[], &[], 64497); + write_sample( + root, + "right2", + "right", + 2, + &[object("rsync://example.net/a.roa", 0x11)], + &[], + 64497, + ); + std::fs::write( + root.join("left.jsonl"), + jsonl(&[ + item("left", 1, "left1/result.ccr", "left1/result.cir"), + item("left", 2, "left2/result.ccr", "left2/result.cir"), + ]), + ) + .unwrap(); + std::fs::write( + root.join("right.jsonl"), + jsonl(&[ + item("right", 1, "right1/result.ccr", "right1/result.cir"), + item("right", 2, "right2/result.ccr", "right2/result.cir"), + ]), + ) + .unwrap(); + run(Args { + left_sequence: root.join("left.jsonl"), + right_sequence: root.join("right.jsonl"), + out_dir: root.join("out"), + align_window_runs: 2, + align_window_secs: 3600, + sample_limit: 20, + warmup_samples: 1, + cooldown_samples: 0, + timeline_sample_limit: 0, + }) + .expect("run"); + let output: Value = serde_json::from_str( + &std::fs::read_to_string(root.join("out/sequence-triage.json")).unwrap(), + ) + .unwrap(); + assert!(class_count(&output, "TEMPORAL_LAG_RESOLVED") > 0); + assert!(class_count(&output, "PERSISTENT_OBJECT_SET_DIVERGENCE") > 0); + } + + #[test] + fn run_classifies_reject_and_output_divergence() { + let temp = tempfile::tempdir().expect("tempdir"); + let root = temp.path(); + let objects = [object("rsync://example.net/a.roa", 0x11)]; + write_sample( + root, + "left1", + "left", + 1, + &objects, + &["rsync://example.net/a.roa"], + 64496, + ); + write_sample(root, "right1", "right", 1, &objects, &[], 64497); + std::fs::write( + root.join("left.jsonl"), + jsonl(&[item("left", 1, "left1/result.ccr", "left1/result.cir")]), + ) + .unwrap(); + std::fs::write( + root.join("right.jsonl"), + jsonl(&[item("right", 1, "right1/result.ccr", "right1/result.cir")]), + ) + .unwrap(); + run(Args { + left_sequence: root.join("left.jsonl"), + right_sequence: root.join("right.jsonl"), + out_dir: root.join("out"), + align_window_runs: 0, + align_window_secs: 0, + sample_limit: 20, + warmup_samples: 0, + cooldown_samples: 0, + timeline_sample_limit: 0, + }) + .expect("run"); + let output: Value = serde_json::from_str( + &std::fs::read_to_string(root.join("out/sequence-triage.json")).unwrap(), + ) + .unwrap(); + assert!(class_count(&output, "PERSISTENT_REJECT_DIVERGENCE") > 0); + assert!(class_count(&output, "PERSISTENT_OUTPUT_DIVERGENCE") > 0); + } + + #[test] + fn run_adjusted_classifies_leading_content_rollover() { + let temp = tempfile::tempdir().expect("tempdir"); + let root = temp.path(); + let uri = "rsync://example.net/a.roa"; + write_sample(root, "left1", "left", 1, &[object(uri, 0x11)], &[], 64496); + write_sample(root, "left2", "left", 2, &[object(uri, 0x22)], &[], 64496); + write_sample(root, "right1", "right", 1, &[object(uri, 0x22)], &[], 64496); + write_sample(root, "right2", "right", 2, &[object(uri, 0x22)], &[], 64496); + std::fs::write( + root.join("left.jsonl"), + jsonl(&[ + item("left", 1, "left1/result.ccr", "left1/result.cir"), + item("left", 2, "left2/result.ccr", "left2/result.cir"), + ]), + ) + .unwrap(); + std::fs::write( + root.join("right.jsonl"), + jsonl(&[ + item("right", 1, "right1/result.ccr", "right1/result.cir"), + item("right", 2, "right2/result.ccr", "right2/result.cir"), + ]), + ) + .unwrap(); + run(Args { + left_sequence: root.join("left.jsonl"), + right_sequence: root.join("right.jsonl"), + out_dir: root.join("out"), + align_window_runs: 0, + align_window_secs: 0, + sample_limit: 20, + warmup_samples: 1, + cooldown_samples: 0, + timeline_sample_limit: 0, + }) + .expect("run"); + let output: Value = serde_json::from_str( + &std::fs::read_to_string(root.join("out/sequence-triage.json")).unwrap(), + ) + .unwrap(); + assert!(adjusted_class_occurrences(&output, "EDGE_LEADING_CONTENT_ROLLOVER") > 0); + assert_eq!( + adjusted_class_occurrences(&output, "STABLE_CONTENT_DIVERGENCE"), + 0 + ); + assert_eq!( + output["adjusted"]["adjustedStablePersistent"]["occurrences"] + .as_u64() + .unwrap(), + 0 + ); + } + + #[test] + fn run_adjusted_classifies_stable_middle_content_divergence() { + let temp = tempfile::tempdir().expect("tempdir"); + let root = temp.path(); + let uri = "rsync://example.net/a.roa"; + for seq in 1..=3 { + write_sample( + root, + &format!("left{seq}"), + "left", + seq, + &[object(uri, 0x11)], + &[], + 64496, + ); + write_sample( + root, + &format!("right{seq}"), + "right", + seq, + &[object(uri, 0x22)], + &[], + 64496, + ); + } + std::fs::write( + root.join("left.jsonl"), + jsonl(&[ + item("left", 1, "left1/result.ccr", "left1/result.cir"), + item("left", 2, "left2/result.ccr", "left2/result.cir"), + item("left", 3, "left3/result.ccr", "left3/result.cir"), + ]), + ) + .unwrap(); + std::fs::write( + root.join("right.jsonl"), + jsonl(&[ + item("right", 1, "right1/result.ccr", "right1/result.cir"), + item("right", 2, "right2/result.ccr", "right2/result.cir"), + item("right", 3, "right3/result.ccr", "right3/result.cir"), + ]), + ) + .unwrap(); + run(Args { + left_sequence: root.join("left.jsonl"), + right_sequence: root.join("right.jsonl"), + out_dir: root.join("out"), + align_window_runs: 0, + align_window_secs: 0, + sample_limit: 20, + warmup_samples: 1, + cooldown_samples: 1, + timeline_sample_limit: 0, + }) + .expect("run"); + let output: Value = serde_json::from_str( + &std::fs::read_to_string(root.join("out/sequence-triage.json")).unwrap(), + ) + .unwrap(); + assert!(adjusted_class_occurrences(&output, "STABLE_CONTENT_DIVERGENCE") > 0); + assert_eq!( + output["adjusted"]["adjustedStablePersistent"]["occurrences"] + .as_u64() + .unwrap(), + 2 + ); + } + + #[test] + fn run_adjusted_filters_trailing_output() { + let temp = tempfile::tempdir().expect("tempdir"); + let root = temp.path(); + let objects = [object("rsync://example.net/a.roa", 0x11)]; + write_sample(root, "left1", "left", 1, &objects, &[], 64496); + write_sample(root, "left2", "left", 2, &objects, &[], 64497); + write_sample(root, "right1", "right", 1, &objects, &[], 64496); + write_sample(root, "right2", "right", 2, &objects, &[], 64496); + std::fs::write( + root.join("left.jsonl"), + jsonl(&[ + item("left", 1, "left1/result.ccr", "left1/result.cir"), + item("left", 2, "left2/result.ccr", "left2/result.cir"), + ]), + ) + .unwrap(); + std::fs::write( + root.join("right.jsonl"), + jsonl(&[ + item("right", 1, "right1/result.ccr", "right1/result.cir"), + item("right", 2, "right2/result.ccr", "right2/result.cir"), + ]), + ) + .unwrap(); + run(Args { + left_sequence: root.join("left.jsonl"), + right_sequence: root.join("right.jsonl"), + out_dir: root.join("out"), + align_window_runs: 0, + align_window_secs: 0, + sample_limit: 20, + warmup_samples: 0, + cooldown_samples: 1, + timeline_sample_limit: 0, + }) + .expect("run"); + let output: Value = serde_json::from_str( + &std::fs::read_to_string(root.join("out/sequence-triage.json")).unwrap(), + ) + .unwrap(); + assert!(adjusted_class_occurrences(&output, "EDGE_TRAILING_UNRESOLVED") > 0); + assert_eq!( + adjusted_class_occurrences(&output, "STABLE_OUTPUT_DIVERGENCE"), + 0 + ); + } + + #[test] + fn run_groups_stable_object_events_by_physical_object() { + let temp = tempfile::tempdir().expect("tempdir"); + let root = temp.path(); + let missing = "rsync://example.net/repo/pp/a.roa"; + write_sample( + root, + "left1", + "left", + 1, + &[object("rsync://example.net/repo/pp/base.roa", 0x10)], + &[], + 64496, + ); + write_sample( + root, + "left2", + "left", + 2, + &[ + object("rsync://example.net/repo/pp/base.roa", 0x10), + object(missing, 0x11), + ], + &[], + 64496, + ); + write_sample( + root, + "left3", + "left", + 3, + &[object("rsync://example.net/repo/pp/base.roa", 0x10)], + &[], + 64496, + ); + for seq in 1..=3 { + write_sample( + root, + &format!("right{seq}"), + "right", + seq, + &[object("rsync://example.net/repo/pp/base.roa", 0x10)], + &[], + 64496, + ); + } + std::fs::write( + root.join("left.jsonl"), + jsonl(&[ + item("left", 1, "left1/result.ccr", "left1/result.cir"), + item("left", 2, "left2/result.ccr", "left2/result.cir"), + item("left", 3, "left3/result.ccr", "left3/result.cir"), + ]), + ) + .unwrap(); + std::fs::write( + root.join("right.jsonl"), + jsonl(&[ + item("right", 1, "right1/result.ccr", "right1/result.cir"), + item("right", 2, "right2/result.ccr", "right2/result.cir"), + item("right", 3, "right3/result.ccr", "right3/result.cir"), + ]), + ) + .unwrap(); + run(Args { + left_sequence: root.join("left.jsonl"), + right_sequence: root.join("right.jsonl"), + out_dir: root.join("out"), + align_window_runs: 0, + align_window_secs: 0, + sample_limit: 20, + warmup_samples: 1, + cooldown_samples: 1, + timeline_sample_limit: 0, + }) + .expect("run"); + let output: Value = serde_json::from_str( + &std::fs::read_to_string(root.join("out/sequence-triage.json")).unwrap(), + ) + .unwrap(); + let groups = output["adjusted"]["stableObjectGroups"].as_array().unwrap(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0]["eventCount"].as_u64(), Some(2)); + assert_eq!(groups[0]["physicalObjectCount"].as_u64(), Some(1)); + assert_eq!( + groups[0]["publicationPoint"].as_str(), + Some("rsync://example.net/repo/pp/") + ); + assert_eq!( + groups[0]["physicalObjects"][0]["eventTypes"] + .as_array() + .unwrap() + .len(), + 2 + ); + } + + #[test] + fn run_sandwich_detects_object_hash_reject_and_output_anomalies() { + let temp = tempfile::tempdir().expect("tempdir"); + let root = temp.path(); + let stable_missing = object("rsync://example.net/pp/missing.roa", 0x11); + let stable_mismatch = object("rsync://example.net/pp/mismatch.roa", 0x22); + let peer_mismatch = object("rsync://example.net/pp/mismatch.roa", 0x33); + write_sample_with_ccr_seq( + root, + "left1", + 1, + &[stable_missing.clone(), stable_mismatch.clone()], + &["rsync://example.net/pp/rejected.roa"], + 9, + 64496, + ); + write_sample_with_ccr_seq( + root, + "left3", + 3, + &[stable_missing, stable_mismatch], + &["rsync://example.net/pp/rejected.roa"], + 9, + 64496, + ); + write_sample_with_ccr_seq(root, "right2", 2, &[peer_mismatch], &[], 10, 64497); + std::fs::write( + root.join("left.jsonl"), + jsonl(&[ + item("left", 1, "left1/result.ccr", "left1/result.cir"), + item("left", 3, "left3/result.ccr", "left3/result.cir"), + ]), + ) + .unwrap(); + std::fs::write( + root.join("right.jsonl"), + jsonl(&[item("right", 2, "right2/result.ccr", "right2/result.cir")]), + ) + .unwrap(); + run(Args { + left_sequence: root.join("left.jsonl"), + right_sequence: root.join("right.jsonl"), + out_dir: root.join("out"), + align_window_runs: 0, + align_window_secs: 0, + sample_limit: 20, + warmup_samples: 0, + cooldown_samples: 0, + timeline_sample_limit: 0, + }) + .expect("run"); + let output: Value = serde_json::from_str( + &std::fs::read_to_string(root.join("out/sequence-triage.json")).unwrap(), + ) + .unwrap(); + assert_eq!( + sandwich_class_occurrences(&output, "PEER_MISSING_STABLE_OBJECT"), + 1 + ); + assert_eq!( + sandwich_class_occurrences(&output, "PEER_HASH_MISMATCH_STABLE_OBJECT"), + 1 + ); + assert_eq!( + sandwich_class_occurrences(&output, "PEER_MISSING_STABLE_REJECT"), + 1 + ); + assert_eq!( + sandwich_class_occurrences(&output, "PEER_MISSING_STABLE_OUTPUT"), + 2 + ); + } + + fn class_count(output: &Value, class: &str) -> u64 { + output["classificationCounts"] + .as_array() + .unwrap() + .iter() + .find(|item| item["classification"].as_str() == Some(class)) + .and_then(|item| item["count"].as_u64()) + .unwrap_or(0) + } + + fn sandwich_class_occurrences(output: &Value, class: &str) -> u64 { + output["sandwich"]["classificationCounts"] + .as_array() + .unwrap() + .iter() + .find(|item| item["classification"].as_str() == Some(class)) + .and_then(|item| item["occurrences"].as_u64()) + .unwrap_or(0) + } + + fn adjusted_class_occurrences(output: &Value, class: &str) -> u64 { + output["adjusted"]["classificationCounts"] + .as_array() + .unwrap() + .iter() + .find(|item| item["classification"].as_str() == Some(class)) + .and_then(|item| item["occurrences"].as_u64()) + .unwrap_or(0) + } + + fn object(uri: &str, byte: u8) -> CirObject { + CirObject { + rsync_uri: uri.to_string(), + sha256: vec![byte; 32], + } + } + + fn write_sample( + root: &Path, + dir: &str, + side: &str, + seq: u32, + objects: &[CirObject], + rejected: &[&str], + asn: u32, + ) { + let dir = root.join(dir); + std::fs::create_dir_all(&dir).unwrap(); + let cir = sample_cir(seq, objects, rejected); + let ccr = sample_ccr(seq, asn); + std::fs::write(dir.join("result.cir"), encode_cir(&cir).unwrap()).unwrap(); + std::fs::write(dir.join("result.ccr"), encode_content_info(&ccr).unwrap()).unwrap(); + let _ = side; + } + + fn write_sample_with_ccr_seq( + root: &Path, + dir: &str, + cir_seq: u32, + objects: &[CirObject], + rejected: &[&str], + ccr_seq: u32, + asn: u32, + ) { + let dir = root.join(dir); + std::fs::create_dir_all(&dir).unwrap(); + let cir = sample_cir(cir_seq, objects, rejected); + let ccr = sample_ccr(ccr_seq, asn); + std::fs::write(dir.join("result.cir"), encode_cir(&cir).unwrap()).unwrap(); + std::fs::write(dir.join("result.ccr"), encode_content_info(&ccr).unwrap()).unwrap(); + } + + fn sample_time(seq: u32) -> OffsetDateTime { + OffsetDateTime::from_unix_timestamp(1_800_000_000 + i64::from(seq * 60)).unwrap() + } + + fn sample_cir( + seq: u32, + objects: &[CirObject], + rejected: &[&str], + ) -> CanonicalInputRepresentation { + let mut objects = objects.to_vec(); + objects.sort_by(|a, b| a.rsync_uri.cmp(&b.rsync_uri)); + let rejected_objects = rejected + .iter() + .map(|uri| CirRejectedObject { + object_uri: (*uri).to_string(), + reason: Some("test".to_string()), + }) + .collect::>(); + CanonicalInputRepresentation { + version: CIR_VERSION_V3, + hash_alg: CirHashAlgorithm::Sha256, + validation_time: sample_time(seq), + objects, + trust_anchors: vec![sample_trust_anchor()], + reject_list_sha256: compute_reject_list_sha256(rejected.iter().copied()), + rejected_objects, + } + } + + fn sample_trust_anchor() -> CirTrustAnchor { + let ta_uri = "rsync://example.net/ta.cer"; + let ta_der = b"ta-der".to_vec(); + CirTrustAnchor { + ta_rsync_uri: ta_uri.to_string(), + tal_uri: "https://example.net/root.tal".to_string(), + tal_bytes: format!("{ta_uri}\n\nAQID\n").into_bytes(), + ta_certificate_der: ta_der.clone(), + ta_certificate_sha256: sha256(&ta_der), + } + } + + fn sample_ccr(seq: u32, asn: u32) -> CcrContentInfo { + let vrps = build_roa_payload_state(&[Vrp { + asn, + prefix: IpPrefix { + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: [192, 0, seq as u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + max_length: 24, + }]) + .unwrap(); + let vaps = build_aspa_payload_state(&[AspaAttestation { + customer_as_id: asn, + provider_as_ids: vec![64497], + }]) + .unwrap(); + CcrContentInfo::new(RpkiCanonicalCacheRepresentation { + version: 0, + hash_alg: CcrDigestAlgorithm::Sha256, + produced_at: sample_time(seq), + mfts: None, + vrps: Some(vrps), + vaps: Some(vaps), + tas: None, + rks: None, + }) + } + + fn item(side: &str, seq: u32, ccr: &str, cir: &str) -> Value { + json!({ + "schemaVersion": 1, + "rpId": format!("{side}-rp"), + "side": side, + "seq": seq, + "runId": format!("{side}-{seq}"), + "syncMode": if seq == 1 { "snapshot" } else { "delta" }, + "status": "success", + "validationTime": format_time(sample_time(seq)), + "ccrPath": ccr, + "cirPath": cir + }) + } + + fn jsonl(items: &[Value]) -> String { + let mut out = String::new(); + for item in items { + out.push_str(&serde_json::to_string(item).unwrap()); + out.push('\n'); + } + out + } +} diff --git a/src/validation/tree_runner.rs b/src/validation/tree_runner.rs index 71c6bef..b51f777 100644 --- a/src/validation/tree_runner.rs +++ b/src/validation/tree_runner.rs @@ -1,3 +1,5 @@ +mod vcir_der; + use crate::analysis::timing::TimingHandle; use crate::audit::{ AuditObjectKind, AuditObjectResult, AuditWarning, ObjectAuditEntry, PublicationPointAudit, @@ -9,7 +11,7 @@ use crate::current_repo_index::CurrentRepoIndexHandle; use crate::data_model::aspa::AspaObject; use crate::data_model::crl::RpkixCrl; use crate::data_model::manifest::ManifestObject; -use crate::data_model::rc::{AccessDescription, ResourceCertificate, SubjectInfoAccess}; +use crate::data_model::rc::{ResourceCertificate, SubjectInfoAccess}; use crate::data_model::roa::{RoaAfi, RoaObject}; use crate::data_model::router_cert::{ BgpsecRouterCertificate, BgpsecRouterCertificateDecodeError, BgpsecRouterCertificatePathError, @@ -61,6 +63,8 @@ use serde_json::json; use x509_parser::prelude::FromDer; use x509_parser::x509::SubjectPublicKeyInfo; +use vcir_der::encode_access_description_der_for_vcir_ccr_projection; + #[derive(Clone, Debug, Default)] pub(crate) struct BuildVcirTimingBreakdown { pub(crate) select_crl_ms: u64, @@ -2850,83 +2854,6 @@ fn build_vcir_ccr_manifest_projection_from_fresh( }) } -fn encode_access_description_der_for_vcir_ccr_projection( - access_description: &AccessDescription, -) -> Result, String> { - let oid = encode_oid_der_for_vcir_ccr_projection(&access_description.access_method_oid)?; - let uri = encode_tlv_for_vcir_ccr_projection( - 0x86, - access_description.access_location.as_bytes().to_vec(), - ); - Ok(encode_sequence_for_vcir_ccr_projection(&[oid, uri])) -} - -fn encode_oid_der_for_vcir_ccr_projection(oid: &str) -> Result, String> { - let arcs = oid - .split('.') - .map(|part| { - part.parse::() - .map_err(|_| format!("unsupported accessMethod OID: {oid}")) - }) - .collect::, _>>()?; - if arcs.len() < 2 { - return Err(format!("unsupported accessMethod OID: {oid}")); - } - if arcs[0] > 2 || (arcs[0] < 2 && arcs[1] >= 40) { - return Err(format!("unsupported accessMethod OID: {oid}")); - } - let mut body = Vec::new(); - body.push((arcs[0] * 40 + arcs[1]) as u8); - for arc in &arcs[2..] { - encode_base128_for_vcir_ccr_projection(*arc, &mut body); - } - Ok(encode_tlv_for_vcir_ccr_projection(0x06, body)) -} - -fn encode_base128_for_vcir_ccr_projection(mut value: u64, out: &mut Vec) { - let mut tmp = vec![(value & 0x7F) as u8]; - value >>= 7; - while value > 0 { - tmp.push(((value & 0x7F) as u8) | 0x80); - value >>= 7; - } - tmp.reverse(); - out.extend_from_slice(&tmp); -} - -fn encode_sequence_for_vcir_ccr_projection(elements: &[Vec]) -> Vec { - let total_len: usize = elements.iter().map(Vec::len).sum(); - let mut buf = Vec::with_capacity(total_len); - for element in elements { - buf.extend_from_slice(element); - } - encode_tlv_for_vcir_ccr_projection(0x30, buf) -} - -fn encode_tlv_for_vcir_ccr_projection(tag: u8, value: Vec) -> Vec { - let mut out = Vec::with_capacity(1 + 9 + value.len()); - out.push(tag); - encode_length_for_vcir_ccr_projection(value.len(), &mut out); - out.extend_from_slice(&value); - out -} - -fn encode_length_for_vcir_ccr_projection(len: usize, out: &mut Vec) { - if len < 0x80 { - out.push(len as u8); - return; - } - let mut bytes = Vec::new(); - let mut value = len; - while value > 0 { - bytes.push((value & 0xFF) as u8); - value >>= 8; - } - bytes.reverse(); - out.push(0x80 | (bytes.len() as u8)); - out.extend_from_slice(&bytes); -} - struct CurrentCrlRef<'a> { file: &'a PackFile, crl: RpkixCrl, @@ -3341,3662 +3268,5 @@ fn vrp_prefix_to_string(vrp: &Vrp) -> String { } #[cfg(test)] -mod tests { - use super::*; - use crate::data_model::rc::ResourceCertificate; - use crate::fetch::rsync::LocalDirRsyncFetcher; - use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher}; - use crate::storage::{ - PackFile, PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, - ValidatedManifestMeta, VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, - VcirAuditSummary, VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirOutputType, - VcirRelatedArtifact, VcirSummary, - }; - use crate::sync::rrdp::Fetcher; - use crate::validation::publication_point::PublicationPointSnapshot; - use crate::validation::tree::PublicationPointRunner; - - use std::process::Command; - use std::sync::Arc; - use std::sync::atomic::{AtomicUsize, Ordering}; - - struct NeverHttpFetcher; - impl Fetcher for NeverHttpFetcher { - fn fetch(&self, _uri: &str) -> Result, String> { - Err("http fetch disabled in test".to_string()) - } - } - - struct FailingRsyncFetcher; - impl RsyncFetcher for FailingRsyncFetcher { - fn fetch_objects( - &self, - _rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - Err(RsyncFetchError::Fetch("rsync disabled in test".to_string())) - } - } - - fn sample_runner_with_ccr_accumulator<'a>( - store: &'a RocksStore, - policy: &'a Policy, - ) -> Rpkiv1PublicationPointRunner<'a> { - Rpkiv1PublicationPointRunner { - store, - policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &FailingRsyncFetcher, - validation_time: time::OffsetDateTime::now_utc(), - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: Some(Mutex::new(CcrAccumulator::new(Vec::new()))), - persist_vcir: true, - } - } - - fn openssl_available() -> bool { - Command::new("openssl") - .arg("version") - .output() - .map(|o| o.status.success()) - .unwrap_or(false) - } - - struct Generated { - issuer_ca_der: Vec, - child_ca_der: Vec, - issuer_crl_der: Vec, - } - - fn run(cmd: &mut Command) { - let out = cmd.output().expect("run command"); - if !out.status.success() { - panic!( - "command failed: {:?}\nstdout={}\nstderr={}", - cmd, - String::from_utf8_lossy(&out.stdout), - String::from_utf8_lossy(&out.stderr) - ); - } - } - - fn generate_chain_and_crl() -> Generated { - assert!(openssl_available(), "openssl is required for this test"); - - let td = tempfile::tempdir().expect("tempdir"); - let dir = td.path(); - - std::fs::create_dir_all(dir.join("newcerts")).expect("newcerts"); - std::fs::write(dir.join("index.txt"), b"").expect("index"); - std::fs::write(dir.join("serial"), b"1000\n").expect("serial"); - std::fs::write(dir.join("crlnumber"), b"1000\n").expect("crlnumber"); - - let cnf = format!( - r#" -[ ca ] -default_ca = CA_default - -[ CA_default ] -dir = {dir} -database = $dir/index.txt -new_certs_dir = $dir/newcerts -certificate = $dir/issuer.pem -private_key = $dir/issuer.key -serial = $dir/serial -crlnumber = $dir/crlnumber -default_md = sha256 -default_days = 365 -default_crl_days = 1 -policy = policy_any -x509_extensions = v3_issuer_ca -crl_extensions = crl_ext -unique_subject = no -copy_extensions = none - -[ policy_any ] -commonName = supplied - -[ req ] -prompt = no -distinguished_name = dn - -[ dn ] -CN = Test Issuer CA - -[ v3_issuer_ca ] -basicConstraints = critical,CA:true -keyUsage = critical, keyCertSign, cRLSign -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid:always -certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 -subjectInfoAccess = caRepository;URI:rsync://example.test/repo/issuer/, rpkiManifest;URI:rsync://example.test/repo/issuer/issuer.mft, rpkiNotify;URI:https://example.test/notification.xml -sbgp-ipAddrBlock = critical, IPv4:10.0.0.0/8 -sbgp-autonomousSysNum = critical, AS:64496-64511 - -[ v3_child_ca ] -basicConstraints = critical,CA:true -keyUsage = critical, keyCertSign, cRLSign -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid:always -crlDistributionPoints = URI:rsync://example.test/repo/issuer/issuer.crl -authorityInfoAccess = caIssuers;URI:rsync://example.test/repo/issuer/issuer.cer -certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 -subjectInfoAccess = caRepository;URI:rsync://example.test/repo/child/, rpkiManifest;URI:rsync://example.test/repo/child/child.mft, rpkiNotify;URI:https://example.test/notification.xml -sbgp-ipAddrBlock = critical, IPv4:10.0.0.0/16 -sbgp-autonomousSysNum = critical, AS:64496 - -[ crl_ext ] -authorityKeyIdentifier = keyid:always -"#, - dir = dir.display() - ); - std::fs::write(dir.join("openssl.cnf"), cnf.as_bytes()).expect("write cnf"); - - run(Command::new("openssl") - .arg("genrsa") - .arg("-out") - .arg(dir.join("issuer.key")) - .arg("2048")); - run(Command::new("openssl") - .arg("req") - .arg("-new") - .arg("-x509") - .arg("-sha256") - .arg("-days") - .arg("365") - .arg("-key") - .arg(dir.join("issuer.key")) - .arg("-config") - .arg(dir.join("openssl.cnf")) - .arg("-extensions") - .arg("v3_issuer_ca") - .arg("-out") - .arg(dir.join("issuer.pem"))); - - run(Command::new("openssl") - .arg("genrsa") - .arg("-out") - .arg(dir.join("child.key")) - .arg("2048")); - run(Command::new("openssl") - .arg("req") - .arg("-new") - .arg("-key") - .arg(dir.join("child.key")) - .arg("-subj") - .arg("/CN=Test Child CA") - .arg("-out") - .arg(dir.join("child.csr"))); - - run(Command::new("openssl") - .arg("ca") - .arg("-batch") - .arg("-config") - .arg(dir.join("openssl.cnf")) - .arg("-in") - .arg(dir.join("child.csr")) - .arg("-extensions") - .arg("v3_child_ca") - .arg("-out") - .arg(dir.join("child.pem"))); - - run(Command::new("openssl") - .arg("x509") - .arg("-in") - .arg(dir.join("issuer.pem")) - .arg("-outform") - .arg("DER") - .arg("-out") - .arg(dir.join("issuer.cer"))); - run(Command::new("openssl") - .arg("x509") - .arg("-in") - .arg(dir.join("child.pem")) - .arg("-outform") - .arg("DER") - .arg("-out") - .arg(dir.join("child.cer"))); - - run(Command::new("openssl") - .arg("ca") - .arg("-gencrl") - .arg("-config") - .arg(dir.join("openssl.cnf")) - .arg("-out") - .arg(dir.join("issuer.crl.pem"))); - run(Command::new("openssl") - .arg("crl") - .arg("-in") - .arg(dir.join("issuer.crl.pem")) - .arg("-outform") - .arg("DER") - .arg("-out") - .arg(dir.join("issuer.crl"))); - - Generated { - issuer_ca_der: std::fs::read(dir.join("issuer.cer")).expect("read issuer der"), - child_ca_der: std::fs::read(dir.join("child.cer")).expect("read child der"), - issuer_crl_der: std::fs::read(dir.join("issuer.crl")).expect("read crl der"), - } - } - - struct GeneratedRouter { - issuer_ca_der: Vec, - router_der: Vec, - issuer_crl_der: Vec, - } - - fn generate_router_cert_with_variant(key_spec: &str, include_eku: bool) -> GeneratedRouter { - assert!(openssl_available(), "openssl is required for this test"); - - let td = tempfile::tempdir().expect("tempdir"); - let dir = td.path(); - - std::fs::create_dir_all(dir.join("newcerts")).expect("newcerts"); - std::fs::write(dir.join("index.txt"), b"").expect("index"); - std::fs::write(dir.join("serial"), b"1000\n").expect("serial"); - std::fs::write(dir.join("crlnumber"), b"1000\n").expect("crlnumber"); - - let eku_line = if include_eku { - "extendedKeyUsage = 1.3.6.1.5.5.7.3.30" - } else { - "" - }; - let cnf = format!( - r#" -[ ca ] -default_ca = CA_default - -[ CA_default ] -dir = {dir} -database = $dir/index.txt -new_certs_dir = $dir/newcerts -certificate = $dir/issuer.pem -private_key = $dir/issuer.key -serial = $dir/serial -crlnumber = $dir/crlnumber -default_md = sha256 -default_days = 365 -default_crl_days = 1 -policy = policy_any -x509_extensions = v3_issuer_ca -crl_extensions = crl_ext -unique_subject = no -copy_extensions = none - -[ policy_any ] -commonName = supplied - -[ req ] -prompt = no -distinguished_name = dn - -[ dn ] -CN = Test Issuer CA - -[ v3_issuer_ca ] -basicConstraints = critical,CA:true -keyUsage = critical, keyCertSign, cRLSign -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid:always -certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 -subjectInfoAccess = caRepository;URI:rsync://example.test/repo/issuer/, rpkiManifest;URI:rsync://example.test/repo/issuer/issuer.mft, rpkiNotify;URI:https://example.test/notification.xml -sbgp-ipAddrBlock = critical, IPv4:10.0.0.0/8 -sbgp-autonomousSysNum = critical, AS:64496-64511 - -[ v3_router ] -keyUsage = critical, digitalSignature -{eku_line} -authorityKeyIdentifier = keyid:always -crlDistributionPoints = URI:rsync://example.test/repo/issuer/issuer.crl -authorityInfoAccess = caIssuers;URI:rsync://example.test/repo/issuer/issuer.cer -certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 -sbgp-autonomousSysNum = critical, AS:64496 - -[ crl_ext ] -authorityKeyIdentifier = keyid:always -"#, - dir = dir.display(), - eku_line = eku_line, - ); - std::fs::write(dir.join("openssl.cnf"), cnf.as_bytes()).expect("write cnf"); - - run(Command::new("openssl") - .arg("genrsa") - .arg("-out") - .arg(dir.join("issuer.key")) - .arg("2048")); - run(Command::new("openssl") - .arg("req") - .arg("-new") - .arg("-x509") - .arg("-sha256") - .arg("-days") - .arg("365") - .arg("-key") - .arg(dir.join("issuer.key")) - .arg("-config") - .arg(dir.join("openssl.cnf")) - .arg("-extensions") - .arg("v3_issuer_ca") - .arg("-out") - .arg(dir.join("issuer.pem"))); - - match key_spec { - "ec-p256" => run(Command::new("openssl") - .arg("ecparam") - .arg("-name") - .arg("prime256v1") - .arg("-genkey") - .arg("-noout") - .arg("-out") - .arg(dir.join("router.key"))), - "ec-p384" => run(Command::new("openssl") - .arg("ecparam") - .arg("-name") - .arg("secp384r1") - .arg("-genkey") - .arg("-noout") - .arg("-out") - .arg(dir.join("router.key"))), - other => panic!("unsupported key_spec {other}"), - } - - run(Command::new("openssl") - .arg("req") - .arg("-new") - .arg("-key") - .arg(dir.join("router.key")) - .arg("-subj") - .arg("/CN=ROUTER-0000FC10/serialNumber=01020304") - .arg("-out") - .arg(dir.join("router.csr"))); - - run(Command::new("openssl") - .arg("ca") - .arg("-batch") - .arg("-config") - .arg(dir.join("openssl.cnf")) - .arg("-in") - .arg(dir.join("router.csr")) - .arg("-extensions") - .arg("v3_router") - .arg("-out") - .arg(dir.join("router.pem"))); - - run(Command::new("openssl") - .arg("x509") - .arg("-in") - .arg(dir.join("issuer.pem")) - .arg("-outform") - .arg("DER") - .arg("-out") - .arg(dir.join("issuer.cer"))); - run(Command::new("openssl") - .arg("x509") - .arg("-in") - .arg(dir.join("router.pem")) - .arg("-outform") - .arg("DER") - .arg("-out") - .arg(dir.join("router.cer"))); - - run(Command::new("openssl") - .arg("ca") - .arg("-gencrl") - .arg("-config") - .arg(dir.join("openssl.cnf")) - .arg("-out") - .arg(dir.join("issuer.crl.pem"))); - run(Command::new("openssl") - .arg("crl") - .arg("-in") - .arg(dir.join("issuer.crl.pem")) - .arg("-outform") - .arg("DER") - .arg("-out") - .arg(dir.join("issuer.crl"))); - - GeneratedRouter { - issuer_ca_der: std::fs::read(dir.join("issuer.cer")).expect("read issuer der"), - router_der: std::fs::read(dir.join("router.cer")).expect("read router der"), - issuer_crl_der: std::fs::read(dir.join("issuer.crl")).expect("read crl der"), - } - } - fn dummy_pack_with_files(files: Vec) -> PublicationPointSnapshot { - let now = time::OffsetDateTime::now_utc(); - PublicationPointSnapshot { - format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_number_be: vec![1], - this_update: PackTime::from_utc_offset_datetime(now), - next_update: PackTime::from_utc_offset_datetime(now + time::Duration::hours(1)), - verified_at: PackTime::from_utc_offset_datetime(now), - manifest_bytes: vec![0x01], - files, - } - } - - fn cernet_publication_point_snapshot_for_vcir_tests() - -> (PublicationPointSnapshot, Vec, time::OffsetDateTime) { - let dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); - let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/"; - let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; - let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); - let manifest_bytes = std::fs::read(dir.join(manifest_file)).expect("read manifest fixture"); - let manifest = - ManifestObject::decode_der(&manifest_bytes).expect("decode manifest fixture"); - let candidate = manifest.manifest.this_update + time::Duration::seconds(60); - let validation_time = if candidate < manifest.manifest.next_update { - candidate - } else { - manifest.manifest.this_update - }; - - let issuer_ca_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - ) - .expect("read issuer ca fixture"); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy { - sync_preference: crate::policy::SyncPreference::RsyncOnly, - ..Policy::default() - }; - - sync_publication_point( - &store, - &policy, - None, - rsync_base_uri, - &NeverHttpFetcher, - &LocalDirRsyncFetcher::new(&dir), - None, - None, - ) - .expect("sync cernet fixture"); - - let pp = crate::validation::manifest::process_manifest_publication_point( - &store, - &policy, - &manifest_rsync_uri, - rsync_base_uri, - issuer_ca_der.as_slice(), - Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - validation_time, - ) - .expect("process manifest publication point"); - - (pp.snapshot, issuer_ca_der, validation_time) - } - - fn sample_vcir_for_projection( - now: time::OffsetDateTime, - child_cert_hash: &str, - ) -> ValidatedCaInstanceResult { - let manifest_uri = "rsync://example.test/repo/issuer/issuer.mft".to_string(); - let current_crl_uri = "rsync://example.test/repo/issuer/issuer.crl".to_string(); - let child_cert_uri = "rsync://example.test/repo/issuer/child.cer".to_string(); - let child_manifest_uri = "rsync://example.test/repo/child/child.mft".to_string(); - let roa_uri = "rsync://example.test/repo/issuer/a.roa".to_string(); - let aspa_uri = "rsync://example.test/repo/issuer/a.asa".to_string(); - let router_uri = "rsync://example.test/repo/issuer/router.cer".to_string(); - let manifest_hash = sha256_hex(b"manifest-bytes"); - let current_crl_hash = sha256_hex(b"current-crl-bytes"); - let roa_hash = sha256_hex(b"roa-bytes"); - let aspa_hash = sha256_hex(b"aspa-bytes"); - let router_hash = sha256_hex(b"router-bytes"); - let ee_hash = sha256_hex(b"ee-cert-bytes"); - let gate_until = PackTime::from_utc_offset_datetime(now + time::Duration::hours(1)); - let ccr_manifest_projection = VcirCcrManifestProjection { - manifest_rsync_uri: manifest_uri.clone(), - manifest_sha256: hex::decode(&manifest_hash).expect("decode manifest hash"), - manifest_size: 2048, - manifest_ee_aki: vec![0x11; 20], - manifest_number_be: vec![1], - manifest_this_update: PackTime::from_utc_offset_datetime(now), - manifest_sia_locations_der: vec![vec![ - 0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05, - b'r', b's', b'y', b'n', b'c', - ]], - subordinate_skis: vec![vec![0x33; 20]], - }; - ValidatedCaInstanceResult { - manifest_rsync_uri: manifest_uri.clone(), - parent_manifest_rsync_uri: None, - tal_id: "test-tal".to_string(), - ca_subject_name: "CN=Issuer".to_string(), - ca_ski: "11".repeat(20), - issuer_ski: "22".repeat(20), - last_successful_validation_time: PackTime::from_utc_offset_datetime(now), - current_manifest_rsync_uri: manifest_uri.clone(), - current_crl_rsync_uri: current_crl_uri.clone(), - validated_manifest_meta: ValidatedManifestMeta { - validated_manifest_number: vec![1], - validated_manifest_this_update: PackTime::from_utc_offset_datetime(now), - validated_manifest_next_update: gate_until.clone(), - }, - ccr_manifest_projection, - instance_gate: VcirInstanceGate { - manifest_next_update: gate_until.clone(), - current_crl_next_update: gate_until.clone(), - self_ca_not_after: PackTime::from_utc_offset_datetime(now + time::Duration::hours(2)), - instance_effective_until: gate_until.clone(), - }, - child_entries: vec![VcirChildEntry { - child_manifest_rsync_uri: child_manifest_uri, - child_cert_rsync_uri: child_cert_uri.clone(), - child_cert_hash: child_cert_hash.to_string(), - child_ski: "33".repeat(20), - child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(), - child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), - child_rrdp_notification_uri: Some("https://example.test/child-notify.xml".to_string()), - child_effective_ip_resources: None, - child_effective_as_resources: None, - accepted_at_validation_time: PackTime::from_utc_offset_datetime(now), - }], - local_outputs: vec![ - VcirLocalOutput { - output_id: sha256_hex(b"vrp-out"), - output_type: VcirOutputType::Vrp, - item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(30)), - source_object_uri: roa_uri.clone(), - source_object_type: "roa".to_string(), - source_object_hash: roa_hash.clone(), - source_ee_cert_hash: ee_hash.clone(), - payload_json: serde_json::json!({"asn": 64496, "prefix": "203.0.113.0/24", "max_length": 24}).to_string(), - rule_hash: sha256_hex(b"roa-rule"), - validation_path_hint: vec![manifest_uri.clone(), roa_uri.clone()], - }, - VcirLocalOutput { - output_id: sha256_hex(b"aspa-out"), - output_type: VcirOutputType::Aspa, - item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(30)), - source_object_uri: aspa_uri.clone(), - source_object_type: "aspa".to_string(), - source_object_hash: aspa_hash.clone(), - source_ee_cert_hash: ee_hash, - payload_json: serde_json::json!({"customer_as_id": 64496, "provider_as_ids": [64497, 64498]}).to_string(), - rule_hash: sha256_hex(b"aspa-rule"), - validation_path_hint: vec![manifest_uri.clone(), aspa_uri.clone()], - }, - VcirLocalOutput { - output_id: sha256_hex(b"router-key-out"), - output_type: VcirOutputType::RouterKey, - item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(30)), - source_object_uri: router_uri.clone(), - source_object_type: "router_key".to_string(), - source_object_hash: router_hash.clone(), - source_ee_cert_hash: router_hash.clone(), - payload_json: serde_json::json!({ - "as_id": 64496, - "ski_hex": "11".repeat(20), - "spki_der_base64": base64::engine::general_purpose::STANDARD.encode([0x30u8, 0x00]), - }).to_string(), - rule_hash: sha256_hex(b"router-key-rule"), - validation_path_hint: vec![manifest_uri.clone(), router_uri.clone()], - }, - ], - related_artifacts: vec![ - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::Manifest, - artifact_kind: VcirArtifactKind::Mft, - uri: Some(manifest_uri.clone()), - sha256: manifest_hash, - object_type: Some("mft".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::CurrentCrl, - artifact_kind: VcirArtifactKind::Crl, - uri: Some(current_crl_uri), - sha256: current_crl_hash, - object_type: Some("crl".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::ChildCaCert, - artifact_kind: VcirArtifactKind::Cer, - uri: Some(child_cert_uri), - sha256: child_cert_hash.to_string(), - object_type: Some("cer".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: VcirArtifactKind::Roa, - uri: Some(roa_uri), - sha256: roa_hash, - object_type: Some("roa".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: VcirArtifactKind::Aspa, - uri: Some(aspa_uri), - sha256: aspa_hash, - object_type: Some("aspa".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }, - ], - summary: VcirSummary { - local_vrp_count: 1, - local_aspa_count: 1, - local_router_key_count: 1, - child_count: 1, - accepted_object_count: 4, - rejected_object_count: 0, - }, - audit_summary: VcirAuditSummary { - failed_fetch_eligible: true, - last_failed_fetch_reason: None, - warning_count: 0, - audit_flags: Vec::new(), - }, - } - } - - #[test] - fn never_http_fetcher_returns_error() { - let f = NeverHttpFetcher; - let err = f.fetch("https://example.test/").unwrap_err(); - assert!(err.contains("disabled"), "{err}"); - } - - #[test] - fn kind_from_rsync_uri_classifies_known_extensions() { - assert_eq!( - kind_from_rsync_uri("rsync://example.test/x.crl"), - AuditObjectKind::Crl - ); - assert_eq!( - kind_from_rsync_uri("rsync://example.test/x.cer"), - AuditObjectKind::Certificate - ); - assert_eq!( - kind_from_rsync_uri("rsync://example.test/x.roa"), - AuditObjectKind::Roa - ); - assert_eq!( - kind_from_rsync_uri("rsync://example.test/x.asa"), - AuditObjectKind::Aspa - ); - assert_eq!( - kind_from_rsync_uri("rsync://example.test/x.bin"), - AuditObjectKind::Other - ); - } - - #[test] - fn build_vcir_local_outputs_prefers_cached_outputs() { - let pack = dummy_pack_with_files(vec![]); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: vec![1], - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - let cached = vec![VcirLocalOutput { - output_id: "cached-output".to_string(), - output_type: VcirOutputType::Vrp, - item_effective_until: pack.next_update.clone(), - source_object_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), - source_object_type: "roa".to_string(), - source_object_hash: sha256_hex(b"cached-roa"), - source_ee_cert_hash: sha256_hex(b"cached-ee"), - payload_json: "{\"asn\":64500}".to_string(), - rule_hash: sha256_hex(b"cached-rule"), - validation_path_hint: vec![pack.manifest_rsync_uri.clone()], - }]; - let outputs = build_vcir_local_outputs( - &ca, - &pack, - &crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: cached.clone(), - warnings: Vec::new(), - stats: crate::validation::objects::ObjectsStats::default(), - audit: Vec::new(), - }, - ) - .expect("reuse cached outputs"); - assert_eq!(outputs, cached); - } - - #[test] - fn persist_vcir_non_repository_evidence_stores_current_ca_cert_only() { - let (pack, issuer_ca_der, validation_time) = - cernet_publication_point_snapshot_for_vcir_tests(); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( - &pack, - &Policy::default(), - issuer_ca_der.as_slice(), - Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - issuer_ca.tbs.extensions.ip_resources.as_ref(), - issuer_ca.tbs.extensions.as_resources.as_ref(), - validation_time, - None, - ); - assert!( - !objects.local_outputs_cache.is_empty(), - "expected local outputs from signed objects" - ); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: issuer_ca_der.clone(), - ca_certificate_rsync_uri: Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), - ), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - persist_vcir_non_repository_evidence(&store, &ca).expect("persist embedded evidence"); - - let issuer_hash = sha256_hex(&issuer_ca_der); - let issuer_entry = store - .get_raw_by_hash_entry(&issuer_hash) - .expect("load issuer raw entry") - .expect("issuer raw entry present"); - assert!( - issuer_entry - .origin_uris - .iter() - .any(|uri| uri.ends_with("BfycW4hQb3wNP4YsiJW-1n6fjro.cer")) - ); - let first_output = objects - .local_outputs_cache - .first() - .expect("first local output"); - assert!( - store - .get_raw_by_hash_entry(&first_output.source_ee_cert_hash) - .expect("load source ee raw") - .is_none() - ); - } - - #[test] - fn build_router_key_local_outputs_encodes_router_key_payloads() { - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: Vec::new(), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - let outputs = build_router_key_local_outputs( - &ca, - &[RouterKeyPayload { - as_id: 64496, - ski: vec![0x11; 20], - spki_der: vec![0x30, 0x00], - source_object_uri: "rsync://example.test/repo/issuer/router.cer".to_string(), - source_object_hash: "11".repeat(32), - source_ee_cert_hash: "11".repeat(32), - item_effective_until: PackTime { - rfc3339_utc: "2026-12-31T00:00:00Z".to_string(), - }, - }], - ); - assert_eq!(outputs.len(), 1); - assert_eq!(outputs[0].output_type, VcirOutputType::RouterKey); - assert_eq!(outputs[0].source_object_type, "router_key"); - assert!(outputs[0].payload_json.contains("spki_der_base64")); - } - - #[test] - fn build_vcir_local_outputs_falls_back_to_decoding_accepted_objects_when_cache_is_empty() { - let (pack, issuer_ca_der, validation_time) = - cernet_publication_point_snapshot_for_vcir_tests(); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( - &pack, - &Policy::default(), - issuer_ca_der.as_slice(), - Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - issuer_ca.tbs.extensions.ip_resources.as_ref(), - issuer_ca.tbs.extensions.as_resources.as_ref(), - validation_time, - None, - ); - let mut objects_without_cache = objects.clone(); - objects_without_cache.local_outputs_cache.clear(); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: issuer_ca_der, - ca_certificate_rsync_uri: Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), - ), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - let local_outputs = build_vcir_local_outputs(&ca, &pack, &objects_without_cache) - .expect("rebuild vcir local outputs"); - assert!(!local_outputs.is_empty()); - assert_eq!(local_outputs.len(), objects.vrps.len()); - assert!( - local_outputs - .iter() - .all(|output| output.output_type == VcirOutputType::Vrp) - ); - } - - #[test] - fn finalize_fresh_publication_point_releases_local_outputs_cache_after_persist() { - let (pack, issuer_ca_der, validation_time) = - cernet_publication_point_snapshot_for_vcir_tests(); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - let mut objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( - &pack, - &Policy::default(), - issuer_ca_der.as_slice(), - Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - issuer_ca.tbs.extensions.ip_resources.as_ref(), - issuer_ca.tbs.extensions.as_resources.as_ref(), - validation_time, - None, - ); - assert!( - !objects.local_outputs_cache.is_empty(), - "expected local outputs from signed objects" - ); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &FailingRsyncFetcher, - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - }; - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: issuer_ca_der.clone(), - ca_certificate_rsync_uri: Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), - ), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - let fresh_point = FreshValidatedPublicationPoint { - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - manifest_number_be: pack.manifest_number_be.clone(), - this_update: pack.this_update.clone(), - next_update: pack.next_update.clone(), - verified_at: pack.verified_at.clone(), - manifest_bytes: pack.manifest_bytes.clone(), - files: pack.files.clone(), - }; - - objects.local_outputs_cache.shrink_to_fit(); - let original_cache_capacity = objects.local_outputs_cache.capacity(); - let finalized = runner - .finalize_fresh_publication_point_from_reducer( - &ca, - &fresh_point, - Vec::new(), - objects, - Vec::new(), - Vec::new(), - None, - None, - 0, - None, - ) - .expect("finalize fresh publication point"); - - assert!( - finalized.result.objects.local_outputs_cache.is_empty(), - "local outputs cache should be released after VCIR persistence" - ); - assert_eq!( - finalized.result.objects.local_outputs_cache.capacity(), - 0, - "released cache should not keep its backing allocation" - ); - assert!(original_cache_capacity > 0); - - let persisted = store - .get_vcir(&pack.manifest_rsync_uri) - .expect("load persisted vcir") - .expect("persisted vcir"); - assert!( - !persisted.local_outputs.is_empty(), - "VCIR should still persist local outputs before cache release" - ); - } - - #[test] - fn persist_vcir_for_fresh_result_stores_vcir_and_audit_indexes_for_real_snapshot() { - let (pack, issuer_ca_der, validation_time) = - cernet_publication_point_snapshot_for_vcir_tests(); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( - &pack, - &Policy::default(), - issuer_ca_der.as_slice(), - Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - issuer_ca.tbs.extensions.ip_resources.as_ref(), - issuer_ca.tbs.extensions.as_resources.as_ref(), - validation_time, - None, - ); - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: issuer_ca_der.clone(), - ca_certificate_rsync_uri: Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), - ), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - - persist_vcir_for_fresh_result(&store, &ca, &pack, &objects, &[], &[], &[], validation_time) - .expect("persist vcir for fresh result"); - - let vcir = store - .get_vcir(&pack.manifest_rsync_uri) - .expect("get vcir") - .expect("vcir exists"); - assert_eq!(vcir.manifest_rsync_uri, pack.manifest_rsync_uri); - assert_eq!(vcir.summary.local_vrp_count as usize, objects.vrps.len()); - assert_eq!( - vcir.ccr_manifest_projection.manifest_rsync_uri, - pack.manifest_rsync_uri - ); - assert_eq!( - vcir.ccr_manifest_projection.manifest_number_be, - pack.manifest_number_be - ); - assert_eq!( - vcir.ccr_manifest_projection.manifest_this_update, - pack.this_update - ); - assert_eq!( - vcir.ccr_manifest_projection.manifest_size, - pack.manifest_bytes.len() as u64 - ); - let first_output = vcir.local_outputs.first().expect("local outputs stored"); - assert!( - store - .get_audit_rule_index_entry( - crate::storage::AuditRuleKind::Roa, - &first_output.rule_hash - ) - .expect("get audit rule index entry") - .is_some() - ); - } - - #[test] - fn build_vcir_ccr_manifest_projection_from_fresh_real_snapshot_matches_manifest_contents() { - let (pack, issuer_ca_der, validation_time) = - cernet_publication_point_snapshot_for_vcir_tests(); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: issuer_ca_der, - ca_certificate_rsync_uri: Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), - ), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - let child_discovery = - discover_children_from_fresh_snapshot_with_audit(&ca, &pack, validation_time, None) - .expect("discover children"); - let child_entries = build_vcir_child_entries(&child_discovery.children, validation_time) - .expect("build child entries"); - - let projection = build_vcir_ccr_manifest_projection_from_fresh(&ca, &pack, &child_entries) - .expect("build ccr manifest projection"); - let manifest = ManifestObject::decode_der(&pack.manifest_bytes).expect("decode manifest"); - let expected_locations = match manifest.signed_object.signed_data.certificates[0] - .resource_cert - .tbs - .extensions - .subject_info_access - .as_ref() - .expect("manifest sia") - { - SubjectInfoAccess::Ee(ee_sia) => ee_sia - .access_descriptions - .iter() - .map(encode_access_description_der_for_vcir_ccr_projection) - .collect::, _>>() - .expect("encode locations"), - SubjectInfoAccess::Ca(_) => panic!("manifest ee SIA should not be CA variant"), - }; - - assert_eq!(projection.manifest_rsync_uri, pack.manifest_rsync_uri); - assert_eq!( - projection.manifest_sha256, - sha2::Sha256::digest(&pack.manifest_bytes).to_vec() - ); - assert_eq!(projection.manifest_size, pack.manifest_bytes.len() as u64); - assert_eq!( - projection.manifest_ee_aki, - manifest.signed_object.signed_data.certificates[0] - .resource_cert - .tbs - .extensions - .authority_key_identifier - .clone() - .expect("manifest aki") - ); - assert_eq!( - projection.manifest_number_be, - manifest.manifest.manifest_number.bytes_be - ); - assert_eq!(projection.manifest_this_update, pack.this_update); - assert_eq!(projection.manifest_sia_locations_der, expected_locations); - let expected_subordinate_skis = child_entries - .iter() - .map(|child| hex::decode(&child.child_ski).expect("decode child ski")) - .collect::>(); - assert_eq!(projection.subordinate_skis, expected_subordinate_skis); - } - - #[test] - fn build_vcir_related_artifacts_classifies_snapshot_files_and_audit_statuses() { - let manifest_bytes = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft", - ), - ) - .expect("read manifest fixture"); - let crl_bytes = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl", - ), - ) - .expect("read crl fixture"); - let pack = PublicationPointSnapshot { - format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_number_be: vec![1], - this_update: PackTime::from_utc_offset_datetime(time::OffsetDateTime::now_utc()), - next_update: PackTime::from_utc_offset_datetime( - time::OffsetDateTime::now_utc() + time::Duration::hours(1), - ), - verified_at: PackTime::from_utc_offset_datetime(time::OffsetDateTime::now_utc()), - manifest_bytes, - files: vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - crl_bytes, - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/child.cer", - vec![1u8, 2], - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/a.roa", - vec![3u8, 4], - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/a.asa", - vec![5u8, 6], - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/a.gbr", - vec![7u8, 8], - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/extra.bin", - vec![9u8], - ), - ], - }; - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: vec![0x11, 0x22], - ca_certificate_rsync_uri: Some( - "rsync://example.test/repo/issuer/issuer.cer".to_string(), - ), - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - let objects = crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: Vec::new(), - stats: crate::validation::objects::ObjectsStats::default(), - audit: vec![ - ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), - sha256_hex: sha256_hex_from_32(&pack.files[2].sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Error, - detail: Some("bad roa".to_string()), - }, - ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/issuer/a.asa".to_string(), - sha256_hex: sha256_hex_from_32(&pack.files[3].sha256), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Skipped, - detail: Some("skipped aspa".to_string()), - }, - ], - }; - let artifacts = build_vcir_related_artifacts( - &ca, - &pack, - "rsync://example.test/repo/issuer/issuer.crl", - &objects, - &[], - ); - assert!( - artifacts - .iter() - .any(|artifact| artifact.artifact_role == VcirArtifactRole::Manifest) - ); - assert!( - artifacts - .iter() - .any(|artifact| artifact.artifact_role == VcirArtifactRole::TrustAnchorCert) - ); - assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() - == Some("rsync://example.test/repo/issuer/issuer.crl") - && artifact.artifact_role == VcirArtifactRole::CurrentCrl)); - assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() - == Some("rsync://example.test/repo/issuer/child.cer") - && artifact.artifact_role == VcirArtifactRole::ChildCaCert)); - assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() - == Some("rsync://example.test/repo/issuer/a.roa") - && artifact.validation_status == VcirArtifactValidationStatus::Rejected)); - assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() - == Some("rsync://example.test/repo/issuer/a.asa") - && artifact.validation_status == VcirArtifactValidationStatus::WarningOnly)); - assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() - == Some("rsync://example.test/repo/issuer/a.gbr") - && artifact.artifact_kind == VcirArtifactKind::Gbr)); - assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() - == Some("rsync://example.test/repo/issuer/extra.bin") - && artifact.artifact_kind == VcirArtifactKind::Other)); - assert!( - !artifacts - .iter() - .any(|artifact| artifact.uri.is_none() - && artifact.sha256 == sha256_hex(b"embedded-ee")), - "embedded EE cert artifacts should no longer be persisted separately" - ); - } - - #[test] - fn select_issuer_crl_from_snapshot_reports_missing_crldp_for_self_signed_cert() { - let ta_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/ta/apnic-ta.cer"), - ) - .expect("read TA fixture"); - - let pack = dummy_pack_with_files(vec![]); - let err = select_issuer_crl_from_snapshot(&ta_der, &pack).unwrap_err(); - assert!(err.contains("CRLDistributionPoints missing"), "{err}"); - } - - #[test] - fn select_issuer_crl_from_snapshot_finds_matching_crl() { - // Use real fixtures to ensure child cert has CRLDP rsync URI and CRL exists. - let child_cert_der = - std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer", - )) - .expect("read child cert fixture"); - let crl_der = std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl", - )) - .expect("read crl fixture"); - - let pack = dummy_pack_with_files(vec![PackFile::from_bytes_compute_sha256( - "rsync://ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl", - crl_der.clone(), - )]); - - let (uri, found) = - select_issuer_crl_from_snapshot(child_cert_der.as_slice(), &pack).expect("find crl"); - assert_eq!( - uri, - "rsync://ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl" - ); - assert_eq!(found, crl_der.as_slice()); - } - - #[test] - fn discover_children_from_fresh_pack_discovers_child_ca() { - let g = generate_chain_and_crl(); - - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der.clone(), - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/child.cer", - g.child_ca_der.clone(), - ), - ]); - - let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: g.issuer_ca_der.clone(), - ca_certificate_rsync_uri: None, - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let now = time::OffsetDateTime::now_utc(); - let children = discover_children_from_fresh_snapshot_with_audit(&issuer, &pack, now, None) - .expect("discover children") - .children; - assert_eq!(children.len(), 1); - assert_eq!( - children[0].discovered_from.parent_manifest_rsync_uri, - issuer.manifest_rsync_uri - ); - assert_eq!( - children[0].discovered_from.child_ca_certificate_rsync_uri, - "rsync://example.test/repo/issuer/child.cer" - ); - assert_eq!( - children[0].handle.rsync_base_uri, - "rsync://example.test/repo/child/".to_string() - ); - assert_eq!( - children[0].handle.manifest_rsync_uri, - "rsync://example.test/repo/child/child.mft".to_string() - ); - assert_eq!( - children[0].handle.publication_point_rsync_uri, - "rsync://example.test/repo/child/".to_string() - ); - assert_eq!( - children[0].handle.rrdp_notification_uri.as_deref(), - Some("https://example.test/notification.xml") - ); - } - - #[test] - fn discover_children_with_audit_records_missing_crl_for_child_certificate() { - let now = time::OffsetDateTime::now_utc(); - - let child_ca_der = - std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer", - )) - .expect("read child ca fixture"); - - // Pack contains the child CA cert but does not contain the CRL referenced by the child - // certificate CRLDistributionPoints extension. - let pack = dummy_pack_with_files(vec![PackFile::from_bytes_compute_sha256( - "rsync://ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer", - child_ca_der, - )]); - - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: vec![1], - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - - let out = discover_children_from_fresh_snapshot_with_audit(&issuer, &pack, now, None) - .expect("discovery should succeed with audit error"); - assert_eq!(out.children.len(), 0); - assert_eq!(out.audits.len(), 1); - assert_eq!( - out.audits[0].rsync_uri, - "rsync://ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer" - ); - assert_eq!(out.audits[0].result, AuditObjectResult::Error); - assert!( - out.audits[0] - .detail - .as_deref() - .unwrap_or("") - .contains("cannot select issuer CRL"), - "expected deterministic CRL selection failure to be recorded" - ); - } - - #[test] - fn runner_offline_rsync_fixture_produces_pack_and_warnings() { - let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); - assert!(fixture_dir.is_dir(), "fixture directory must exist"); - - let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); - let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; - let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); - - // Pick a validation_time inside the fixture manifest's validity window to keep this - // test stable across wall-clock time. - let fixture_manifest_bytes = - std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); - let fixture_manifest = - crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) - .expect("decode manifest fixture"); - let validation_time = { - let this_update = fixture_manifest.manifest.this_update; - let next_update = fixture_manifest.manifest.next_update; - let candidate = this_update + time::Duration::seconds(60); - if candidate < next_update { - candidate - } else { - this_update - } - }; - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy { - sync_preference: crate::policy::SyncPreference::RsyncOnly, - ..Policy::default() - }; - - let runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - }; - - // For this fixture-driven smoke, we provide the correct issuer CA certificate (the CA for - // this publication point) so ROA EE certificate paths can validate. - let issuer_ca_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - ) - .expect("read issuer ca fixture"); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - - let handle = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: issuer_ca_der, - ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: rsync_base_uri.clone(), - manifest_rsync_uri: manifest_rsync_uri.clone(), - publication_point_rsync_uri: rsync_base_uri.clone(), - rrdp_notification_uri: None, - }; - - let out = runner - .run_publication_point(&handle) - .expect("run publication point"); - assert_eq!(out.source, PublicationPointSource::Fresh); - let pack = out.snapshot.expect("fresh run pack"); - assert_eq!(pack.manifest_rsync_uri, manifest_rsync_uri); - assert!(pack.files.len() > 1); - assert!( - out.objects.vrps.len() > 1, - "expected to extract VRPs from ROAs" - ); - - let vcir = store - .get_vcir(&manifest_rsync_uri) - .expect("get vcir") - .expect("vcir exists after fresh run"); - assert_eq!(vcir.manifest_rsync_uri, manifest_rsync_uri); - assert_eq!(vcir.tal_id, "test-tal"); - assert!( - vcir.local_outputs - .iter() - .any(|output| output.output_type == crate::storage::VcirOutputType::Vrp), - "expected VCIR local_outputs to contain VRP entries" - ); - let first_vrp = vcir - .local_outputs - .iter() - .find(|output| output.output_type == crate::storage::VcirOutputType::Vrp) - .expect("first VCIR VRP output"); - let audit_rule = store - .get_audit_rule_index_entry(crate::storage::AuditRuleKind::Roa, &first_vrp.rule_hash) - .expect("get audit rule index") - .expect("audit rule index exists"); - assert_eq!(audit_rule.manifest_rsync_uri, manifest_rsync_uri); - assert_eq!(audit_rule.output_id, first_vrp.output_id); - let trace = crate::audit_trace::trace_rule_to_root( - &store, - crate::storage::AuditRuleKind::Roa, - &first_vrp.rule_hash, - ) - .expect("trace rule") - .expect("trace exists"); - assert_eq!(trace.chain_leaf_to_root.len(), 1); - assert_eq!( - trace.chain_leaf_to_root[0].manifest_rsync_uri, - manifest_rsync_uri - ); - assert!(trace.source_object_raw.raw_present); - assert!(trace.source_ee_cert_raw.raw_present); - } - - #[test] - fn runner_rsync_dedup_skips_second_sync_for_same_base() { - let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); - assert!(fixture_dir.is_dir(), "fixture directory must exist"); - - let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); - let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; - let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); - - let fixture_manifest_bytes = - std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); - let fixture_manifest = - crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) - .expect("decode manifest fixture"); - let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy { - sync_preference: crate::policy::SyncPreference::RsyncOnly, - ..Policy::default() - }; - - let issuer_ca_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - ) - .expect("read issuer ca fixture"); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - - let handle = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: issuer_ca_der, - ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: rsync_base_uri.clone(), - manifest_rsync_uri: manifest_rsync_uri.clone(), - publication_point_rsync_uri: rsync_base_uri.clone(), - rrdp_notification_uri: None, - }; - - struct CountingRsyncFetcher { - inner: LocalDirRsyncFetcher, - calls: Arc, - } - impl RsyncFetcher for CountingRsyncFetcher { - fn fetch_objects( - &self, - rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - self.calls.fetch_add(1, Ordering::SeqCst); - self.inner.fetch_objects(rsync_base_uri) - } - } - - let calls = Arc::new(AtomicUsize::new(0)); - let rsync = CountingRsyncFetcher { - inner: LocalDirRsyncFetcher::new(&fixture_dir), - calls: calls.clone(), - }; - - let runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &rsync, - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - }; - - let first = runner.run_publication_point(&handle).expect("first run ok"); - assert_eq!(first.source, PublicationPointSource::Fresh); - - let second = runner - .run_publication_point(&handle) - .expect("second run ok"); - assert_eq!(second.source, PublicationPointSource::Fresh); - - assert_eq!( - calls.load(Ordering::SeqCst), - 1, - "rsync should be called once" - ); - } - - #[test] - fn runner_rsync_dedup_skips_second_sync_for_same_module_scope() { - let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); - assert!(fixture_dir.is_dir(), "fixture directory must exist"); - - let first_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); - let second_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/sub/".to_string(); - let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; - let manifest_rsync_uri = format!("{first_base_uri}{manifest_file}"); - - let fixture_manifest_bytes = - std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); - let fixture_manifest = - crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) - .expect("decode manifest fixture"); - let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy { - sync_preference: crate::policy::SyncPreference::RsyncOnly, - ..Policy::default() - }; - - let issuer_ca_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - ) - .expect("read issuer ca fixture"); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - - let handle = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: issuer_ca_der, - ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: first_base_uri.clone(), - manifest_rsync_uri: manifest_rsync_uri.clone(), - publication_point_rsync_uri: first_base_uri.clone(), - rrdp_notification_uri: None, - }; - let second_handle = CaInstanceHandle { - rsync_base_uri: second_base_uri.clone(), - publication_point_rsync_uri: second_base_uri.clone(), - ..handle.clone() - }; - - struct ModuleScopeRsyncFetcher { - inner: LocalDirRsyncFetcher, - calls: Arc, - } - impl RsyncFetcher for ModuleScopeRsyncFetcher { - fn fetch_objects( - &self, - rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - self.calls.fetch_add(1, Ordering::SeqCst); - self.inner.fetch_objects(rsync_base_uri) - } - - fn dedup_key(&self, _rsync_base_uri: &str) -> String { - "rsync://rpki.cernet.net/repo/".to_string() - } - } - - let calls = Arc::new(AtomicUsize::new(0)); - let rsync = ModuleScopeRsyncFetcher { - inner: LocalDirRsyncFetcher::new(&fixture_dir), - calls: calls.clone(), - }; - - let runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &rsync, - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - }; - - let first = runner.run_publication_point(&handle).expect("first run ok"); - assert_eq!(first.source, PublicationPointSource::Fresh); - - let second = runner - .run_publication_point(&second_handle) - .expect("second run ok"); - assert!(matches!( - second.source, - PublicationPointSource::Fresh | PublicationPointSource::VcirCurrentInstance - )); - - assert_eq!( - calls.load(Ordering::SeqCst), - 1, - "module-scope dedup should skip second sync" - ); - } - - #[test] - fn runner_rsync_dedup_works_in_rsync_only_mode_even_when_rrdp_notify_exists() { - let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); - assert!(fixture_dir.is_dir(), "fixture directory must exist"); - - let first_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); - let second_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/sub/".to_string(); - let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; - let manifest_rsync_uri = format!("{first_base_uri}{manifest_file}"); - - let fixture_manifest_bytes = - std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); - let fixture_manifest = - crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) - .expect("decode manifest fixture"); - let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy { - sync_preference: crate::policy::SyncPreference::RsyncOnly, - ..Policy::default() - }; - - let issuer_ca_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - ) - .expect("read issuer ca fixture"); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - - let handle = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: issuer_ca_der, - ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: first_base_uri.clone(), - manifest_rsync_uri: manifest_rsync_uri.clone(), - publication_point_rsync_uri: first_base_uri.clone(), - rrdp_notification_uri: Some("https://rrdp.example.test/notification.xml".to_string()), - }; - let second_handle = CaInstanceHandle { - rsync_base_uri: second_base_uri.clone(), - publication_point_rsync_uri: second_base_uri.clone(), - ..handle.clone() - }; - - struct ModuleScopeRsyncFetcher { - inner: LocalDirRsyncFetcher, - calls: Arc, - } - impl RsyncFetcher for ModuleScopeRsyncFetcher { - fn fetch_objects( - &self, - rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - self.calls.fetch_add(1, Ordering::SeqCst); - self.inner.fetch_objects(rsync_base_uri) - } - - fn dedup_key(&self, _rsync_base_uri: &str) -> String { - "rsync://rpki.cernet.net/repo/".to_string() - } - } - - let calls = Arc::new(AtomicUsize::new(0)); - let rsync = ModuleScopeRsyncFetcher { - inner: LocalDirRsyncFetcher::new(&fixture_dir), - calls: calls.clone(), - }; - - let runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &rsync, - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: true, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - }; - - let first = runner.run_publication_point(&handle).expect("first run ok"); - assert_eq!(first.source, PublicationPointSource::Fresh); - - let second = runner - .run_publication_point(&second_handle) - .expect("second run ok"); - assert!(matches!( - second.source, - PublicationPointSource::Fresh | PublicationPointSource::VcirCurrentInstance - )); - - assert_eq!( - calls.load(Ordering::SeqCst), - 1, - "rsync-only mode must deduplicate by rsync scope even when RRDP notification is present" - ); - } - - #[test] - fn runner_when_repo_sync_fails_uses_current_instance_vcir_and_keeps_children_empty_for_fixture() - { - let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); - assert!(fixture_dir.is_dir(), "fixture directory must exist"); - - let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); - let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; - let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); - - let fixture_manifest_bytes = - std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); - let fixture_manifest = - crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) - .expect("decode manifest fixture"); - let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy { - sync_preference: crate::policy::SyncPreference::RsyncOnly, - ..Policy::default() - }; - - let issuer_ca_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - ) - .expect("read issuer ca fixture"); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - - let handle = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: issuer_ca_der, - ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: rsync_base_uri.clone(), - manifest_rsync_uri: manifest_rsync_uri.clone(), - publication_point_rsync_uri: rsync_base_uri.clone(), - rrdp_notification_uri: None, - }; - - // First: successful fresh run to populate the latest VCIR baseline. - let ok_runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - }; - let first = ok_runner - .run_publication_point(&handle) - .expect("first run ok"); - assert_eq!(first.source, PublicationPointSource::Fresh); - assert!( - first.discovered_children.is_empty(), - "fixture has no child .cer" - ); - - // Second: repo sync fails, but we can still reuse current-instance VCIR. - let bad_runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &FailingRsyncFetcher, - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - }; - let second = bad_runner - .run_publication_point(&handle) - .expect("should reuse current-instance VCIR"); - assert_eq!(second.source, PublicationPointSource::VcirCurrentInstance); - assert!(second.discovered_children.is_empty()); - assert!( - second - .warnings - .iter() - .any(|w| w.message.contains("repo sync failed")), - "expected warning about repo sync failure" - ); - } - - #[test] - fn build_publication_point_audit_emits_no_audit_entry_for_duplicate_pack_uri() { - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256("rsync://example.test/repo/dup.roa", vec![1u8]), - PackFile::from_bytes_compute_sha256("rsync://example.test/repo/dup.roa", vec![2u8]), - ]); - let pp = crate::validation::manifest::PublicationPointResult { - source: crate::validation::manifest::PublicationPointSource::VcirCurrentInstance, - snapshot: pack.clone(), - warnings: Vec::new(), - }; - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: vec![1], - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - let objects = crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: Vec::new(), - stats: crate::validation::objects::ObjectsStats::default(), - audit: Vec::new(), - }; - - let audit = build_publication_point_audit_from_snapshot( - &ca, - pp.source, - None, - None, - None, - None, - &pp.snapshot, - &[], - &objects, - &[], - ); - assert_eq!(audit.source, "vcir_current_instance"); - assert_eq!(audit.repo_sync_phase, None); - assert_eq!(audit.repo_terminal_state, "fallback_current_instance"); - assert!( - audit - .objects - .iter() - .any(|e| e.detail.as_deref() == Some("skipped: no audit entry")), - "expected a duplicate key to produce a 'no audit entry' placeholder" - ); - } - - #[test] - fn build_publication_point_audit_marks_invalid_crl_as_error_and_overlays_roa_audit() { - let now = time::OffsetDateTime::now_utc(); - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/bad.crl", - vec![0u8], - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/x.roa", - vec![1u8], - ), - ]); - - let pp = crate::validation::manifest::PublicationPointResult { - source: crate::validation::manifest::PublicationPointSource::Fresh, - snapshot: pack.clone(), - warnings: Vec::new(), - }; - - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: vec![1], - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - - let objects = crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: Vec::new(), - stats: crate::validation::objects::ObjectsStats::default(), - audit: vec![ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/issuer/x.roa".to_string(), - sha256_hex: sha256_hex_from_32(&pack.files[1].sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Ok, - detail: None, - }], - }; - - let audit = build_publication_point_audit_from_snapshot( - &issuer, - pp.source, - Some("rsync"), - Some("rsync_only_ok"), - Some(123), - Some("none"), - &pp.snapshot, - &[], - &objects, - &[], - ); - assert_eq!(audit.objects[0].kind, AuditObjectKind::Manifest); - assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync")); - assert_eq!(audit.repo_sync_phase.as_deref(), Some("rsync_only_ok")); - assert_eq!(audit.repo_sync_duration_ms, Some(123)); - assert_eq!(audit.repo_sync_error.as_deref(), Some("none")); - assert_eq!(audit.repo_terminal_state, "fresh"); - - let crl = audit - .objects - .iter() - .find(|e| e.rsync_uri.ends_with("bad.crl")) - .expect("crl entry"); - assert!(matches!(crl.result, AuditObjectResult::Error)); - - let roa = audit - .objects - .iter() - .find(|e| e.rsync_uri.ends_with("x.roa")) - .expect("roa entry"); - assert!(matches!(roa.result, AuditObjectResult::Ok)); - - // Smoke that time fields are populated from pack. - assert!(audit.verified_at_rfc3339_utc.contains('T')); - assert!(audit.this_update_rfc3339_utc.contains('T')); - assert!(audit.next_update_rfc3339_utc.contains('T')); - let _ = now; - } - - #[test] - fn discover_children_with_router_certificate_records_ok_audit_and_no_child() { - let g = generate_router_cert_with_variant("ec-p256", true); - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der.clone(), - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/router.cer", - g.router_der.clone(), - ), - ]); - - let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: g.issuer_ca_der.clone(), - ca_certificate_rsync_uri: Some( - "rsync://example.test/repo/issuer/issuer.cer".to_string(), - ), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let out = discover_children_from_fresh_snapshot_with_audit( - &issuer, - &pack, - time::OffsetDateTime::now_utc(), - None, - ) - .expect("discover router cert"); - assert!(out.children.is_empty()); - assert_eq!(out.audits.len(), 1); - assert!(matches!(out.audits[0].result, AuditObjectResult::Ok)); - assert!( - out.audits[0] - .detail - .as_deref() - .unwrap_or("") - .contains("validated BGPsec router certificate") - ); - } - - #[test] - fn discover_children_with_non_router_ee_certificate_records_skipped_audit() { - let g = generate_router_cert_with_variant("ec-p256", false); - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der.clone(), - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/router-no-eku.cer", - g.router_der.clone(), - ), - ]); - - let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: g.issuer_ca_der.clone(), - ca_certificate_rsync_uri: Some( - "rsync://example.test/repo/issuer/issuer.cer".to_string(), - ), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let out = discover_children_from_fresh_snapshot_with_audit( - &issuer, - &pack, - time::OffsetDateTime::now_utc(), - None, - ) - .expect("discover non-router cert"); - assert!(out.children.is_empty()); - assert_eq!(out.audits.len(), 1); - assert!(matches!(out.audits[0].result, AuditObjectResult::Skipped)); - assert!( - out.audits[0] - .detail - .as_deref() - .unwrap_or("") - .contains("not a CA resource certificate or BGPsec router certificate") - ); - } - - #[test] - fn discover_children_with_invalid_router_certificate_records_error_audit() { - let g = generate_router_cert_with_variant("ec-p384", true); - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der.clone(), - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/router-invalid.cer", - g.router_der.clone(), - ), - ]); - - let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: g.issuer_ca_der.clone(), - ca_certificate_rsync_uri: Some( - "rsync://example.test/repo/issuer/issuer.cer".to_string(), - ), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let out = discover_children_from_fresh_snapshot_with_audit( - &issuer, - &pack, - time::OffsetDateTime::now_utc(), - None, - ) - .expect("discover invalid router cert"); - assert!(out.children.is_empty()); - assert_eq!(out.audits.len(), 1); - assert!(matches!(out.audits[0].result, AuditObjectResult::Error)); - assert!( - out.audits[0] - .detail - .as_deref() - .unwrap_or("") - .contains("router certificate validation failed") - ); - } - - #[test] - fn discover_children_with_audit_records_decode_error_for_corrupt_cer() { - let g = generate_chain_and_crl(); - - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der.clone(), - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/corrupt.cer", - vec![0u8], - ), - ]); - - let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: g.issuer_ca_der.clone(), - ca_certificate_rsync_uri: None, - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let now = time::OffsetDateTime::now_utc(); - let out = discover_children_from_fresh_snapshot_with_audit(&issuer, &pack, now, None) - .expect("discover children"); - assert!(out.children.is_empty()); - assert_eq!(out.audits.len(), 1); - assert!(matches!(out.audits[0].result, AuditObjectResult::Error)); - } - - #[test] - fn select_issuer_crl_uri_for_child_covers_missing_and_not_found_paths() { - let g = generate_chain_and_crl(); - let child = ResourceCertificate::decode_der(&g.child_ca_der).expect("decode child cert"); - - let empty: std::collections::HashMap = - std::collections::HashMap::new(); - let err = select_issuer_crl_uri_for_child(&child, &empty).unwrap_err(); - assert!(err.contains("no CRL available"), "{err}"); - - let ta_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/ta/apnic-ta.cer"), - ) - .expect("read TA fixture"); - let ta = ResourceCertificate::decode_der(&ta_der).expect("decode TA fixture"); - let mut cache = std::collections::HashMap::new(); - cache.insert( - "rsync://example.test/repo/issuer/issuer.crl".to_string(), - CachedIssuerCrl::Pending(g.issuer_crl_der.clone()), - ); - let err = select_issuer_crl_uri_for_child(&ta, &cache).unwrap_err(); - assert!(err.contains("CRLDistributionPoints missing"), "{err}"); - - let mut wrong = std::collections::HashMap::new(); - wrong.insert( - "rsync://example.test/repo/issuer/other.crl".to_string(), - CachedIssuerCrl::Pending(g.issuer_crl_der), - ); - let err = select_issuer_crl_uri_for_child(&child, &wrong).unwrap_err(); - assert!( - err.contains("not found in publication point snapshot"), - "{err}" - ); - } - - #[test] - fn ensure_issuer_crl_verified_promotes_pending_cache_entry() { - let g = generate_chain_and_crl(); - let mut cache = std::collections::HashMap::new(); - let crl_uri = "rsync://example.test/repo/issuer/issuer.crl".to_string(); - cache.insert( - crl_uri.clone(), - CachedIssuerCrl::Pending(g.issuer_crl_der.clone()), - ); - - let first = ensure_issuer_crl_verified(&crl_uri, &mut cache, &g.issuer_ca_der) - .expect("verify pending CRL"); - assert!(first.revoked_serials.is_empty()); - assert!(matches!(cache.get(&crl_uri), Some(CachedIssuerCrl::Ok(_)))); - - let second = ensure_issuer_crl_verified(&crl_uri, &mut cache, &g.issuer_ca_der) - .expect("reuse verified CRL"); - assert!(second.revoked_serials.is_empty()); - } - - #[test] - fn discover_children_with_invalid_issuer_der_records_error_audit() { - let g = generate_chain_and_crl(); - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der.clone(), - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/child.cer", - g.child_ca_der.clone(), - ), - ]); - - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: vec![0u8], - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let out = discover_children_from_fresh_snapshot_with_audit( - &issuer, - &pack, - time::OffsetDateTime::now_utc(), - None, - ) - .expect("discover children with invalid issuer der"); - assert!(out.children.is_empty()); - assert_eq!(out.audits.len(), 1); - assert!(matches!(out.audits[0].result, AuditObjectResult::Error)); - assert!( - out.audits[0] - .detail - .as_deref() - .unwrap_or("") - .contains("issuer CA decode failed") - ); - } - - #[test] - fn project_current_instance_vcir_reuses_local_outputs_and_restores_children() { - let now = time::OffsetDateTime::now_utc(); - let g = generate_chain_and_crl(); - let child_cert_hash = sha256_hex(&g.child_ca_der); - let vcir = sample_vcir_for_projection(now, &child_cert_hash); - - let store_dir = tempfile::tempdir().expect("store dir"); - let main_db = store_dir.path().join("work-db"); - let repo_bytes_db = store_dir.path().join("repo-bytes.db"); - let store = RocksStore::open_with_external_repo_bytes(&main_db, &repo_bytes_db) - .expect("open rocksdb with external repo bytes"); - store.put_vcir(&vcir).expect("put vcir"); - store - .put_blob_bytes_batch(&[(child_cert_hash.clone(), g.child_ca_der.clone())]) - .expect("put child cert repo bytes"); - assert!( - store - .get_raw_by_hash_entry(&child_cert_hash) - .expect("lookup child raw_by_hash") - .is_none(), - "child cert restoration should not require raw_by_hash entries" - ); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: Vec::new(), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), - }; - - let projection = project_current_instance_vcir_on_failed_fetch( - &store, - &ca, - &ManifestFreshError::RepoSyncFailed { - detail: "synthetic".to_string(), - }, - now, - ) - .expect("project vcir"); - - assert_eq!( - projection.source, - PublicationPointSource::VcirCurrentInstance - ); - assert_eq!(projection.objects.vrps.len(), 1); - assert_eq!(projection.objects.aspas.len(), 1); - assert_eq!(projection.objects.router_keys.len(), 1); - assert_eq!(projection.discovered_children.len(), 1); - assert_eq!( - projection.discovered_children[0].handle.manifest_rsync_uri, - "rsync://example.test/repo/child/child.mft" - ); - assert_eq!( - projection.ccr_manifest_projection.as_ref(), - Some(&vcir.ccr_manifest_projection) - ); - assert!( - projection.snapshot.is_none(), - "current-instance reuse should not reconstruct a byte-backed snapshot" - ); - assert!( - !projection - .warnings - .iter() - .any(|warning| warning.message.contains("manifest failed fetch")), - "successful current-instance reuse should not duplicate the fresh fetch error" - ); - assert!( - !projection - .warnings - .iter() - .any(|warning| warning.message.contains("using latest validated result")), - "successful current-instance reuse should be tracked by source, not warning" - ); - assert!( - !projection - .warnings - .iter() - .any(|warning| warning.message.contains("manifest raw bytes missing")), - "successful current-instance reuse should not load repo bytes for audit reconstruction" - ); - assert!( - !projection - .warnings - .iter() - .any(|warning| warning.message.contains("child certificate bytes missing")), - "child discovery restoration should read child certs from repo bytes" - ); - } - - #[test] - fn project_current_instance_vcir_returns_no_output_when_instance_gate_expired() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); - let this_update = PackTime::from_utc_offset_datetime(now - time::Duration::minutes(2)); - let expired = PackTime::from_utc_offset_datetime(now - time::Duration::minutes(1)); - vcir.validated_manifest_meta.validated_manifest_this_update = this_update; - vcir.validated_manifest_meta.validated_manifest_next_update = expired.clone(); - vcir.instance_gate.manifest_next_update = expired.clone(); - vcir.instance_gate.current_crl_next_update = expired.clone(); - vcir.instance_gate.instance_effective_until = expired; - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - store.put_vcir(&vcir).expect("put vcir"); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: Vec::new(), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let projection = project_current_instance_vcir_on_failed_fetch( - &store, - &ca, - &ManifestFreshError::RepoSyncFailed { - detail: "synthetic".to_string(), - }, - now, - ) - .expect("project vcir"); - - assert_eq!( - projection.source, - PublicationPointSource::FailedFetchNoCache - ); - assert!(projection.ccr_manifest_projection.is_none()); - assert!(projection.objects.vrps.is_empty()); - assert!(projection.objects.aspas.is_empty()); - assert!(projection.discovered_children.is_empty()); - } - - #[test] - fn project_current_instance_vcir_keeps_real_fresh_validation_warning() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let vcir = sample_vcir_for_projection(now, &child_cert_hash); - - let store_dir = tempfile::tempdir().expect("store dir"); - let main_db = store_dir.path().join("work-db"); - let repo_bytes_db = store_dir.path().join("repo-bytes.db"); - let store = RocksStore::open_with_external_repo_bytes(&main_db, &repo_bytes_db) - .expect("open rocksdb with external repo bytes"); - store.put_vcir(&vcir).expect("put vcir"); - store - .put_blob_bytes_batch(&[(child_cert_hash, b"child-cert".to_vec())]) - .expect("put child cert repo bytes"); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: Vec::new(), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let projection = project_current_instance_vcir_on_failed_fetch( - &store, - &ca, - &ManifestFreshError::HashMismatch { - rsync_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), - }, - now, - ) - .expect("project vcir"); - - assert_eq!( - projection.source, - PublicationPointSource::VcirCurrentInstance - ); - assert!( - projection - .warnings - .iter() - .any(|warning| { warning.message.contains("manifest file hash mismatch") }) - ); - assert!( - !projection - .warnings - .iter() - .any(|warning| warning.message.contains("using latest validated result")), - "successful current-instance reuse should not emit bookkeeping warnings" - ); - } - - #[test] - fn project_current_instance_vcir_returns_no_output_when_latest_result_missing() { - let now = time::OffsetDateTime::now_utc(); - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: Vec::new(), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let projection = project_current_instance_vcir_on_failed_fetch( - &store, - &ca, - &ManifestFreshError::RepoSyncFailed { - detail: "synthetic".to_string(), - }, - now, - ) - .expect("project without cached vcir"); - - assert_eq!( - projection.source, - PublicationPointSource::FailedFetchNoCache - ); - assert!(projection.vcir.is_none()); - assert!(projection.ccr_manifest_projection.is_none()); - assert!(projection.snapshot.is_none()); - assert!(projection.objects.audit.is_empty()); - assert!(projection.discovered_children.is_empty()); - assert!(projection.warnings.iter().any(|warning| { - warning - .message - .contains("no latest validated result for current CA instance") - })); - } - - #[test] - fn project_current_instance_vcir_returns_no_output_when_latest_result_is_ineligible() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); - vcir.audit_summary.failed_fetch_eligible = false; - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - store.put_vcir(&vcir).expect("put vcir"); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: Vec::new(), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let projection = project_current_instance_vcir_on_failed_fetch( - &store, - &ca, - &ManifestFreshError::RepoSyncFailed { - detail: "synthetic".to_string(), - }, - now, - ) - .expect("project ineligible vcir"); - - assert_eq!( - projection.source, - PublicationPointSource::FailedFetchNoCache - ); - assert!(projection.vcir.is_some()); - assert!(projection.ccr_manifest_projection.is_none()); - assert!(projection.snapshot.is_none()); - assert!(projection.discovered_children.is_empty()); - assert!(projection.warnings.iter().any(|warning| { - warning - .message - .contains("latest VCIR is not marked failed-fetch eligible") - })); - } - - #[test] - fn project_current_instance_vcir_rejects_mismatched_ccr_projection_uri() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); - vcir.ccr_manifest_projection.manifest_rsync_uri = - "rsync://example.test/repo/issuer/other.mft".to_string(); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - store.put_vcir(&vcir).expect("put vcir"); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: Vec::new(), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let err = project_current_instance_vcir_on_failed_fetch( - &store, - &ca, - &ManifestFreshError::RepoSyncFailed { - detail: "synthetic".to_string(), - }, - now, - ) - .unwrap_err(); - - assert!( - err.contains("vcir CCR manifest projection URI mismatch"), - "{err}" - ); - } - - #[test] - fn fresh_and_reuse_paths_produce_equivalent_ccr_manifest_projection() { - let (pack, issuer_ca_der, validation_time) = - cernet_publication_point_snapshot_for_vcir_tests(); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: issuer_ca_der, - ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - let child_discovery = - discover_children_from_fresh_snapshot_with_audit(&ca, &pack, validation_time, None) - .expect("discover children"); - let fresh_vcir = build_vcir_from_fresh_result( - &ca, - &pack, - &empty_objects_output(), - &[], - &child_discovery.audits, - &child_discovery.children, - validation_time, - ) - .expect("build fresh vcir"); - - let reuse_projection = reuse_ccr_manifest_projection_from_vcir(&ca, &fresh_vcir) - .expect("reuse projection from vcir"); - - assert_eq!(fresh_vcir.ccr_manifest_projection, reuse_projection); - } - - #[test] - fn append_ccr_manifest_projection_from_reuse_requires_projection_for_current_instance() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let runner = sample_runner_with_ccr_accumulator(&store, &policy); - - let err = runner - .append_ccr_manifest_projection_from_reuse(&VcirReuseProjection { - source: PublicationPointSource::VcirCurrentInstance, - vcir: None, - ccr_manifest_projection: None, - snapshot: None, - objects: empty_objects_output(), - child_audits: Vec::new(), - discovered_children: Vec::new(), - warnings: Vec::new(), - }) - .unwrap_err(); - - assert!(err.contains("missing CCR manifest projection"), "{err}"); - assert_eq!( - runner - .ccr_accumulator_snapshot() - .expect("ccr accumulator snapshot") - .manifest_count(), - 0 - ); - } - - #[test] - fn append_ccr_manifest_projection_from_reuse_skips_failed_fetch_no_cache() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let runner = sample_runner_with_ccr_accumulator(&store, &policy); - - runner - .append_ccr_manifest_projection_from_reuse(&VcirReuseProjection { - source: PublicationPointSource::FailedFetchNoCache, - vcir: None, - ccr_manifest_projection: None, - snapshot: None, - objects: empty_objects_output(), - child_audits: Vec::new(), - discovered_children: Vec::new(), - warnings: Vec::new(), - }) - .expect("failed-fetch no-cache should not append"); - - assert_eq!( - runner - .ccr_accumulator_snapshot() - .expect("ccr accumulator snapshot") - .manifest_count(), - 0 - ); - } - - #[test] - fn parse_snapshot_time_value_reports_invalid_timestamp() { - let err = parse_snapshot_time_value(&PackTime { - rfc3339_utc: "not-a-time".to_string(), - }) - .unwrap_err(); - - assert!(err.contains("invalid RFC3339 time 'not-a-time'"), "{err}"); - } - - #[test] - fn build_objects_output_from_vcir_tracks_expired_and_invalid_cached_outputs() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); - - let bad_time_uri = "rsync://example.test/repo/issuer/bad-time.roa".to_string(); - let expired_uri = "rsync://example.test/repo/issuer/expired.asa".to_string(); - let bad_json_uri = "rsync://example.test/repo/issuer/bad-json.roa".to_string(); - let bad_prefix_uri = "rsync://example.test/repo/issuer/bad-prefix.roa".to_string(); - let bad_aspa_uri = "rsync://example.test/repo/issuer/bad-aspa.asa".to_string(); - - for (uri, kind) in [ - (bad_time_uri.clone(), VcirArtifactKind::Roa), - (expired_uri.clone(), VcirArtifactKind::Aspa), - (bad_json_uri.clone(), VcirArtifactKind::Roa), - (bad_prefix_uri.clone(), VcirArtifactKind::Roa), - (bad_aspa_uri.clone(), VcirArtifactKind::Aspa), - ] { - vcir.related_artifacts.push(VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: kind, - uri: Some(uri.clone()), - sha256: sha256_hex(uri.as_bytes()), - object_type: Some( - match kind { - VcirArtifactKind::Aspa => "aspa", - _ => "roa", - } - .to_string(), - ), - validation_status: VcirArtifactValidationStatus::Accepted, - }); - } - - vcir.local_outputs.push(VcirLocalOutput { - output_id: sha256_hex(b"bad-time"), - output_type: VcirOutputType::Vrp, - item_effective_until: PackTime { - rfc3339_utc: "bad-time-value".to_string(), - }, - source_object_uri: bad_time_uri.clone(), - source_object_type: "roa".to_string(), - source_object_hash: sha256_hex(b"bad-time-src"), - source_ee_cert_hash: sha256_hex(b"bad-time-ee"), - payload_json: - serde_json::json!({"asn": 64496, "prefix": "203.0.113.0/24", "max_length": 24}) - .to_string(), - rule_hash: sha256_hex(b"bad-time-rule"), - validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), bad_time_uri.clone()], - }); - vcir.local_outputs.push(VcirLocalOutput { - output_id: sha256_hex(b"expired"), - output_type: VcirOutputType::Aspa, - item_effective_until: PackTime::from_utc_offset_datetime( - now - time::Duration::minutes(1), - ), - source_object_uri: expired_uri.clone(), - source_object_type: "aspa".to_string(), - source_object_hash: sha256_hex(b"expired-src"), - source_ee_cert_hash: sha256_hex(b"expired-ee"), - payload_json: serde_json::json!({"customer_as_id": 64500, "provider_as_ids": [64501]}) - .to_string(), - rule_hash: sha256_hex(b"expired-rule"), - validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), expired_uri.clone()], - }); - vcir.local_outputs.push(VcirLocalOutput { - output_id: sha256_hex(b"bad-json"), - output_type: VcirOutputType::Vrp, - item_effective_until: PackTime::from_utc_offset_datetime( - now + time::Duration::minutes(5), - ), - source_object_uri: bad_json_uri.clone(), - source_object_type: "roa".to_string(), - source_object_hash: sha256_hex(b"bad-json-src"), - source_ee_cert_hash: sha256_hex(b"bad-json-ee"), - payload_json: "{not-json".to_string(), - rule_hash: sha256_hex(b"bad-json-rule"), - validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), bad_json_uri.clone()], - }); - vcir.local_outputs.push(VcirLocalOutput { - output_id: sha256_hex(b"bad-prefix"), - output_type: VcirOutputType::Vrp, - item_effective_until: PackTime::from_utc_offset_datetime( - now + time::Duration::minutes(5), - ), - source_object_uri: bad_prefix_uri.clone(), - source_object_type: "roa".to_string(), - source_object_hash: sha256_hex(b"bad-prefix-src"), - source_ee_cert_hash: sha256_hex(b"bad-prefix-ee"), - payload_json: - serde_json::json!({"asn": 64510, "prefix": "203.0.113.0", "max_length": 24}) - .to_string(), - rule_hash: sha256_hex(b"bad-prefix-rule"), - validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), bad_prefix_uri.clone()], - }); - vcir.local_outputs.push(VcirLocalOutput { - output_id: sha256_hex(b"bad-aspa"), - output_type: VcirOutputType::Aspa, - item_effective_until: PackTime::from_utc_offset_datetime( - now + time::Duration::minutes(5), - ), - source_object_uri: bad_aspa_uri.clone(), - source_object_type: "aspa".to_string(), - source_object_hash: sha256_hex(b"bad-aspa-src"), - source_ee_cert_hash: sha256_hex(b"bad-aspa-ee"), - payload_json: serde_json::json!({"customer_as_id": 64520}).to_string(), - rule_hash: sha256_hex(b"bad-aspa-rule"), - validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), bad_aspa_uri.clone()], - }); - - let mut warnings = Vec::new(); - let output = build_objects_output_from_vcir(&vcir, now, &mut warnings); - - assert_eq!(output.vrps.len(), 1); - assert_eq!(output.aspas.len(), 1); - assert_eq!(output.stats.roa_total, 4); - assert_eq!(output.stats.roa_ok, 1); - assert_eq!(output.stats.aspa_total, 3); - assert_eq!(output.stats.aspa_ok, 1); - assert!(warnings.iter().any(|warning| { - warning - .message - .contains("cached local output has invalid item_effective_until") - })); - assert!(warnings.iter().any(|warning| { - warning - .message - .contains("cached ROA local output parse failed") - })); - assert!(warnings.iter().any(|warning| { - warning - .message - .contains("cached ASPA local output parse failed") - })); - assert!(output.audit.iter().any(|entry| { - entry.rsync_uri == expired_uri - && matches!(entry.result, AuditObjectResult::Skipped) - && entry.detail.as_deref() == Some("skipped: cached local output expired") - })); - assert!(output.audit.iter().any(|entry| { - entry.rsync_uri == bad_time_uri && matches!(entry.result, AuditObjectResult::Error) - })); - assert!(output.audit.iter().any(|entry| { - entry.rsync_uri == bad_prefix_uri - && matches!(entry.result, AuditObjectResult::Error) - && entry - .detail - .as_deref() - .unwrap_or("") - .contains("cached ROA local output parse failed") - })); - } - - #[test] - fn build_publication_point_audit_from_vcir_uses_vcir_metadata_and_overlays_child_and_object_audits() - { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); - vcir.related_artifacts - .retain(|artifact| artifact.artifact_role != VcirArtifactRole::Manifest); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: Vec::new(), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), - }; - let runner_warnings = vec![Warning::new("runner warning")]; - let objects = crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: vec![Warning::new("objects warning")], - stats: crate::validation::objects::ObjectsStats::default(), - audit: vec![ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), - sha256_hex: sha256_hex(b"override-roa"), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Error, - detail: Some("overridden from object audit".to_string()), - }], - }; - let child_audits = vec![ObjectAuditEntry { - rsync_uri: vcir.child_entries[0].child_cert_rsync_uri.clone(), - sha256_hex: vcir.child_entries[0].child_cert_hash.clone(), - kind: AuditObjectKind::Certificate, - result: AuditObjectResult::Ok, - detail: Some("restored child CA instance from VCIR".to_string()), - }]; - - let audit = build_publication_point_audit_from_vcir( - &ca, - PublicationPointSource::VcirCurrentInstance, - Some("rsync"), - Some("rrdp_failed_rsync_failed"), - Some(456), - Some("rsync failed"), - Some(&vcir), - None, - &runner_warnings, - &objects, - &child_audits, - ); - - assert_eq!(audit.source, "vcir_current_instance"); - assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync")); - assert_eq!( - audit.repo_sync_phase.as_deref(), - Some("rrdp_failed_rsync_failed") - ); - assert_eq!(audit.repo_sync_duration_ms, Some(456)); - assert_eq!(audit.repo_sync_error.as_deref(), Some("rsync failed")); - assert_eq!(audit.repo_terminal_state, "fallback_current_instance"); - assert_eq!(audit.objects[0].rsync_uri, vcir.current_manifest_rsync_uri); - assert_eq!(audit.objects[0].kind, AuditObjectKind::Manifest); - assert_eq!( - audit.this_update_rfc3339_utc, - vcir.validated_manifest_meta - .validated_manifest_this_update - .rfc3339_utc - ); - assert_eq!( - audit.next_update_rfc3339_utc, - vcir.validated_manifest_meta - .validated_manifest_next_update - .rfc3339_utc - ); - assert_eq!( - audit.verified_at_rfc3339_utc, - vcir.last_successful_validation_time.rfc3339_utc - ); - assert_eq!(audit.warnings.len(), 2); - assert!(audit.objects.iter().any(|entry| { - entry.rsync_uri == "rsync://example.test/repo/issuer/a.roa" - && matches!(entry.result, AuditObjectResult::Error) - && entry.detail.as_deref() == Some("overridden from object audit") - })); - assert!(audit.objects.iter().any(|entry| { - entry.rsync_uri == vcir.child_entries[0].child_cert_rsync_uri - && matches!(entry.result, AuditObjectResult::Ok) - })); - } - - #[test] - fn build_publication_point_audit_from_vcir_failed_no_cache_keeps_current_reject_only() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let vcir = sample_vcir_for_projection(now, &child_cert_hash); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: Vec::new(), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), - }; - let objects = crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: Vec::new(), - stats: crate::validation::objects::ObjectsStats::default(), - audit: vec![ObjectAuditEntry { - rsync_uri: vcir.current_manifest_rsync_uri.clone(), - sha256_hex: sha256_hex(b"current-manifest"), - kind: AuditObjectKind::Manifest, - result: AuditObjectResult::Error, - detail: Some("manifest is not valid at validation_time".to_string()), - }], - }; - - let audit = build_publication_point_audit_from_vcir( - &ca, - PublicationPointSource::FailedFetchNoCache, - Some("rsync"), - Some("rsync_only_ok"), - Some(123), - None, - Some(&vcir), - None, - &[Warning::new("latest VCIR instance_gate expired")], - &objects, - &[], - ); - - assert_eq!(audit.source, "failed_fetch_no_cache"); - assert_eq!(audit.repo_terminal_state, "failed_no_cache"); - assert_eq!( - audit.this_update_rfc3339_utc, - vcir.validated_manifest_meta - .validated_manifest_this_update - .rfc3339_utc - ); - assert_eq!(audit.objects.len(), 1); - assert_eq!(audit.objects[0].rsync_uri, vcir.current_manifest_rsync_uri); - assert!(matches!(audit.objects[0].result, AuditObjectResult::Error)); - assert!( - !audit - .objects - .iter() - .any(|entry| entry.rsync_uri == "rsync://example.test/repo/issuer/a.roa"), - "failed-no-cache must not expand old VCIR related artifacts into current-run audit", - ); - assert!( - !audit - .objects - .iter() - .any(|entry| entry.rsync_uri == "rsync://example.test/repo/issuer/issuer.crl"), - "failed-no-cache must not expose old CRL as current-run CIR input", - ); - } - - #[test] - fn rejected_manifest_audit_entry_for_failed_fetch_uses_current_repo_hash() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let runner = sample_runner_with_ccr_accumulator(&store, &policy); - let manifest_uri = "rsync://example.test/repo/issuer/issuer.mft"; - let manifest_hash = sha256_hex(b"manifest-bytes"); - store - .put_blob_bytes_batch(&[(manifest_hash.clone(), b"manifest-bytes".to_vec())]) - .expect("put manifest bytes"); - store - .put_repository_view_entry(&crate::storage::RepositoryViewEntry { - rsync_uri: manifest_uri.to_string(), - current_hash: Some(manifest_hash.clone()), - repository_source: Some("rsync://example.test/repo/issuer/".to_string()), - object_type: Some("mft".to_string()), - state: crate::storage::RepositoryViewState::Present, - }) - .expect("put repository view"); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: Vec::new(), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: manifest_uri.to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let entry = runner - .rejected_manifest_audit_entry_for_failed_fetch( - &ca, - &ManifestFreshError::StaleOrEarly { - this_update_rfc3339_utc: "2026-05-27T08:37:07Z".to_string(), - next_update_rfc3339_utc: "2026-05-28T10:01:07Z".to_string(), - validation_time_rfc3339_utc: "2026-05-28T10:11:00Z".to_string(), - }, - ) - .expect("rejected manifest audit entry"); - - assert_eq!(entry.rsync_uri, manifest_uri); - assert_eq!(entry.sha256_hex, manifest_hash); - assert_eq!(entry.kind, AuditObjectKind::Manifest); - assert_eq!(entry.result, AuditObjectResult::Error); - assert!( - entry - .detail - .as_deref() - .unwrap_or("") - .contains("manifest is not valid at validation_time") - ); - } - - #[test] - fn build_publication_point_audit_from_vcir_without_cached_inputs_returns_empty_listing() { - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: Vec::new(), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let audit = build_publication_point_audit_from_vcir( - &ca, - PublicationPointSource::FailedFetchNoCache, - Some("rsync"), - Some("rsync_only_failed"), - Some(789), - Some("load from network failed, fallback to cache"), - None, - None, - &[Warning::new("runner warning")], - &crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: vec![Warning::new("object warning")], - stats: crate::validation::objects::ObjectsStats::default(), - audit: Vec::new(), - }, - &[], - ); - - assert_eq!(audit.source, "failed_fetch_no_cache"); - assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync")); - assert_eq!(audit.repo_sync_phase.as_deref(), Some("rsync_only_failed")); - assert_eq!(audit.repo_sync_duration_ms, Some(789)); - assert_eq!( - audit.repo_sync_error.as_deref(), - Some("load from network failed, fallback to cache") - ); - assert_eq!(audit.repo_terminal_state, "failed_no_cache"); - assert!(audit.this_update_rfc3339_utc.is_empty()); - assert!(audit.next_update_rfc3339_utc.is_empty()); - assert!(audit.verified_at_rfc3339_utc.is_empty()); - assert_eq!(audit.warnings.len(), 2); - assert!(audit.objects.is_empty()); - } - - #[test] - fn effective_repo_sync_duration_uses_runtime_duration_for_failures() { - assert_eq!(effective_repo_sync_duration_ms(0, Some(12), false), 12); - assert_eq!(effective_repo_sync_duration_ms(5, Some(12), false), 12); - assert_eq!(effective_repo_sync_duration_ms(20, Some(12), false), 20); - assert_eq!(effective_repo_sync_duration_ms(5, None, false), 5); - assert_eq!(effective_repo_sync_duration_ms(5, Some(12), true), 5); - } - - #[test] - fn reconstruct_snapshot_from_vcir_reports_missing_manifest_and_related_raw_bytes() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); - let dup_uri = "rsync://example.test/repo/issuer/dup.roa".to_string(); - vcir.related_artifacts.push(VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: VcirArtifactKind::Roa, - uri: Some(dup_uri.clone()), - sha256: sha256_hex(b"dup-roa-1"), - object_type: Some("roa".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }); - vcir.related_artifacts.push(VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: VcirArtifactKind::Roa, - uri: Some(dup_uri.clone()), - sha256: sha256_hex(b"dup-roa-2"), - object_type: Some("roa".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }); - vcir.related_artifacts.push(VcirRelatedArtifact { - artifact_role: VcirArtifactRole::IssuerCert, - artifact_kind: VcirArtifactKind::Cer, - uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), - sha256: sha256_hex(b"issuer-cert"), - object_type: Some("cer".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: Vec::new(), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let mut warnings = Vec::new(); - assert!(reconstruct_snapshot_from_vcir(&store, &ca, &vcir, &mut warnings).is_none()); - assert!(warnings.iter().any(|warning| { - warning - .message - .contains("manifest raw bytes missing for VCIR audit reconstruction") - })); - - let manifest_bytes = b"manifest-bytes".to_vec(); - let current_crl_bytes = b"current-crl-bytes".to_vec(); - let child_bytes = b"child-cert".to_vec(); - let roa_bytes = b"roa-bytes".to_vec(); - for (bytes, uri, object_type) in [ - ( - manifest_bytes.clone(), - Some(vcir.manifest_rsync_uri.clone()), - Some("mft".to_string()), - ), - ( - current_crl_bytes, - Some(vcir.current_crl_rsync_uri.clone()), - Some("crl".to_string()), - ), - ( - child_bytes, - Some(vcir.child_entries[0].child_cert_rsync_uri.clone()), - Some("cer".to_string()), - ), - ( - roa_bytes, - Some("rsync://example.test/repo/issuer/a.roa".to_string()), - Some("roa".to_string()), - ), - ] { - let mut entry = RawByHashEntry::from_bytes(sha256_hex(&bytes), bytes); - if let Some(uri) = uri { - entry.origin_uris.push(uri); - } - entry.object_type = object_type; - entry.encoding = Some("der".to_string()); - store.put_raw_by_hash_entry(&entry).expect("put raw entry"); - } - - warnings.clear(); - let pack = reconstruct_snapshot_from_vcir(&store, &ca, &vcir, &mut warnings) - .expect("reconstruct pack with partial related artifacts"); - assert_eq!(pack.manifest_bytes, manifest_bytes); - assert_eq!(pack.files.len(), 3, "crl + child cert + roa only"); - assert!( - pack.files - .iter() - .any(|file| file.rsync_uri.ends_with("issuer.crl")) - ); - assert!( - pack.files - .iter() - .any(|file| file.rsync_uri.ends_with("child.cer")) - ); - assert!( - pack.files - .iter() - .any(|file| file.rsync_uri.ends_with("a.roa")) - ); - assert!( - !pack - .files - .iter() - .any(|file| file.rsync_uri.ends_with("issuer.cer")) - ); - assert!(warnings.iter().any(|warning| { - warning - .message - .contains("related artifact raw bytes missing for VCIR audit reconstruction") - })); - } - - #[test] - fn reconstruct_snapshot_from_vcir_reads_repo_bytes_without_raw_entries() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let vcir = sample_vcir_for_projection(now, &child_cert_hash); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: Vec::new(), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let store_dir = tempfile::tempdir().expect("store dir"); - let main_db = store_dir.path().join("work-db"); - let repo_bytes_db = store_dir.path().join("repo-bytes.db"); - let store = RocksStore::open_with_external_repo_bytes(&main_db, &repo_bytes_db) - .expect("open rocksdb with external repo bytes"); - let repo_blobs = [ - b"manifest-bytes".to_vec(), - b"current-crl-bytes".to_vec(), - b"child-cert".to_vec(), - b"roa-bytes".to_vec(), - b"aspa-bytes".to_vec(), - ] - .into_iter() - .map(|bytes| (sha256_hex(&bytes), bytes)) - .collect::>(); - store - .put_blob_bytes_batch(&repo_blobs) - .expect("put external repo bytes"); - - let manifest_hash = vcir - .related_artifacts - .iter() - .find(|artifact| artifact.artifact_role == VcirArtifactRole::Manifest) - .expect("manifest artifact") - .sha256 - .clone(); - assert!( - store - .get_raw_by_hash_entry(&manifest_hash) - .expect("raw manifest lookup") - .is_none(), - "repo object bytes must not require raw_by_hash entries" - ); - - let mut warnings = Vec::new(); - let pack = reconstruct_snapshot_from_vcir(&store, &ca, &vcir, &mut warnings) - .expect("reconstruct pack from external repo bytes"); - assert_eq!(pack.manifest_bytes, b"manifest-bytes".to_vec()); - assert_eq!(pack.files.len(), 4, "crl + child cert + roa + aspa"); - assert!( - warnings.iter().all(|warning| { - !warning - .message - .contains("raw bytes missing for VCIR audit reconstruction") - }), - "external repo bytes should satisfy VCIR audit reconstruction without raw warnings" - ); - } - - #[test] - fn runner_dedup_paths_execute_with_timing_enabled() { - let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); - let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); - let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; - let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); - let fixture_manifest_bytes = - std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); - let fixture_manifest = - crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) - .expect("decode manifest fixture"); - let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let issuer_ca_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - ) - .expect("read issuer ca fixture"); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - let handle = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate_der: issuer_ca_der, - ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: rsync_base_uri.clone(), - manifest_rsync_uri: manifest_rsync_uri.clone(), - publication_point_rsync_uri: rsync_base_uri.clone(), - rrdp_notification_uri: Some("https://example.test/notification.xml".to_string()), - }; - let timing = - crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { - recorded_at_utc_rfc3339: "2026-03-11T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-03-11T00:00:00Z".to_string(), - tal_url: None, - db_path: None, - }); - let policy_rrdp = Policy::default(); - let runner_rrdp = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy_rrdp, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), - validation_time, - timing: Some(timing.clone()), - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: true, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - }; - let first = runner_rrdp - .run_publication_point(&handle) - .expect("rrdp fallback to rsync"); - assert_eq!(first.source, PublicationPointSource::Fresh); - let second = runner_rrdp - .run_publication_point(&handle) - .expect("rrdp dedup skip"); - assert_eq!(second.source, PublicationPointSource::Fresh); - - let policy_rsync = Policy { - sync_preference: crate::policy::SyncPreference::RsyncOnly, - ..Policy::default() - }; - let runner_rsync = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy_rsync, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), - validation_time, - timing: Some(timing), - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - }; - let third = runner_rsync - .run_publication_point(&handle) - .expect("rsync first run"); - assert_eq!(third.source, PublicationPointSource::Fresh); - let fourth = runner_rsync - .run_publication_point(&handle) - .expect("rsync dedup run"); - assert_eq!(fourth.source, PublicationPointSource::Fresh); - assert_eq!( - crate::fetch::rsync::normalize_rsync_base_uri("rsync://example.test/repo"), - "rsync://example.test/repo/" - ); - } -} +#[path = "tree_runner/tests.rs"] +mod tests; diff --git a/src/validation/tree_runner/tests.rs b/src/validation/tree_runner/tests.rs new file mode 100644 index 0000000..a884a71 --- /dev/null +++ b/src/validation/tree_runner/tests.rs @@ -0,0 +1,3621 @@ +use super::*; +use crate::data_model::rc::ResourceCertificate; +use crate::fetch::rsync::LocalDirRsyncFetcher; +use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher}; +use crate::storage::{ + PackFile, PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, + ValidatedManifestMeta, VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, + VcirAuditSummary, VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirOutputType, + VcirRelatedArtifact, VcirSummary, +}; +use crate::sync::rrdp::Fetcher; +use crate::validation::publication_point::PublicationPointSnapshot; +use crate::validation::tree::PublicationPointRunner; + +use std::process::Command; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +struct NeverHttpFetcher; +impl Fetcher for NeverHttpFetcher { + fn fetch(&self, _uri: &str) -> Result, String> { + Err("http fetch disabled in test".to_string()) + } +} + +struct FailingRsyncFetcher; +impl RsyncFetcher for FailingRsyncFetcher { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + Err(RsyncFetchError::Fetch("rsync disabled in test".to_string())) + } +} + +fn sample_runner_with_ccr_accumulator<'a>( + store: &'a RocksStore, + policy: &'a Policy, +) -> Rpkiv1PublicationPointRunner<'a> { + Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time: time::OffsetDateTime::now_utc(), + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: Some(Mutex::new(CcrAccumulator::new(Vec::new()))), + persist_vcir: true, + } +} + +fn openssl_available() -> bool { + Command::new("openssl") + .arg("version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +struct Generated { + issuer_ca_der: Vec, + child_ca_der: Vec, + issuer_crl_der: Vec, +} + +fn run(cmd: &mut Command) { + let out = cmd.output().expect("run command"); + if !out.status.success() { + panic!( + "command failed: {:?}\nstdout={}\nstderr={}", + cmd, + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + } +} + +fn generate_chain_and_crl() -> Generated { + assert!(openssl_available(), "openssl is required for this test"); + + let td = tempfile::tempdir().expect("tempdir"); + let dir = td.path(); + + std::fs::create_dir_all(dir.join("newcerts")).expect("newcerts"); + std::fs::write(dir.join("index.txt"), b"").expect("index"); + std::fs::write(dir.join("serial"), b"1000\n").expect("serial"); + std::fs::write(dir.join("crlnumber"), b"1000\n").expect("crlnumber"); + + let cnf = format!( + r#" +[ ca ] +default_ca = CA_default + +[ CA_default ] +dir = {dir} +database = $dir/index.txt +new_certs_dir = $dir/newcerts +certificate = $dir/issuer.pem +private_key = $dir/issuer.key +serial = $dir/serial +crlnumber = $dir/crlnumber +default_md = sha256 +default_days = 365 +default_crl_days = 1 +policy = policy_any +x509_extensions = v3_issuer_ca +crl_extensions = crl_ext +unique_subject = no +copy_extensions = none + +[ policy_any ] +commonName = supplied + +[ req ] +prompt = no +distinguished_name = dn + +[ dn ] +CN = Test Issuer CA + +[ v3_issuer_ca ] +basicConstraints = critical,CA:true +keyUsage = critical, keyCertSign, cRLSign +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always +certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 +subjectInfoAccess = caRepository;URI:rsync://example.test/repo/issuer/, rpkiManifest;URI:rsync://example.test/repo/issuer/issuer.mft, rpkiNotify;URI:https://example.test/notification.xml +sbgp-ipAddrBlock = critical, IPv4:10.0.0.0/8 +sbgp-autonomousSysNum = critical, AS:64496-64511 + +[ v3_child_ca ] +basicConstraints = critical,CA:true +keyUsage = critical, keyCertSign, cRLSign +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always +crlDistributionPoints = URI:rsync://example.test/repo/issuer/issuer.crl +authorityInfoAccess = caIssuers;URI:rsync://example.test/repo/issuer/issuer.cer +certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 +subjectInfoAccess = caRepository;URI:rsync://example.test/repo/child/, rpkiManifest;URI:rsync://example.test/repo/child/child.mft, rpkiNotify;URI:https://example.test/notification.xml +sbgp-ipAddrBlock = critical, IPv4:10.0.0.0/16 +sbgp-autonomousSysNum = critical, AS:64496 + +[ crl_ext ] +authorityKeyIdentifier = keyid:always +"#, + dir = dir.display() + ); + std::fs::write(dir.join("openssl.cnf"), cnf.as_bytes()).expect("write cnf"); + + run(Command::new("openssl") + .arg("genrsa") + .arg("-out") + .arg(dir.join("issuer.key")) + .arg("2048")); + run(Command::new("openssl") + .arg("req") + .arg("-new") + .arg("-x509") + .arg("-sha256") + .arg("-days") + .arg("365") + .arg("-key") + .arg(dir.join("issuer.key")) + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-extensions") + .arg("v3_issuer_ca") + .arg("-out") + .arg(dir.join("issuer.pem"))); + + run(Command::new("openssl") + .arg("genrsa") + .arg("-out") + .arg(dir.join("child.key")) + .arg("2048")); + run(Command::new("openssl") + .arg("req") + .arg("-new") + .arg("-key") + .arg(dir.join("child.key")) + .arg("-subj") + .arg("/CN=Test Child CA") + .arg("-out") + .arg(dir.join("child.csr"))); + + run(Command::new("openssl") + .arg("ca") + .arg("-batch") + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-in") + .arg(dir.join("child.csr")) + .arg("-extensions") + .arg("v3_child_ca") + .arg("-out") + .arg(dir.join("child.pem"))); + + run(Command::new("openssl") + .arg("x509") + .arg("-in") + .arg(dir.join("issuer.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("issuer.cer"))); + run(Command::new("openssl") + .arg("x509") + .arg("-in") + .arg(dir.join("child.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("child.cer"))); + + run(Command::new("openssl") + .arg("ca") + .arg("-gencrl") + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-out") + .arg(dir.join("issuer.crl.pem"))); + run(Command::new("openssl") + .arg("crl") + .arg("-in") + .arg(dir.join("issuer.crl.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("issuer.crl"))); + + Generated { + issuer_ca_der: std::fs::read(dir.join("issuer.cer")).expect("read issuer der"), + child_ca_der: std::fs::read(dir.join("child.cer")).expect("read child der"), + issuer_crl_der: std::fs::read(dir.join("issuer.crl")).expect("read crl der"), + } +} + +struct GeneratedRouter { + issuer_ca_der: Vec, + router_der: Vec, + issuer_crl_der: Vec, +} + +fn generate_router_cert_with_variant(key_spec: &str, include_eku: bool) -> GeneratedRouter { + assert!(openssl_available(), "openssl is required for this test"); + + let td = tempfile::tempdir().expect("tempdir"); + let dir = td.path(); + + std::fs::create_dir_all(dir.join("newcerts")).expect("newcerts"); + std::fs::write(dir.join("index.txt"), b"").expect("index"); + std::fs::write(dir.join("serial"), b"1000\n").expect("serial"); + std::fs::write(dir.join("crlnumber"), b"1000\n").expect("crlnumber"); + + let eku_line = if include_eku { + "extendedKeyUsage = 1.3.6.1.5.5.7.3.30" + } else { + "" + }; + let cnf = format!( + r#" +[ ca ] +default_ca = CA_default + +[ CA_default ] +dir = {dir} +database = $dir/index.txt +new_certs_dir = $dir/newcerts +certificate = $dir/issuer.pem +private_key = $dir/issuer.key +serial = $dir/serial +crlnumber = $dir/crlnumber +default_md = sha256 +default_days = 365 +default_crl_days = 1 +policy = policy_any +x509_extensions = v3_issuer_ca +crl_extensions = crl_ext +unique_subject = no +copy_extensions = none + +[ policy_any ] +commonName = supplied + +[ req ] +prompt = no +distinguished_name = dn + +[ dn ] +CN = Test Issuer CA + +[ v3_issuer_ca ] +basicConstraints = critical,CA:true +keyUsage = critical, keyCertSign, cRLSign +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always +certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 +subjectInfoAccess = caRepository;URI:rsync://example.test/repo/issuer/, rpkiManifest;URI:rsync://example.test/repo/issuer/issuer.mft, rpkiNotify;URI:https://example.test/notification.xml +sbgp-ipAddrBlock = critical, IPv4:10.0.0.0/8 +sbgp-autonomousSysNum = critical, AS:64496-64511 + +[ v3_router ] +keyUsage = critical, digitalSignature +{eku_line} +authorityKeyIdentifier = keyid:always +crlDistributionPoints = URI:rsync://example.test/repo/issuer/issuer.crl +authorityInfoAccess = caIssuers;URI:rsync://example.test/repo/issuer/issuer.cer +certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 +sbgp-autonomousSysNum = critical, AS:64496 + +[ crl_ext ] +authorityKeyIdentifier = keyid:always +"#, + dir = dir.display(), + eku_line = eku_line, + ); + std::fs::write(dir.join("openssl.cnf"), cnf.as_bytes()).expect("write cnf"); + + run(Command::new("openssl") + .arg("genrsa") + .arg("-out") + .arg(dir.join("issuer.key")) + .arg("2048")); + run(Command::new("openssl") + .arg("req") + .arg("-new") + .arg("-x509") + .arg("-sha256") + .arg("-days") + .arg("365") + .arg("-key") + .arg(dir.join("issuer.key")) + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-extensions") + .arg("v3_issuer_ca") + .arg("-out") + .arg(dir.join("issuer.pem"))); + + match key_spec { + "ec-p256" => run(Command::new("openssl") + .arg("ecparam") + .arg("-name") + .arg("prime256v1") + .arg("-genkey") + .arg("-noout") + .arg("-out") + .arg(dir.join("router.key"))), + "ec-p384" => run(Command::new("openssl") + .arg("ecparam") + .arg("-name") + .arg("secp384r1") + .arg("-genkey") + .arg("-noout") + .arg("-out") + .arg(dir.join("router.key"))), + other => panic!("unsupported key_spec {other}"), + } + + run(Command::new("openssl") + .arg("req") + .arg("-new") + .arg("-key") + .arg(dir.join("router.key")) + .arg("-subj") + .arg("/CN=ROUTER-0000FC10/serialNumber=01020304") + .arg("-out") + .arg(dir.join("router.csr"))); + + run(Command::new("openssl") + .arg("ca") + .arg("-batch") + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-in") + .arg(dir.join("router.csr")) + .arg("-extensions") + .arg("v3_router") + .arg("-out") + .arg(dir.join("router.pem"))); + + run(Command::new("openssl") + .arg("x509") + .arg("-in") + .arg(dir.join("issuer.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("issuer.cer"))); + run(Command::new("openssl") + .arg("x509") + .arg("-in") + .arg(dir.join("router.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("router.cer"))); + + run(Command::new("openssl") + .arg("ca") + .arg("-gencrl") + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-out") + .arg(dir.join("issuer.crl.pem"))); + run(Command::new("openssl") + .arg("crl") + .arg("-in") + .arg(dir.join("issuer.crl.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("issuer.crl"))); + + GeneratedRouter { + issuer_ca_der: std::fs::read(dir.join("issuer.cer")).expect("read issuer der"), + router_der: std::fs::read(dir.join("router.cer")).expect("read router der"), + issuer_crl_der: std::fs::read(dir.join("issuer.crl")).expect("read crl der"), + } +} +fn dummy_pack_with_files(files: Vec) -> PublicationPointSnapshot { + let now = time::OffsetDateTime::now_utc(); + PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_number_be: vec![1], + this_update: PackTime::from_utc_offset_datetime(now), + next_update: PackTime::from_utc_offset_datetime(now + time::Duration::hours(1)), + verified_at: PackTime::from_utc_offset_datetime(now), + manifest_bytes: vec![0x01], + files, + } +} + +fn cernet_publication_point_snapshot_for_vcir_tests() +-> (PublicationPointSnapshot, Vec, time::OffsetDateTime) { + let dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/"; + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + let manifest_bytes = std::fs::read(dir.join(manifest_file)).expect("read manifest fixture"); + let manifest = ManifestObject::decode_der(&manifest_bytes).expect("decode manifest fixture"); + let candidate = manifest.manifest.this_update + time::Duration::seconds(60); + let validation_time = if candidate < manifest.manifest.next_update { + candidate + } else { + manifest.manifest.this_update + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + sync_publication_point( + &store, + &policy, + None, + rsync_base_uri, + &NeverHttpFetcher, + &LocalDirRsyncFetcher::new(&dir), + None, + None, + ) + .expect("sync cernet fixture"); + + let pp = crate::validation::manifest::process_manifest_publication_point( + &store, + &policy, + &manifest_rsync_uri, + rsync_base_uri, + issuer_ca_der.as_slice(), + Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + validation_time, + ) + .expect("process manifest publication point"); + + (pp.snapshot, issuer_ca_der, validation_time) +} + +fn sample_vcir_for_projection( + now: time::OffsetDateTime, + child_cert_hash: &str, +) -> ValidatedCaInstanceResult { + let manifest_uri = "rsync://example.test/repo/issuer/issuer.mft".to_string(); + let current_crl_uri = "rsync://example.test/repo/issuer/issuer.crl".to_string(); + let child_cert_uri = "rsync://example.test/repo/issuer/child.cer".to_string(); + let child_manifest_uri = "rsync://example.test/repo/child/child.mft".to_string(); + let roa_uri = "rsync://example.test/repo/issuer/a.roa".to_string(); + let aspa_uri = "rsync://example.test/repo/issuer/a.asa".to_string(); + let router_uri = "rsync://example.test/repo/issuer/router.cer".to_string(); + let manifest_hash = sha256_hex(b"manifest-bytes"); + let current_crl_hash = sha256_hex(b"current-crl-bytes"); + let roa_hash = sha256_hex(b"roa-bytes"); + let aspa_hash = sha256_hex(b"aspa-bytes"); + let router_hash = sha256_hex(b"router-bytes"); + let ee_hash = sha256_hex(b"ee-cert-bytes"); + let gate_until = PackTime::from_utc_offset_datetime(now + time::Duration::hours(1)); + let ccr_manifest_projection = VcirCcrManifestProjection { + manifest_rsync_uri: manifest_uri.clone(), + manifest_sha256: hex::decode(&manifest_hash).expect("decode manifest hash"), + manifest_size: 2048, + manifest_ee_aki: vec![0x11; 20], + manifest_number_be: vec![1], + manifest_this_update: PackTime::from_utc_offset_datetime(now), + manifest_sia_locations_der: vec![vec![ + 0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05, + b'r', b's', b'y', b'n', b'c', + ]], + subordinate_skis: vec![vec![0x33; 20]], + }; + ValidatedCaInstanceResult { + manifest_rsync_uri: manifest_uri.clone(), + parent_manifest_rsync_uri: None, + tal_id: "test-tal".to_string(), + ca_subject_name: "CN=Issuer".to_string(), + ca_ski: "11".repeat(20), + issuer_ski: "22".repeat(20), + last_successful_validation_time: PackTime::from_utc_offset_datetime(now), + current_manifest_rsync_uri: manifest_uri.clone(), + current_crl_rsync_uri: current_crl_uri.clone(), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: vec![1], + validated_manifest_this_update: PackTime::from_utc_offset_datetime(now), + validated_manifest_next_update: gate_until.clone(), + }, + ccr_manifest_projection, + instance_gate: VcirInstanceGate { + manifest_next_update: gate_until.clone(), + current_crl_next_update: gate_until.clone(), + self_ca_not_after: PackTime::from_utc_offset_datetime(now + time::Duration::hours(2)), + instance_effective_until: gate_until.clone(), + }, + child_entries: vec![VcirChildEntry { + child_manifest_rsync_uri: child_manifest_uri, + child_cert_rsync_uri: child_cert_uri.clone(), + child_cert_hash: child_cert_hash.to_string(), + child_ski: "33".repeat(20), + child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(), + child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), + child_rrdp_notification_uri: Some("https://example.test/child-notify.xml".to_string()), + child_effective_ip_resources: None, + child_effective_as_resources: None, + accepted_at_validation_time: PackTime::from_utc_offset_datetime(now), + }], + local_outputs: vec![ + VcirLocalOutput { + output_id: sha256_hex(b"vrp-out"), + output_type: VcirOutputType::Vrp, + item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(30)), + source_object_uri: roa_uri.clone(), + source_object_type: "roa".to_string(), + source_object_hash: roa_hash.clone(), + source_ee_cert_hash: ee_hash.clone(), + payload_json: serde_json::json!({"asn": 64496, "prefix": "203.0.113.0/24", "max_length": 24}).to_string(), + rule_hash: sha256_hex(b"roa-rule"), + validation_path_hint: vec![manifest_uri.clone(), roa_uri.clone()], + }, + VcirLocalOutput { + output_id: sha256_hex(b"aspa-out"), + output_type: VcirOutputType::Aspa, + item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(30)), + source_object_uri: aspa_uri.clone(), + source_object_type: "aspa".to_string(), + source_object_hash: aspa_hash.clone(), + source_ee_cert_hash: ee_hash, + payload_json: serde_json::json!({"customer_as_id": 64496, "provider_as_ids": [64497, 64498]}).to_string(), + rule_hash: sha256_hex(b"aspa-rule"), + validation_path_hint: vec![manifest_uri.clone(), aspa_uri.clone()], + }, + VcirLocalOutput { + output_id: sha256_hex(b"router-key-out"), + output_type: VcirOutputType::RouterKey, + item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(30)), + source_object_uri: router_uri.clone(), + source_object_type: "router_key".to_string(), + source_object_hash: router_hash.clone(), + source_ee_cert_hash: router_hash.clone(), + payload_json: serde_json::json!({ + "as_id": 64496, + "ski_hex": "11".repeat(20), + "spki_der_base64": base64::engine::general_purpose::STANDARD.encode([0x30u8, 0x00]), + }).to_string(), + rule_hash: sha256_hex(b"router-key-rule"), + validation_path_hint: vec![manifest_uri.clone(), router_uri.clone()], + }, + ], + related_artifacts: vec![ + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::Manifest, + artifact_kind: VcirArtifactKind::Mft, + uri: Some(manifest_uri.clone()), + sha256: manifest_hash, + object_type: Some("mft".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::CurrentCrl, + artifact_kind: VcirArtifactKind::Crl, + uri: Some(current_crl_uri), + sha256: current_crl_hash, + object_type: Some("crl".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::ChildCaCert, + artifact_kind: VcirArtifactKind::Cer, + uri: Some(child_cert_uri), + sha256: child_cert_hash.to_string(), + object_type: Some("cer".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some(roa_uri), + sha256: roa_hash, + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Aspa, + uri: Some(aspa_uri), + sha256: aspa_hash, + object_type: Some("aspa".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + }, + ], + summary: VcirSummary { + local_vrp_count: 1, + local_aspa_count: 1, + local_router_key_count: 1, + child_count: 1, + accepted_object_count: 4, + rejected_object_count: 0, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible: true, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: Vec::new(), + }, + } +} + +#[test] +fn never_http_fetcher_returns_error() { + let f = NeverHttpFetcher; + let err = f.fetch("https://example.test/").unwrap_err(); + assert!(err.contains("disabled"), "{err}"); +} + +#[test] +fn kind_from_rsync_uri_classifies_known_extensions() { + assert_eq!( + kind_from_rsync_uri("rsync://example.test/x.crl"), + AuditObjectKind::Crl + ); + assert_eq!( + kind_from_rsync_uri("rsync://example.test/x.cer"), + AuditObjectKind::Certificate + ); + assert_eq!( + kind_from_rsync_uri("rsync://example.test/x.roa"), + AuditObjectKind::Roa + ); + assert_eq!( + kind_from_rsync_uri("rsync://example.test/x.asa"), + AuditObjectKind::Aspa + ); + assert_eq!( + kind_from_rsync_uri("rsync://example.test/x.bin"), + AuditObjectKind::Other + ); +} + +#[test] +fn build_vcir_local_outputs_prefers_cached_outputs() { + let pack = dummy_pack_with_files(vec![]); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: vec![1], + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let cached = vec![VcirLocalOutput { + output_id: "cached-output".to_string(), + output_type: VcirOutputType::Vrp, + item_effective_until: pack.next_update.clone(), + source_object_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), + source_object_type: "roa".to_string(), + source_object_hash: sha256_hex(b"cached-roa"), + source_ee_cert_hash: sha256_hex(b"cached-ee"), + payload_json: "{\"asn\":64500}".to_string(), + rule_hash: sha256_hex(b"cached-rule"), + validation_path_hint: vec![pack.manifest_rsync_uri.clone()], + }]; + let outputs = build_vcir_local_outputs( + &ca, + &pack, + &crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: cached.clone(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: Vec::new(), + }, + ) + .expect("reuse cached outputs"); + assert_eq!(outputs, cached); +} + +#[test] +fn persist_vcir_non_repository_evidence_stores_current_ca_cert_only() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( + &pack, + &Policy::default(), + issuer_ca_der.as_slice(), + Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + issuer_ca.tbs.extensions.ip_resources.as_ref(), + issuer_ca.tbs.extensions.as_resources.as_ref(), + validation_time, + None, + ); + assert!( + !objects.local_outputs_cache.is_empty(), + "expected local outputs from signed objects" + ); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: issuer_ca_der.clone(), + ca_certificate_rsync_uri: Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), + ), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + persist_vcir_non_repository_evidence(&store, &ca).expect("persist embedded evidence"); + + let issuer_hash = sha256_hex(&issuer_ca_der); + let issuer_entry = store + .get_raw_by_hash_entry(&issuer_hash) + .expect("load issuer raw entry") + .expect("issuer raw entry present"); + assert!( + issuer_entry + .origin_uris + .iter() + .any(|uri| uri.ends_with("BfycW4hQb3wNP4YsiJW-1n6fjro.cer")) + ); + let first_output = objects + .local_outputs_cache + .first() + .expect("first local output"); + assert!( + store + .get_raw_by_hash_entry(&first_output.source_ee_cert_hash) + .expect("load source ee raw") + .is_none() + ); +} + +#[test] +fn build_router_key_local_outputs_encodes_router_key_payloads() { + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: Vec::new(), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let outputs = build_router_key_local_outputs( + &ca, + &[RouterKeyPayload { + as_id: 64496, + ski: vec![0x11; 20], + spki_der: vec![0x30, 0x00], + source_object_uri: "rsync://example.test/repo/issuer/router.cer".to_string(), + source_object_hash: "11".repeat(32), + source_ee_cert_hash: "11".repeat(32), + item_effective_until: PackTime { + rfc3339_utc: "2026-12-31T00:00:00Z".to_string(), + }, + }], + ); + assert_eq!(outputs.len(), 1); + assert_eq!(outputs[0].output_type, VcirOutputType::RouterKey); + assert_eq!(outputs[0].source_object_type, "router_key"); + assert!(outputs[0].payload_json.contains("spki_der_base64")); +} + +#[test] +fn build_vcir_local_outputs_falls_back_to_decoding_accepted_objects_when_cache_is_empty() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( + &pack, + &Policy::default(), + issuer_ca_der.as_slice(), + Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + issuer_ca.tbs.extensions.ip_resources.as_ref(), + issuer_ca.tbs.extensions.as_resources.as_ref(), + validation_time, + None, + ); + let mut objects_without_cache = objects.clone(); + objects_without_cache.local_outputs_cache.clear(); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: issuer_ca_der, + ca_certificate_rsync_uri: Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), + ), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let local_outputs = build_vcir_local_outputs(&ca, &pack, &objects_without_cache) + .expect("rebuild vcir local outputs"); + assert!(!local_outputs.is_empty()); + assert_eq!(local_outputs.len(), objects.vrps.len()); + assert!( + local_outputs + .iter() + .all(|output| output.output_type == VcirOutputType::Vrp) + ); +} + +#[test] +fn finalize_fresh_publication_point_releases_local_outputs_cache_after_persist() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let mut objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( + &pack, + &Policy::default(), + issuer_ca_der.as_slice(), + Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + issuer_ca.tbs.extensions.ip_resources.as_ref(), + issuer_ca.tbs.extensions.as_resources.as_ref(), + validation_time, + None, + ); + assert!( + !objects.local_outputs_cache.is_empty(), + "expected local outputs from signed objects" + ); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + }; + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: issuer_ca_der.clone(), + ca_certificate_rsync_uri: Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), + ), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let fresh_point = FreshValidatedPublicationPoint { + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + manifest_number_be: pack.manifest_number_be.clone(), + this_update: pack.this_update.clone(), + next_update: pack.next_update.clone(), + verified_at: pack.verified_at.clone(), + manifest_bytes: pack.manifest_bytes.clone(), + files: pack.files.clone(), + }; + + objects.local_outputs_cache.shrink_to_fit(); + let original_cache_capacity = objects.local_outputs_cache.capacity(); + let finalized = runner + .finalize_fresh_publication_point_from_reducer( + &ca, + &fresh_point, + Vec::new(), + objects, + Vec::new(), + Vec::new(), + None, + None, + 0, + None, + ) + .expect("finalize fresh publication point"); + + assert!( + finalized.result.objects.local_outputs_cache.is_empty(), + "local outputs cache should be released after VCIR persistence" + ); + assert_eq!( + finalized.result.objects.local_outputs_cache.capacity(), + 0, + "released cache should not keep its backing allocation" + ); + assert!(original_cache_capacity > 0); + + let persisted = store + .get_vcir(&pack.manifest_rsync_uri) + .expect("load persisted vcir") + .expect("persisted vcir"); + assert!( + !persisted.local_outputs.is_empty(), + "VCIR should still persist local outputs before cache release" + ); +} + +#[test] +fn persist_vcir_for_fresh_result_stores_vcir_and_audit_indexes_for_real_snapshot() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( + &pack, + &Policy::default(), + issuer_ca_der.as_slice(), + Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + issuer_ca.tbs.extensions.ip_resources.as_ref(), + issuer_ca.tbs.extensions.as_resources.as_ref(), + validation_time, + None, + ); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: issuer_ca_der.clone(), + ca_certificate_rsync_uri: Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), + ), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + + persist_vcir_for_fresh_result(&store, &ca, &pack, &objects, &[], &[], &[], validation_time) + .expect("persist vcir for fresh result"); + + let vcir = store + .get_vcir(&pack.manifest_rsync_uri) + .expect("get vcir") + .expect("vcir exists"); + assert_eq!(vcir.manifest_rsync_uri, pack.manifest_rsync_uri); + assert_eq!(vcir.summary.local_vrp_count as usize, objects.vrps.len()); + assert_eq!( + vcir.ccr_manifest_projection.manifest_rsync_uri, + pack.manifest_rsync_uri + ); + assert_eq!( + vcir.ccr_manifest_projection.manifest_number_be, + pack.manifest_number_be + ); + assert_eq!( + vcir.ccr_manifest_projection.manifest_this_update, + pack.this_update + ); + assert_eq!( + vcir.ccr_manifest_projection.manifest_size, + pack.manifest_bytes.len() as u64 + ); + let first_output = vcir.local_outputs.first().expect("local outputs stored"); + assert!( + store + .get_audit_rule_index_entry(crate::storage::AuditRuleKind::Roa, &first_output.rule_hash) + .expect("get audit rule index entry") + .is_some() + ); +} + +#[test] +fn build_vcir_ccr_manifest_projection_from_fresh_real_snapshot_matches_manifest_contents() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: issuer_ca_der, + ca_certificate_rsync_uri: Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), + ), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let child_discovery = + discover_children_from_fresh_snapshot_with_audit(&ca, &pack, validation_time, None) + .expect("discover children"); + let child_entries = build_vcir_child_entries(&child_discovery.children, validation_time) + .expect("build child entries"); + + let projection = build_vcir_ccr_manifest_projection_from_fresh(&ca, &pack, &child_entries) + .expect("build ccr manifest projection"); + let manifest = ManifestObject::decode_der(&pack.manifest_bytes).expect("decode manifest"); + let expected_locations = match manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .subject_info_access + .as_ref() + .expect("manifest sia") + { + SubjectInfoAccess::Ee(ee_sia) => ee_sia + .access_descriptions + .iter() + .map(encode_access_description_der_for_vcir_ccr_projection) + .collect::, _>>() + .expect("encode locations"), + SubjectInfoAccess::Ca(_) => panic!("manifest ee SIA should not be CA variant"), + }; + + assert_eq!(projection.manifest_rsync_uri, pack.manifest_rsync_uri); + assert_eq!( + projection.manifest_sha256, + sha2::Sha256::digest(&pack.manifest_bytes).to_vec() + ); + assert_eq!(projection.manifest_size, pack.manifest_bytes.len() as u64); + assert_eq!( + projection.manifest_ee_aki, + manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .authority_key_identifier + .clone() + .expect("manifest aki") + ); + assert_eq!( + projection.manifest_number_be, + manifest.manifest.manifest_number.bytes_be + ); + assert_eq!(projection.manifest_this_update, pack.this_update); + assert_eq!(projection.manifest_sia_locations_der, expected_locations); + let expected_subordinate_skis = child_entries + .iter() + .map(|child| hex::decode(&child.child_ski).expect("decode child ski")) + .collect::>(); + assert_eq!(projection.subordinate_skis, expected_subordinate_skis); +} + +#[test] +fn build_vcir_related_artifacts_classifies_snapshot_files_and_audit_statuses() { + let manifest_bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft", + ), + ) + .expect("read manifest fixture"); + let crl_bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl", + ), + ) + .expect("read crl fixture"); + let pack = PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_number_be: vec![1], + this_update: PackTime::from_utc_offset_datetime(time::OffsetDateTime::now_utc()), + next_update: PackTime::from_utc_offset_datetime( + time::OffsetDateTime::now_utc() + time::Duration::hours(1), + ), + verified_at: PackTime::from_utc_offset_datetime(time::OffsetDateTime::now_utc()), + manifest_bytes, + files: vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + crl_bytes, + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + vec![1u8, 2], + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/a.roa", + vec![3u8, 4], + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/a.asa", + vec![5u8, 6], + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/a.gbr", + vec![7u8, 8], + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/extra.bin", + vec![9u8], + ), + ], + }; + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: vec![0x11, 0x22], + ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: vec![ + ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), + sha256_hex: sha256_hex_from_32(&pack.files[2].sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some("bad roa".to_string()), + }, + ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/issuer/a.asa".to_string(), + sha256_hex: sha256_hex_from_32(&pack.files[3].sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some("skipped aspa".to_string()), + }, + ], + }; + let artifacts = build_vcir_related_artifacts( + &ca, + &pack, + "rsync://example.test/repo/issuer/issuer.crl", + &objects, + &[], + ); + assert!( + artifacts + .iter() + .any(|artifact| artifact.artifact_role == VcirArtifactRole::Manifest) + ); + assert!( + artifacts + .iter() + .any(|artifact| artifact.artifact_role == VcirArtifactRole::TrustAnchorCert) + ); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/issuer.crl") + && artifact.artifact_role == VcirArtifactRole::CurrentCrl)); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/child.cer") + && artifact.artifact_role == VcirArtifactRole::ChildCaCert)); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/a.roa") + && artifact.validation_status == VcirArtifactValidationStatus::Rejected)); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/a.asa") + && artifact.validation_status == VcirArtifactValidationStatus::WarningOnly)); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/a.gbr") + && artifact.artifact_kind == VcirArtifactKind::Gbr)); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/extra.bin") + && artifact.artifact_kind == VcirArtifactKind::Other)); + assert!( + !artifacts + .iter() + .any(|artifact| artifact.uri.is_none() + && artifact.sha256 == sha256_hex(b"embedded-ee")), + "embedded EE cert artifacts should no longer be persisted separately" + ); +} + +#[test] +fn select_issuer_crl_from_snapshot_reports_missing_crldp_for_self_signed_cert() { + let ta_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ta/apnic-ta.cer"), + ) + .expect("read TA fixture"); + + let pack = dummy_pack_with_files(vec![]); + let err = select_issuer_crl_from_snapshot(&ta_der, &pack).unwrap_err(); + assert!(err.contains("CRLDistributionPoints missing"), "{err}"); +} + +#[test] +fn select_issuer_crl_from_snapshot_finds_matching_crl() { + // Use real fixtures to ensure child cert has CRLDP rsync URI and CRL exists. + let child_cert_der = + std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer", + )) + .expect("read child cert fixture"); + let crl_der = + std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl", + )) + .expect("read crl fixture"); + + let pack = dummy_pack_with_files(vec![PackFile::from_bytes_compute_sha256( + "rsync://ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl", + crl_der.clone(), + )]); + + let (uri, found) = + select_issuer_crl_from_snapshot(child_cert_der.as_slice(), &pack).expect("find crl"); + assert_eq!( + uri, + "rsync://ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl" + ); + assert_eq!(found, crl_der.as_slice()); +} + +#[test] +fn discover_children_from_fresh_pack_discovers_child_ca() { + let g = generate_chain_and_crl(); + + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + g.child_ca_der.clone(), + ), + ]); + + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: g.issuer_ca_der.clone(), + ca_certificate_rsync_uri: None, + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let now = time::OffsetDateTime::now_utc(); + let children = discover_children_from_fresh_snapshot_with_audit(&issuer, &pack, now, None) + .expect("discover children") + .children; + assert_eq!(children.len(), 1); + assert_eq!( + children[0].discovered_from.parent_manifest_rsync_uri, + issuer.manifest_rsync_uri + ); + assert_eq!( + children[0].discovered_from.child_ca_certificate_rsync_uri, + "rsync://example.test/repo/issuer/child.cer" + ); + assert_eq!( + children[0].handle.rsync_base_uri, + "rsync://example.test/repo/child/".to_string() + ); + assert_eq!( + children[0].handle.manifest_rsync_uri, + "rsync://example.test/repo/child/child.mft".to_string() + ); + assert_eq!( + children[0].handle.publication_point_rsync_uri, + "rsync://example.test/repo/child/".to_string() + ); + assert_eq!( + children[0].handle.rrdp_notification_uri.as_deref(), + Some("https://example.test/notification.xml") + ); +} + +#[test] +fn discover_children_with_audit_records_missing_crl_for_child_certificate() { + let now = time::OffsetDateTime::now_utc(); + + let child_ca_der = + std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer", + )) + .expect("read child ca fixture"); + + // Pack contains the child CA cert but does not contain the CRL referenced by the child + // certificate CRLDistributionPoints extension. + let pack = dummy_pack_with_files(vec![PackFile::from_bytes_compute_sha256( + "rsync://ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer", + child_ca_der, + )]); + + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: vec![1], + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + + let out = discover_children_from_fresh_snapshot_with_audit(&issuer, &pack, now, None) + .expect("discovery should succeed with audit error"); + assert_eq!(out.children.len(), 0); + assert_eq!(out.audits.len(), 1); + assert_eq!( + out.audits[0].rsync_uri, + "rsync://ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer" + ); + assert_eq!(out.audits[0].result, AuditObjectResult::Error); + assert!( + out.audits[0] + .detail + .as_deref() + .unwrap_or("") + .contains("cannot select issuer CRL"), + "expected deterministic CRL selection failure to be recorded" + ); +} + +#[test] +fn runner_offline_rsync_fixture_produces_pack_and_warnings() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + + // Pick a validation_time inside the fixture manifest's validity window to keep this + // test stable across wall-clock time. + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = { + let this_update = fixture_manifest.manifest.this_update; + let next_update = fixture_manifest.manifest.next_update; + let candidate = this_update + time::Duration::seconds(60); + if candidate < next_update { + candidate + } else { + this_update + } + }; + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + }; + + // For this fixture-driven smoke, we provide the correct issuer CA certificate (the CA for + // this publication point) so ROA EE certificate paths can validate. + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: issuer_ca_der, + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri.clone(), + rrdp_notification_uri: None, + }; + + let out = runner + .run_publication_point(&handle) + .expect("run publication point"); + assert_eq!(out.source, PublicationPointSource::Fresh); + let pack = out.snapshot.expect("fresh run pack"); + assert_eq!(pack.manifest_rsync_uri, manifest_rsync_uri); + assert!(pack.files.len() > 1); + assert!( + out.objects.vrps.len() > 1, + "expected to extract VRPs from ROAs" + ); + + let vcir = store + .get_vcir(&manifest_rsync_uri) + .expect("get vcir") + .expect("vcir exists after fresh run"); + assert_eq!(vcir.manifest_rsync_uri, manifest_rsync_uri); + assert_eq!(vcir.tal_id, "test-tal"); + assert!( + vcir.local_outputs + .iter() + .any(|output| output.output_type == crate::storage::VcirOutputType::Vrp), + "expected VCIR local_outputs to contain VRP entries" + ); + let first_vrp = vcir + .local_outputs + .iter() + .find(|output| output.output_type == crate::storage::VcirOutputType::Vrp) + .expect("first VCIR VRP output"); + let audit_rule = store + .get_audit_rule_index_entry(crate::storage::AuditRuleKind::Roa, &first_vrp.rule_hash) + .expect("get audit rule index") + .expect("audit rule index exists"); + assert_eq!(audit_rule.manifest_rsync_uri, manifest_rsync_uri); + assert_eq!(audit_rule.output_id, first_vrp.output_id); + let trace = crate::audit_trace::trace_rule_to_root( + &store, + crate::storage::AuditRuleKind::Roa, + &first_vrp.rule_hash, + ) + .expect("trace rule") + .expect("trace exists"); + assert_eq!(trace.chain_leaf_to_root.len(), 1); + assert_eq!( + trace.chain_leaf_to_root[0].manifest_rsync_uri, + manifest_rsync_uri + ); + assert!(trace.source_object_raw.raw_present); + assert!(trace.source_ee_cert_raw.raw_present); +} + +#[test] +fn runner_rsync_dedup_skips_second_sync_for_same_base() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: issuer_ca_der, + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri.clone(), + rrdp_notification_uri: None, + }; + + struct CountingRsyncFetcher { + inner: LocalDirRsyncFetcher, + calls: Arc, + } + impl RsyncFetcher for CountingRsyncFetcher { + fn fetch_objects( + &self, + rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + self.calls.fetch_add(1, Ordering::SeqCst); + self.inner.fetch_objects(rsync_base_uri) + } + } + + let calls = Arc::new(AtomicUsize::new(0)); + let rsync = CountingRsyncFetcher { + inner: LocalDirRsyncFetcher::new(&fixture_dir), + calls: calls.clone(), + }; + + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &rsync, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + }; + + let first = runner.run_publication_point(&handle).expect("first run ok"); + assert_eq!(first.source, PublicationPointSource::Fresh); + + let second = runner + .run_publication_point(&handle) + .expect("second run ok"); + assert_eq!(second.source, PublicationPointSource::Fresh); + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "rsync should be called once" + ); +} + +#[test] +fn runner_rsync_dedup_skips_second_sync_for_same_module_scope() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let first_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let second_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/sub/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{first_base_uri}{manifest_file}"); + + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: issuer_ca_der, + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: first_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: first_base_uri.clone(), + rrdp_notification_uri: None, + }; + let second_handle = CaInstanceHandle { + rsync_base_uri: second_base_uri.clone(), + publication_point_rsync_uri: second_base_uri.clone(), + ..handle.clone() + }; + + struct ModuleScopeRsyncFetcher { + inner: LocalDirRsyncFetcher, + calls: Arc, + } + impl RsyncFetcher for ModuleScopeRsyncFetcher { + fn fetch_objects( + &self, + rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + self.calls.fetch_add(1, Ordering::SeqCst); + self.inner.fetch_objects(rsync_base_uri) + } + + fn dedup_key(&self, _rsync_base_uri: &str) -> String { + "rsync://rpki.cernet.net/repo/".to_string() + } + } + + let calls = Arc::new(AtomicUsize::new(0)); + let rsync = ModuleScopeRsyncFetcher { + inner: LocalDirRsyncFetcher::new(&fixture_dir), + calls: calls.clone(), + }; + + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &rsync, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + }; + + let first = runner.run_publication_point(&handle).expect("first run ok"); + assert_eq!(first.source, PublicationPointSource::Fresh); + + let second = runner + .run_publication_point(&second_handle) + .expect("second run ok"); + assert!(matches!( + second.source, + PublicationPointSource::Fresh | PublicationPointSource::VcirCurrentInstance + )); + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "module-scope dedup should skip second sync" + ); +} + +#[test] +fn runner_rsync_dedup_works_in_rsync_only_mode_even_when_rrdp_notify_exists() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let first_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let second_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/sub/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{first_base_uri}{manifest_file}"); + + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: issuer_ca_der, + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: first_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: first_base_uri.clone(), + rrdp_notification_uri: Some("https://rrdp.example.test/notification.xml".to_string()), + }; + let second_handle = CaInstanceHandle { + rsync_base_uri: second_base_uri.clone(), + publication_point_rsync_uri: second_base_uri.clone(), + ..handle.clone() + }; + + struct ModuleScopeRsyncFetcher { + inner: LocalDirRsyncFetcher, + calls: Arc, + } + impl RsyncFetcher for ModuleScopeRsyncFetcher { + fn fetch_objects( + &self, + rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + self.calls.fetch_add(1, Ordering::SeqCst); + self.inner.fetch_objects(rsync_base_uri) + } + + fn dedup_key(&self, _rsync_base_uri: &str) -> String { + "rsync://rpki.cernet.net/repo/".to_string() + } + } + + let calls = Arc::new(AtomicUsize::new(0)); + let rsync = ModuleScopeRsyncFetcher { + inner: LocalDirRsyncFetcher::new(&fixture_dir), + calls: calls.clone(), + }; + + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &rsync, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + }; + + let first = runner.run_publication_point(&handle).expect("first run ok"); + assert_eq!(first.source, PublicationPointSource::Fresh); + + let second = runner + .run_publication_point(&second_handle) + .expect("second run ok"); + assert!(matches!( + second.source, + PublicationPointSource::Fresh | PublicationPointSource::VcirCurrentInstance + )); + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "rsync-only mode must deduplicate by rsync scope even when RRDP notification is present" + ); +} + +#[test] +fn runner_when_repo_sync_fails_uses_current_instance_vcir_and_keeps_children_empty_for_fixture() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: issuer_ca_der, + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri.clone(), + rrdp_notification_uri: None, + }; + + // First: successful fresh run to populate the latest VCIR baseline. + let ok_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + }; + let first = ok_runner + .run_publication_point(&handle) + .expect("first run ok"); + assert_eq!(first.source, PublicationPointSource::Fresh); + assert!( + first.discovered_children.is_empty(), + "fixture has no child .cer" + ); + + // Second: repo sync fails, but we can still reuse current-instance VCIR. + let bad_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + }; + let second = bad_runner + .run_publication_point(&handle) + .expect("should reuse current-instance VCIR"); + assert_eq!(second.source, PublicationPointSource::VcirCurrentInstance); + assert!(second.discovered_children.is_empty()); + assert!( + second + .warnings + .iter() + .any(|w| w.message.contains("repo sync failed")), + "expected warning about repo sync failure" + ); +} + +#[test] +fn build_publication_point_audit_emits_no_audit_entry_for_duplicate_pack_uri() { + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256("rsync://example.test/repo/dup.roa", vec![1u8]), + PackFile::from_bytes_compute_sha256("rsync://example.test/repo/dup.roa", vec![2u8]), + ]); + let pp = crate::validation::manifest::PublicationPointResult { + source: crate::validation::manifest::PublicationPointSource::VcirCurrentInstance, + snapshot: pack.clone(), + warnings: Vec::new(), + }; + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: vec![1], + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: Vec::new(), + }; + + let audit = build_publication_point_audit_from_snapshot( + &ca, + pp.source, + None, + None, + None, + None, + &pp.snapshot, + &[], + &objects, + &[], + ); + assert_eq!(audit.source, "vcir_current_instance"); + assert_eq!(audit.repo_sync_phase, None); + assert_eq!(audit.repo_terminal_state, "fallback_current_instance"); + assert!( + audit + .objects + .iter() + .any(|e| e.detail.as_deref() == Some("skipped: no audit entry")), + "expected a duplicate key to produce a 'no audit entry' placeholder" + ); +} + +#[test] +fn build_publication_point_audit_marks_invalid_crl_as_error_and_overlays_roa_audit() { + let now = time::OffsetDateTime::now_utc(); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256("rsync://example.test/repo/issuer/bad.crl", vec![0u8]), + PackFile::from_bytes_compute_sha256("rsync://example.test/repo/issuer/x.roa", vec![1u8]), + ]); + + let pp = crate::validation::manifest::PublicationPointResult { + source: crate::validation::manifest::PublicationPointSource::Fresh, + snapshot: pack.clone(), + warnings: Vec::new(), + }; + + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: vec![1], + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: vec![ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/issuer/x.roa".to_string(), + sha256_hex: sha256_hex_from_32(&pack.files[1].sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Ok, + detail: None, + }], + }; + + let audit = build_publication_point_audit_from_snapshot( + &issuer, + pp.source, + Some("rsync"), + Some("rsync_only_ok"), + Some(123), + Some("none"), + &pp.snapshot, + &[], + &objects, + &[], + ); + assert_eq!(audit.objects[0].kind, AuditObjectKind::Manifest); + assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync")); + assert_eq!(audit.repo_sync_phase.as_deref(), Some("rsync_only_ok")); + assert_eq!(audit.repo_sync_duration_ms, Some(123)); + assert_eq!(audit.repo_sync_error.as_deref(), Some("none")); + assert_eq!(audit.repo_terminal_state, "fresh"); + + let crl = audit + .objects + .iter() + .find(|e| e.rsync_uri.ends_with("bad.crl")) + .expect("crl entry"); + assert!(matches!(crl.result, AuditObjectResult::Error)); + + let roa = audit + .objects + .iter() + .find(|e| e.rsync_uri.ends_with("x.roa")) + .expect("roa entry"); + assert!(matches!(roa.result, AuditObjectResult::Ok)); + + // Smoke that time fields are populated from pack. + assert!(audit.verified_at_rfc3339_utc.contains('T')); + assert!(audit.this_update_rfc3339_utc.contains('T')); + assert!(audit.next_update_rfc3339_utc.contains('T')); + let _ = now; +} + +#[test] +fn discover_children_with_router_certificate_records_ok_audit_and_no_child() { + let g = generate_router_cert_with_variant("ec-p256", true); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/router.cer", + g.router_der.clone(), + ), + ]); + + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: g.issuer_ca_der.clone(), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let out = discover_children_from_fresh_snapshot_with_audit( + &issuer, + &pack, + time::OffsetDateTime::now_utc(), + None, + ) + .expect("discover router cert"); + assert!(out.children.is_empty()); + assert_eq!(out.audits.len(), 1); + assert!(matches!(out.audits[0].result, AuditObjectResult::Ok)); + assert!( + out.audits[0] + .detail + .as_deref() + .unwrap_or("") + .contains("validated BGPsec router certificate") + ); +} + +#[test] +fn discover_children_with_non_router_ee_certificate_records_skipped_audit() { + let g = generate_router_cert_with_variant("ec-p256", false); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/router-no-eku.cer", + g.router_der.clone(), + ), + ]); + + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: g.issuer_ca_der.clone(), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let out = discover_children_from_fresh_snapshot_with_audit( + &issuer, + &pack, + time::OffsetDateTime::now_utc(), + None, + ) + .expect("discover non-router cert"); + assert!(out.children.is_empty()); + assert_eq!(out.audits.len(), 1); + assert!(matches!(out.audits[0].result, AuditObjectResult::Skipped)); + assert!( + out.audits[0] + .detail + .as_deref() + .unwrap_or("") + .contains("not a CA resource certificate or BGPsec router certificate") + ); +} + +#[test] +fn discover_children_with_invalid_router_certificate_records_error_audit() { + let g = generate_router_cert_with_variant("ec-p384", true); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/router-invalid.cer", + g.router_der.clone(), + ), + ]); + + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: g.issuer_ca_der.clone(), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let out = discover_children_from_fresh_snapshot_with_audit( + &issuer, + &pack, + time::OffsetDateTime::now_utc(), + None, + ) + .expect("discover invalid router cert"); + assert!(out.children.is_empty()); + assert_eq!(out.audits.len(), 1); + assert!(matches!(out.audits[0].result, AuditObjectResult::Error)); + assert!( + out.audits[0] + .detail + .as_deref() + .unwrap_or("") + .contains("router certificate validation failed") + ); +} + +#[test] +fn discover_children_with_audit_records_decode_error_for_corrupt_cer() { + let g = generate_chain_and_crl(); + + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/corrupt.cer", + vec![0u8], + ), + ]); + + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: g.issuer_ca_der.clone(), + ca_certificate_rsync_uri: None, + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let now = time::OffsetDateTime::now_utc(); + let out = discover_children_from_fresh_snapshot_with_audit(&issuer, &pack, now, None) + .expect("discover children"); + assert!(out.children.is_empty()); + assert_eq!(out.audits.len(), 1); + assert!(matches!(out.audits[0].result, AuditObjectResult::Error)); +} + +#[test] +fn select_issuer_crl_uri_for_child_covers_missing_and_not_found_paths() { + let g = generate_chain_and_crl(); + let child = ResourceCertificate::decode_der(&g.child_ca_der).expect("decode child cert"); + + let empty: std::collections::HashMap = + std::collections::HashMap::new(); + let err = select_issuer_crl_uri_for_child(&child, &empty).unwrap_err(); + assert!(err.contains("no CRL available"), "{err}"); + + let ta_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ta/apnic-ta.cer"), + ) + .expect("read TA fixture"); + let ta = ResourceCertificate::decode_der(&ta_der).expect("decode TA fixture"); + let mut cache = std::collections::HashMap::new(); + cache.insert( + "rsync://example.test/repo/issuer/issuer.crl".to_string(), + CachedIssuerCrl::Pending(g.issuer_crl_der.clone()), + ); + let err = select_issuer_crl_uri_for_child(&ta, &cache).unwrap_err(); + assert!(err.contains("CRLDistributionPoints missing"), "{err}"); + + let mut wrong = std::collections::HashMap::new(); + wrong.insert( + "rsync://example.test/repo/issuer/other.crl".to_string(), + CachedIssuerCrl::Pending(g.issuer_crl_der), + ); + let err = select_issuer_crl_uri_for_child(&child, &wrong).unwrap_err(); + assert!( + err.contains("not found in publication point snapshot"), + "{err}" + ); +} + +#[test] +fn ensure_issuer_crl_verified_promotes_pending_cache_entry() { + let g = generate_chain_and_crl(); + let mut cache = std::collections::HashMap::new(); + let crl_uri = "rsync://example.test/repo/issuer/issuer.crl".to_string(); + cache.insert( + crl_uri.clone(), + CachedIssuerCrl::Pending(g.issuer_crl_der.clone()), + ); + + let first = ensure_issuer_crl_verified(&crl_uri, &mut cache, &g.issuer_ca_der) + .expect("verify pending CRL"); + assert!(first.revoked_serials.is_empty()); + assert!(matches!(cache.get(&crl_uri), Some(CachedIssuerCrl::Ok(_)))); + + let second = ensure_issuer_crl_verified(&crl_uri, &mut cache, &g.issuer_ca_der) + .expect("reuse verified CRL"); + assert!(second.revoked_serials.is_empty()); +} + +#[test] +fn discover_children_with_invalid_issuer_der_records_error_audit() { + let g = generate_chain_and_crl(); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + g.child_ca_der.clone(), + ), + ]); + + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: vec![0u8], + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let out = discover_children_from_fresh_snapshot_with_audit( + &issuer, + &pack, + time::OffsetDateTime::now_utc(), + None, + ) + .expect("discover children with invalid issuer der"); + assert!(out.children.is_empty()); + assert_eq!(out.audits.len(), 1); + assert!(matches!(out.audits[0].result, AuditObjectResult::Error)); + assert!( + out.audits[0] + .detail + .as_deref() + .unwrap_or("") + .contains("issuer CA decode failed") + ); +} + +#[test] +fn project_current_instance_vcir_reuses_local_outputs_and_restores_children() { + let now = time::OffsetDateTime::now_utc(); + let g = generate_chain_and_crl(); + let child_cert_hash = sha256_hex(&g.child_ca_der); + let vcir = sample_vcir_for_projection(now, &child_cert_hash); + + let store_dir = tempfile::tempdir().expect("store dir"); + let main_db = store_dir.path().join("work-db"); + let repo_bytes_db = store_dir.path().join("repo-bytes.db"); + let store = RocksStore::open_with_external_repo_bytes(&main_db, &repo_bytes_db) + .expect("open rocksdb with external repo bytes"); + store.put_vcir(&vcir).expect("put vcir"); + store + .put_blob_bytes_batch(&[(child_cert_hash.clone(), g.child_ca_der.clone())]) + .expect("put child cert repo bytes"); + assert!( + store + .get_raw_by_hash_entry(&child_cert_hash) + .expect("lookup child raw_by_hash") + .is_none(), + "child cert restoration should not require raw_by_hash entries" + ); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: Vec::new(), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + }; + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + now, + ) + .expect("project vcir"); + + assert_eq!( + projection.source, + PublicationPointSource::VcirCurrentInstance + ); + assert_eq!(projection.objects.vrps.len(), 1); + assert_eq!(projection.objects.aspas.len(), 1); + assert_eq!(projection.objects.router_keys.len(), 1); + assert_eq!(projection.discovered_children.len(), 1); + assert_eq!( + projection.discovered_children[0].handle.manifest_rsync_uri, + "rsync://example.test/repo/child/child.mft" + ); + assert_eq!( + projection.ccr_manifest_projection.as_ref(), + Some(&vcir.ccr_manifest_projection) + ); + assert!( + projection.snapshot.is_none(), + "current-instance reuse should not reconstruct a byte-backed snapshot" + ); + assert!( + !projection + .warnings + .iter() + .any(|warning| warning.message.contains("manifest failed fetch")), + "successful current-instance reuse should not duplicate the fresh fetch error" + ); + assert!( + !projection + .warnings + .iter() + .any(|warning| warning.message.contains("using latest validated result")), + "successful current-instance reuse should be tracked by source, not warning" + ); + assert!( + !projection + .warnings + .iter() + .any(|warning| warning.message.contains("manifest raw bytes missing")), + "successful current-instance reuse should not load repo bytes for audit reconstruction" + ); + assert!( + !projection + .warnings + .iter() + .any(|warning| warning.message.contains("child certificate bytes missing")), + "child discovery restoration should read child certs from repo bytes" + ); +} + +#[test] +fn project_current_instance_vcir_returns_no_output_when_instance_gate_expired() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + let this_update = PackTime::from_utc_offset_datetime(now - time::Duration::minutes(2)); + let expired = PackTime::from_utc_offset_datetime(now - time::Duration::minutes(1)); + vcir.validated_manifest_meta.validated_manifest_this_update = this_update; + vcir.validated_manifest_meta.validated_manifest_next_update = expired.clone(); + vcir.instance_gate.manifest_next_update = expired.clone(); + vcir.instance_gate.current_crl_next_update = expired.clone(); + vcir.instance_gate.instance_effective_until = expired; + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + store.put_vcir(&vcir).expect("put vcir"); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: Vec::new(), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + now, + ) + .expect("project vcir"); + + assert_eq!( + projection.source, + PublicationPointSource::FailedFetchNoCache + ); + assert!(projection.ccr_manifest_projection.is_none()); + assert!(projection.objects.vrps.is_empty()); + assert!(projection.objects.aspas.is_empty()); + assert!(projection.discovered_children.is_empty()); +} + +#[test] +fn project_current_instance_vcir_keeps_real_fresh_validation_warning() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let vcir = sample_vcir_for_projection(now, &child_cert_hash); + + let store_dir = tempfile::tempdir().expect("store dir"); + let main_db = store_dir.path().join("work-db"); + let repo_bytes_db = store_dir.path().join("repo-bytes.db"); + let store = RocksStore::open_with_external_repo_bytes(&main_db, &repo_bytes_db) + .expect("open rocksdb with external repo bytes"); + store.put_vcir(&vcir).expect("put vcir"); + store + .put_blob_bytes_batch(&[(child_cert_hash, b"child-cert".to_vec())]) + .expect("put child cert repo bytes"); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: Vec::new(), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::HashMismatch { + rsync_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), + }, + now, + ) + .expect("project vcir"); + + assert_eq!( + projection.source, + PublicationPointSource::VcirCurrentInstance + ); + assert!( + projection + .warnings + .iter() + .any(|warning| { warning.message.contains("manifest file hash mismatch") }) + ); + assert!( + !projection + .warnings + .iter() + .any(|warning| warning.message.contains("using latest validated result")), + "successful current-instance reuse should not emit bookkeeping warnings" + ); +} + +#[test] +fn project_current_instance_vcir_returns_no_output_when_latest_result_missing() { + let now = time::OffsetDateTime::now_utc(); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: Vec::new(), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + now, + ) + .expect("project without cached vcir"); + + assert_eq!( + projection.source, + PublicationPointSource::FailedFetchNoCache + ); + assert!(projection.vcir.is_none()); + assert!(projection.ccr_manifest_projection.is_none()); + assert!(projection.snapshot.is_none()); + assert!(projection.objects.audit.is_empty()); + assert!(projection.discovered_children.is_empty()); + assert!(projection.warnings.iter().any(|warning| { + warning + .message + .contains("no latest validated result for current CA instance") + })); +} + +#[test] +fn project_current_instance_vcir_returns_no_output_when_latest_result_is_ineligible() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + vcir.audit_summary.failed_fetch_eligible = false; + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + store.put_vcir(&vcir).expect("put vcir"); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: Vec::new(), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + now, + ) + .expect("project ineligible vcir"); + + assert_eq!( + projection.source, + PublicationPointSource::FailedFetchNoCache + ); + assert!(projection.vcir.is_some()); + assert!(projection.ccr_manifest_projection.is_none()); + assert!(projection.snapshot.is_none()); + assert!(projection.discovered_children.is_empty()); + assert!(projection.warnings.iter().any(|warning| { + warning + .message + .contains("latest VCIR is not marked failed-fetch eligible") + })); +} + +#[test] +fn project_current_instance_vcir_rejects_mismatched_ccr_projection_uri() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + vcir.ccr_manifest_projection.manifest_rsync_uri = + "rsync://example.test/repo/issuer/other.mft".to_string(); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + store.put_vcir(&vcir).expect("put vcir"); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: Vec::new(), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let err = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + now, + ) + .unwrap_err(); + + assert!( + err.contains("vcir CCR manifest projection URI mismatch"), + "{err}" + ); +} + +#[test] +fn fresh_and_reuse_paths_produce_equivalent_ccr_manifest_projection() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: issuer_ca_der, + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let child_discovery = + discover_children_from_fresh_snapshot_with_audit(&ca, &pack, validation_time, None) + .expect("discover children"); + let fresh_vcir = build_vcir_from_fresh_result( + &ca, + &pack, + &empty_objects_output(), + &[], + &child_discovery.audits, + &child_discovery.children, + validation_time, + ) + .expect("build fresh vcir"); + + let reuse_projection = reuse_ccr_manifest_projection_from_vcir(&ca, &fresh_vcir) + .expect("reuse projection from vcir"); + + assert_eq!(fresh_vcir.ccr_manifest_projection, reuse_projection); +} + +#[test] +fn append_ccr_manifest_projection_from_reuse_requires_projection_for_current_instance() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let runner = sample_runner_with_ccr_accumulator(&store, &policy); + + let err = runner + .append_ccr_manifest_projection_from_reuse(&VcirReuseProjection { + source: PublicationPointSource::VcirCurrentInstance, + vcir: None, + ccr_manifest_projection: None, + snapshot: None, + objects: empty_objects_output(), + child_audits: Vec::new(), + discovered_children: Vec::new(), + warnings: Vec::new(), + }) + .unwrap_err(); + + assert!(err.contains("missing CCR manifest projection"), "{err}"); + assert_eq!( + runner + .ccr_accumulator_snapshot() + .expect("ccr accumulator snapshot") + .manifest_count(), + 0 + ); +} + +#[test] +fn append_ccr_manifest_projection_from_reuse_skips_failed_fetch_no_cache() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let runner = sample_runner_with_ccr_accumulator(&store, &policy); + + runner + .append_ccr_manifest_projection_from_reuse(&VcirReuseProjection { + source: PublicationPointSource::FailedFetchNoCache, + vcir: None, + ccr_manifest_projection: None, + snapshot: None, + objects: empty_objects_output(), + child_audits: Vec::new(), + discovered_children: Vec::new(), + warnings: Vec::new(), + }) + .expect("failed-fetch no-cache should not append"); + + assert_eq!( + runner + .ccr_accumulator_snapshot() + .expect("ccr accumulator snapshot") + .manifest_count(), + 0 + ); +} + +#[test] +fn parse_snapshot_time_value_reports_invalid_timestamp() { + let err = parse_snapshot_time_value(&PackTime { + rfc3339_utc: "not-a-time".to_string(), + }) + .unwrap_err(); + + assert!(err.contains("invalid RFC3339 time 'not-a-time'"), "{err}"); +} + +#[test] +fn build_objects_output_from_vcir_tracks_expired_and_invalid_cached_outputs() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + + let bad_time_uri = "rsync://example.test/repo/issuer/bad-time.roa".to_string(); + let expired_uri = "rsync://example.test/repo/issuer/expired.asa".to_string(); + let bad_json_uri = "rsync://example.test/repo/issuer/bad-json.roa".to_string(); + let bad_prefix_uri = "rsync://example.test/repo/issuer/bad-prefix.roa".to_string(); + let bad_aspa_uri = "rsync://example.test/repo/issuer/bad-aspa.asa".to_string(); + + for (uri, kind) in [ + (bad_time_uri.clone(), VcirArtifactKind::Roa), + (expired_uri.clone(), VcirArtifactKind::Aspa), + (bad_json_uri.clone(), VcirArtifactKind::Roa), + (bad_prefix_uri.clone(), VcirArtifactKind::Roa), + (bad_aspa_uri.clone(), VcirArtifactKind::Aspa), + ] { + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: kind, + uri: Some(uri.clone()), + sha256: sha256_hex(uri.as_bytes()), + object_type: Some( + match kind { + VcirArtifactKind::Aspa => "aspa", + _ => "roa", + } + .to_string(), + ), + validation_status: VcirArtifactValidationStatus::Accepted, + }); + } + + vcir.local_outputs.push(VcirLocalOutput { + output_id: sha256_hex(b"bad-time"), + output_type: VcirOutputType::Vrp, + item_effective_until: PackTime { + rfc3339_utc: "bad-time-value".to_string(), + }, + source_object_uri: bad_time_uri.clone(), + source_object_type: "roa".to_string(), + source_object_hash: sha256_hex(b"bad-time-src"), + source_ee_cert_hash: sha256_hex(b"bad-time-ee"), + payload_json: + serde_json::json!({"asn": 64496, "prefix": "203.0.113.0/24", "max_length": 24}) + .to_string(), + rule_hash: sha256_hex(b"bad-time-rule"), + validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), bad_time_uri.clone()], + }); + vcir.local_outputs.push(VcirLocalOutput { + output_id: sha256_hex(b"expired"), + output_type: VcirOutputType::Aspa, + item_effective_until: PackTime::from_utc_offset_datetime(now - time::Duration::minutes(1)), + source_object_uri: expired_uri.clone(), + source_object_type: "aspa".to_string(), + source_object_hash: sha256_hex(b"expired-src"), + source_ee_cert_hash: sha256_hex(b"expired-ee"), + payload_json: serde_json::json!({"customer_as_id": 64500, "provider_as_ids": [64501]}) + .to_string(), + rule_hash: sha256_hex(b"expired-rule"), + validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), expired_uri.clone()], + }); + vcir.local_outputs.push(VcirLocalOutput { + output_id: sha256_hex(b"bad-json"), + output_type: VcirOutputType::Vrp, + item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)), + source_object_uri: bad_json_uri.clone(), + source_object_type: "roa".to_string(), + source_object_hash: sha256_hex(b"bad-json-src"), + source_ee_cert_hash: sha256_hex(b"bad-json-ee"), + payload_json: "{not-json".to_string(), + rule_hash: sha256_hex(b"bad-json-rule"), + validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), bad_json_uri.clone()], + }); + vcir.local_outputs.push(VcirLocalOutput { + output_id: sha256_hex(b"bad-prefix"), + output_type: VcirOutputType::Vrp, + item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)), + source_object_uri: bad_prefix_uri.clone(), + source_object_type: "roa".to_string(), + source_object_hash: sha256_hex(b"bad-prefix-src"), + source_ee_cert_hash: sha256_hex(b"bad-prefix-ee"), + payload_json: serde_json::json!({"asn": 64510, "prefix": "203.0.113.0", "max_length": 24}) + .to_string(), + rule_hash: sha256_hex(b"bad-prefix-rule"), + validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), bad_prefix_uri.clone()], + }); + vcir.local_outputs.push(VcirLocalOutput { + output_id: sha256_hex(b"bad-aspa"), + output_type: VcirOutputType::Aspa, + item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)), + source_object_uri: bad_aspa_uri.clone(), + source_object_type: "aspa".to_string(), + source_object_hash: sha256_hex(b"bad-aspa-src"), + source_ee_cert_hash: sha256_hex(b"bad-aspa-ee"), + payload_json: serde_json::json!({"customer_as_id": 64520}).to_string(), + rule_hash: sha256_hex(b"bad-aspa-rule"), + validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), bad_aspa_uri.clone()], + }); + + let mut warnings = Vec::new(); + let output = build_objects_output_from_vcir(&vcir, now, &mut warnings); + + assert_eq!(output.vrps.len(), 1); + assert_eq!(output.aspas.len(), 1); + assert_eq!(output.stats.roa_total, 4); + assert_eq!(output.stats.roa_ok, 1); + assert_eq!(output.stats.aspa_total, 3); + assert_eq!(output.stats.aspa_ok, 1); + assert!(warnings.iter().any(|warning| { + warning + .message + .contains("cached local output has invalid item_effective_until") + })); + assert!(warnings.iter().any(|warning| { + warning + .message + .contains("cached ROA local output parse failed") + })); + assert!(warnings.iter().any(|warning| { + warning + .message + .contains("cached ASPA local output parse failed") + })); + assert!(output.audit.iter().any(|entry| { + entry.rsync_uri == expired_uri + && matches!(entry.result, AuditObjectResult::Skipped) + && entry.detail.as_deref() == Some("skipped: cached local output expired") + })); + assert!(output.audit.iter().any(|entry| { + entry.rsync_uri == bad_time_uri && matches!(entry.result, AuditObjectResult::Error) + })); + assert!(output.audit.iter().any(|entry| { + entry.rsync_uri == bad_prefix_uri + && matches!(entry.result, AuditObjectResult::Error) + && entry + .detail + .as_deref() + .unwrap_or("") + .contains("cached ROA local output parse failed") + })); +} + +#[test] +fn build_publication_point_audit_from_vcir_uses_vcir_metadata_and_overlays_child_and_object_audits() +{ + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + vcir.related_artifacts + .retain(|artifact| artifact.artifact_role != VcirArtifactRole::Manifest); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: Vec::new(), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + }; + let runner_warnings = vec![Warning::new("runner warning")]; + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: vec![Warning::new("objects warning")], + stats: crate::validation::objects::ObjectsStats::default(), + audit: vec![ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), + sha256_hex: sha256_hex(b"override-roa"), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some("overridden from object audit".to_string()), + }], + }; + let child_audits = vec![ObjectAuditEntry { + rsync_uri: vcir.child_entries[0].child_cert_rsync_uri.clone(), + sha256_hex: vcir.child_entries[0].child_cert_hash.clone(), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Ok, + detail: Some("restored child CA instance from VCIR".to_string()), + }]; + + let audit = build_publication_point_audit_from_vcir( + &ca, + PublicationPointSource::VcirCurrentInstance, + Some("rsync"), + Some("rrdp_failed_rsync_failed"), + Some(456), + Some("rsync failed"), + Some(&vcir), + None, + &runner_warnings, + &objects, + &child_audits, + ); + + assert_eq!(audit.source, "vcir_current_instance"); + assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync")); + assert_eq!( + audit.repo_sync_phase.as_deref(), + Some("rrdp_failed_rsync_failed") + ); + assert_eq!(audit.repo_sync_duration_ms, Some(456)); + assert_eq!(audit.repo_sync_error.as_deref(), Some("rsync failed")); + assert_eq!(audit.repo_terminal_state, "fallback_current_instance"); + assert_eq!(audit.objects[0].rsync_uri, vcir.current_manifest_rsync_uri); + assert_eq!(audit.objects[0].kind, AuditObjectKind::Manifest); + assert_eq!( + audit.this_update_rfc3339_utc, + vcir.validated_manifest_meta + .validated_manifest_this_update + .rfc3339_utc + ); + assert_eq!( + audit.next_update_rfc3339_utc, + vcir.validated_manifest_meta + .validated_manifest_next_update + .rfc3339_utc + ); + assert_eq!( + audit.verified_at_rfc3339_utc, + vcir.last_successful_validation_time.rfc3339_utc + ); + assert_eq!(audit.warnings.len(), 2); + assert!(audit.objects.iter().any(|entry| { + entry.rsync_uri == "rsync://example.test/repo/issuer/a.roa" + && matches!(entry.result, AuditObjectResult::Error) + && entry.detail.as_deref() == Some("overridden from object audit") + })); + assert!(audit.objects.iter().any(|entry| { + entry.rsync_uri == vcir.child_entries[0].child_cert_rsync_uri + && matches!(entry.result, AuditObjectResult::Ok) + })); +} + +#[test] +fn build_publication_point_audit_from_vcir_failed_no_cache_keeps_current_reject_only() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let vcir = sample_vcir_for_projection(now, &child_cert_hash); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: Vec::new(), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + }; + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: vec![ObjectAuditEntry { + rsync_uri: vcir.current_manifest_rsync_uri.clone(), + sha256_hex: sha256_hex(b"current-manifest"), + kind: AuditObjectKind::Manifest, + result: AuditObjectResult::Error, + detail: Some("manifest is not valid at validation_time".to_string()), + }], + }; + + let audit = build_publication_point_audit_from_vcir( + &ca, + PublicationPointSource::FailedFetchNoCache, + Some("rsync"), + Some("rsync_only_ok"), + Some(123), + None, + Some(&vcir), + None, + &[Warning::new("latest VCIR instance_gate expired")], + &objects, + &[], + ); + + assert_eq!(audit.source, "failed_fetch_no_cache"); + assert_eq!(audit.repo_terminal_state, "failed_no_cache"); + assert_eq!( + audit.this_update_rfc3339_utc, + vcir.validated_manifest_meta + .validated_manifest_this_update + .rfc3339_utc + ); + assert_eq!(audit.objects.len(), 1); + assert_eq!(audit.objects[0].rsync_uri, vcir.current_manifest_rsync_uri); + assert!(matches!(audit.objects[0].result, AuditObjectResult::Error)); + assert!( + !audit + .objects + .iter() + .any(|entry| entry.rsync_uri == "rsync://example.test/repo/issuer/a.roa"), + "failed-no-cache must not expand old VCIR related artifacts into current-run audit", + ); + assert!( + !audit + .objects + .iter() + .any(|entry| entry.rsync_uri == "rsync://example.test/repo/issuer/issuer.crl"), + "failed-no-cache must not expose old CRL as current-run CIR input", + ); +} + +#[test] +fn rejected_manifest_audit_entry_for_failed_fetch_uses_current_repo_hash() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let runner = sample_runner_with_ccr_accumulator(&store, &policy); + let manifest_uri = "rsync://example.test/repo/issuer/issuer.mft"; + let manifest_hash = sha256_hex(b"manifest-bytes"); + store + .put_blob_bytes_batch(&[(manifest_hash.clone(), b"manifest-bytes".to_vec())]) + .expect("put manifest bytes"); + store + .put_repository_view_entry(&crate::storage::RepositoryViewEntry { + rsync_uri: manifest_uri.to_string(), + current_hash: Some(manifest_hash.clone()), + repository_source: Some("rsync://example.test/repo/issuer/".to_string()), + object_type: Some("mft".to_string()), + state: crate::storage::RepositoryViewState::Present, + }) + .expect("put repository view"); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: Vec::new(), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: manifest_uri.to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let entry = runner + .rejected_manifest_audit_entry_for_failed_fetch( + &ca, + &ManifestFreshError::StaleOrEarly { + this_update_rfc3339_utc: "2026-05-27T08:37:07Z".to_string(), + next_update_rfc3339_utc: "2026-05-28T10:01:07Z".to_string(), + validation_time_rfc3339_utc: "2026-05-28T10:11:00Z".to_string(), + }, + ) + .expect("rejected manifest audit entry"); + + assert_eq!(entry.rsync_uri, manifest_uri); + assert_eq!(entry.sha256_hex, manifest_hash); + assert_eq!(entry.kind, AuditObjectKind::Manifest); + assert_eq!(entry.result, AuditObjectResult::Error); + assert!( + entry + .detail + .as_deref() + .unwrap_or("") + .contains("manifest is not valid at validation_time") + ); +} + +#[test] +fn build_publication_point_audit_from_vcir_without_cached_inputs_returns_empty_listing() { + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: Vec::new(), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let audit = build_publication_point_audit_from_vcir( + &ca, + PublicationPointSource::FailedFetchNoCache, + Some("rsync"), + Some("rsync_only_failed"), + Some(789), + Some("load from network failed, fallback to cache"), + None, + None, + &[Warning::new("runner warning")], + &crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: vec![Warning::new("object warning")], + stats: crate::validation::objects::ObjectsStats::default(), + audit: Vec::new(), + }, + &[], + ); + + assert_eq!(audit.source, "failed_fetch_no_cache"); + assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync")); + assert_eq!(audit.repo_sync_phase.as_deref(), Some("rsync_only_failed")); + assert_eq!(audit.repo_sync_duration_ms, Some(789)); + assert_eq!( + audit.repo_sync_error.as_deref(), + Some("load from network failed, fallback to cache") + ); + assert_eq!(audit.repo_terminal_state, "failed_no_cache"); + assert!(audit.this_update_rfc3339_utc.is_empty()); + assert!(audit.next_update_rfc3339_utc.is_empty()); + assert!(audit.verified_at_rfc3339_utc.is_empty()); + assert_eq!(audit.warnings.len(), 2); + assert!(audit.objects.is_empty()); +} + +#[test] +fn effective_repo_sync_duration_uses_runtime_duration_for_failures() { + assert_eq!(effective_repo_sync_duration_ms(0, Some(12), false), 12); + assert_eq!(effective_repo_sync_duration_ms(5, Some(12), false), 12); + assert_eq!(effective_repo_sync_duration_ms(20, Some(12), false), 20); + assert_eq!(effective_repo_sync_duration_ms(5, None, false), 5); + assert_eq!(effective_repo_sync_duration_ms(5, Some(12), true), 5); +} + +#[test] +fn reconstruct_snapshot_from_vcir_reports_missing_manifest_and_related_raw_bytes() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + let dup_uri = "rsync://example.test/repo/issuer/dup.roa".to_string(); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some(dup_uri.clone()), + sha256: sha256_hex(b"dup-roa-1"), + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + }); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some(dup_uri.clone()), + sha256: sha256_hex(b"dup-roa-2"), + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + }); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::IssuerCert, + artifact_kind: VcirArtifactKind::Cer, + uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + sha256: sha256_hex(b"issuer-cert"), + object_type: Some("cer".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + }); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: Vec::new(), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let mut warnings = Vec::new(); + assert!(reconstruct_snapshot_from_vcir(&store, &ca, &vcir, &mut warnings).is_none()); + assert!(warnings.iter().any(|warning| { + warning + .message + .contains("manifest raw bytes missing for VCIR audit reconstruction") + })); + + let manifest_bytes = b"manifest-bytes".to_vec(); + let current_crl_bytes = b"current-crl-bytes".to_vec(); + let child_bytes = b"child-cert".to_vec(); + let roa_bytes = b"roa-bytes".to_vec(); + for (bytes, uri, object_type) in [ + ( + manifest_bytes.clone(), + Some(vcir.manifest_rsync_uri.clone()), + Some("mft".to_string()), + ), + ( + current_crl_bytes, + Some(vcir.current_crl_rsync_uri.clone()), + Some("crl".to_string()), + ), + ( + child_bytes, + Some(vcir.child_entries[0].child_cert_rsync_uri.clone()), + Some("cer".to_string()), + ), + ( + roa_bytes, + Some("rsync://example.test/repo/issuer/a.roa".to_string()), + Some("roa".to_string()), + ), + ] { + let mut entry = RawByHashEntry::from_bytes(sha256_hex(&bytes), bytes); + if let Some(uri) = uri { + entry.origin_uris.push(uri); + } + entry.object_type = object_type; + entry.encoding = Some("der".to_string()); + store.put_raw_by_hash_entry(&entry).expect("put raw entry"); + } + + warnings.clear(); + let pack = reconstruct_snapshot_from_vcir(&store, &ca, &vcir, &mut warnings) + .expect("reconstruct pack with partial related artifacts"); + assert_eq!(pack.manifest_bytes, manifest_bytes); + assert_eq!(pack.files.len(), 3, "crl + child cert + roa only"); + assert!( + pack.files + .iter() + .any(|file| file.rsync_uri.ends_with("issuer.crl")) + ); + assert!( + pack.files + .iter() + .any(|file| file.rsync_uri.ends_with("child.cer")) + ); + assert!( + pack.files + .iter() + .any(|file| file.rsync_uri.ends_with("a.roa")) + ); + assert!( + !pack + .files + .iter() + .any(|file| file.rsync_uri.ends_with("issuer.cer")) + ); + assert!(warnings.iter().any(|warning| { + warning + .message + .contains("related artifact raw bytes missing for VCIR audit reconstruction") + })); +} + +#[test] +fn reconstruct_snapshot_from_vcir_reads_repo_bytes_without_raw_entries() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let vcir = sample_vcir_for_projection(now, &child_cert_hash); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: Vec::new(), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let store_dir = tempfile::tempdir().expect("store dir"); + let main_db = store_dir.path().join("work-db"); + let repo_bytes_db = store_dir.path().join("repo-bytes.db"); + let store = RocksStore::open_with_external_repo_bytes(&main_db, &repo_bytes_db) + .expect("open rocksdb with external repo bytes"); + let repo_blobs = [ + b"manifest-bytes".to_vec(), + b"current-crl-bytes".to_vec(), + b"child-cert".to_vec(), + b"roa-bytes".to_vec(), + b"aspa-bytes".to_vec(), + ] + .into_iter() + .map(|bytes| (sha256_hex(&bytes), bytes)) + .collect::>(); + store + .put_blob_bytes_batch(&repo_blobs) + .expect("put external repo bytes"); + + let manifest_hash = vcir + .related_artifacts + .iter() + .find(|artifact| artifact.artifact_role == VcirArtifactRole::Manifest) + .expect("manifest artifact") + .sha256 + .clone(); + assert!( + store + .get_raw_by_hash_entry(&manifest_hash) + .expect("raw manifest lookup") + .is_none(), + "repo object bytes must not require raw_by_hash entries" + ); + + let mut warnings = Vec::new(); + let pack = reconstruct_snapshot_from_vcir(&store, &ca, &vcir, &mut warnings) + .expect("reconstruct pack from external repo bytes"); + assert_eq!(pack.manifest_bytes, b"manifest-bytes".to_vec()); + assert_eq!(pack.files.len(), 4, "crl + child cert + roa + aspa"); + assert!( + warnings.iter().all(|warning| { + !warning + .message + .contains("raw bytes missing for VCIR audit reconstruction") + }), + "external repo bytes should satisfy VCIR audit reconstruction without raw warnings" + ); +} + +#[test] +fn runner_dedup_paths_execute_with_timing_enabled() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: issuer_ca_der, + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri.clone(), + rrdp_notification_uri: Some("https://example.test/notification.xml".to_string()), + }; + let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-03-11T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-03-11T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let policy_rrdp = Policy::default(); + let runner_rrdp = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy_rrdp, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: Some(timing.clone()), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + }; + let first = runner_rrdp + .run_publication_point(&handle) + .expect("rrdp fallback to rsync"); + assert_eq!(first.source, PublicationPointSource::Fresh); + let second = runner_rrdp + .run_publication_point(&handle) + .expect("rrdp dedup skip"); + assert_eq!(second.source, PublicationPointSource::Fresh); + + let policy_rsync = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + let runner_rsync = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy_rsync, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: Some(timing), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + }; + let third = runner_rsync + .run_publication_point(&handle) + .expect("rsync first run"); + assert_eq!(third.source, PublicationPointSource::Fresh); + let fourth = runner_rsync + .run_publication_point(&handle) + .expect("rsync dedup run"); + assert_eq!(fourth.source, PublicationPointSource::Fresh); + assert_eq!( + crate::fetch::rsync::normalize_rsync_base_uri("rsync://example.test/repo"), + "rsync://example.test/repo/" + ); +} diff --git a/src/validation/tree_runner/vcir_der.rs b/src/validation/tree_runner/vcir_der.rs new file mode 100644 index 0000000..3db7d69 --- /dev/null +++ b/src/validation/tree_runner/vcir_der.rs @@ -0,0 +1,78 @@ +use crate::data_model::rc::AccessDescription; + +pub(super) fn encode_access_description_der_for_vcir_ccr_projection( + access_description: &AccessDescription, +) -> Result, String> { + let oid = encode_oid_der_for_vcir_ccr_projection(&access_description.access_method_oid)?; + let uri = encode_tlv_for_vcir_ccr_projection( + 0x86, + access_description.access_location.as_bytes().to_vec(), + ); + Ok(encode_sequence_for_vcir_ccr_projection(&[oid, uri])) +} + +fn encode_oid_der_for_vcir_ccr_projection(oid: &str) -> Result, String> { + let arcs = oid + .split('.') + .map(|part| { + part.parse::() + .map_err(|_| format!("unsupported accessMethod OID: {oid}")) + }) + .collect::, _>>()?; + if arcs.len() < 2 { + return Err(format!("unsupported accessMethod OID: {oid}")); + } + if arcs[0] > 2 || (arcs[0] < 2 && arcs[1] >= 40) { + return Err(format!("unsupported accessMethod OID: {oid}")); + } + let mut body = Vec::new(); + body.push((arcs[0] * 40 + arcs[1]) as u8); + for arc in &arcs[2..] { + encode_base128_for_vcir_ccr_projection(*arc, &mut body); + } + Ok(encode_tlv_for_vcir_ccr_projection(0x06, body)) +} + +fn encode_base128_for_vcir_ccr_projection(mut value: u64, out: &mut Vec) { + let mut tmp = vec![(value & 0x7F) as u8]; + value >>= 7; + while value > 0 { + tmp.push(((value & 0x7F) as u8) | 0x80); + value >>= 7; + } + tmp.reverse(); + out.extend_from_slice(&tmp); +} + +fn encode_sequence_for_vcir_ccr_projection(elements: &[Vec]) -> Vec { + let total_len: usize = elements.iter().map(Vec::len).sum(); + let mut buf = Vec::with_capacity(total_len); + for element in elements { + buf.extend_from_slice(element); + } + encode_tlv_for_vcir_ccr_projection(0x30, buf) +} + +fn encode_tlv_for_vcir_ccr_projection(tag: u8, value: Vec) -> Vec { + let mut out = Vec::with_capacity(1 + 9 + value.len()); + out.push(tag); + encode_length_for_vcir_ccr_projection(value.len(), &mut out); + out.extend_from_slice(&value); + out +} + +fn encode_length_for_vcir_ccr_projection(len: usize, out: &mut Vec) { + if len < 0x80 { + out.push(len as u8); + return; + } + let mut bytes = Vec::new(); + let mut value = len; + while value > 0 { + bytes.push((value & 0xFF) as u8); + value >>= 8; + } + bytes.reverse(); + out.push(0x80 | (bytes.len() as u8)); + out.extend_from_slice(&bytes); +}