From 4045f9a3a5e86a38cfd725e92e9f9d7656e14fbb Mon Sep 17 00:00:00 2001 From: yuyr Date: Sat, 6 Jun 2026 15:16:38 +0800 Subject: [PATCH] =?UTF-8?q?20260605=20ROA=E5=A2=9E=E9=87=8F=E9=AA=8C?= =?UTF-8?q?=E8=AF=81=E5=92=8Crsync=20host=E5=A4=B1=E8=B4=A5=E7=9F=AD?= =?UTF-8?q?=E8=B7=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/coverage.sh | 2 +- scripts/soak/portable-soak.env.example | 16 +- scripts/soak/run_soak.sh | 16 +- src/analysis/timing.rs | 8 + src/cli.rs | 183 ++- src/cli/tests.rs | 83 ++ src/fetch/rsync.rs | 10 + src/fetch/rsync_system.rs | 72 +- src/parallel/repo_runtime.rs | 35 +- src/parallel/repo_scheduler.rs | 878 +++++++++++++- src/parallel/repo_worker.rs | 7 + src/parallel/run_coordinator.rs | 11 + src/parallel/types.rs | 5 + src/storage.rs | 12 +- src/validation/objects.rs | 1160 ++++++++++++++++++- src/validation/run.rs | 1 + src/validation/run_tree_from_tal.rs | 220 +++- src/validation/tree.rs | 12 +- src/validation/tree_parallel.rs | 36 +- src/validation/tree_runner.rs | 139 ++- src/validation/tree_runner/tests.rs | 139 ++- tests/test_apnic_stats_live_stage2.rs | 2 + tests/test_apnic_tree_live_m15.rs | 2 + tests/test_deterministic_semantics_m4.rs | 1 + tests/test_run_tree_from_tal_offline_m17.rs | 6 + tests/test_tree_failure_handling.rs | 2 + tests/test_tree_traversal_m14.rs | 13 + 27 files changed, 2850 insertions(+), 221 deletions(-) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 25f9c00..51d0db3 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -27,7 +27,7 @@ cleanup() { } trap cleanup EXIT -IGNORE_REGEX='src/bin/repository_view_stats\.rs|src/bin/db_stats\.rs|src/bin/rrdp_state_dump\.rs|src/bin/ccr_dump\.rs|src/bin/ccr_verify\.rs|src/bin/ccr_to_routinator_csv\.rs|src/bin/ccr_to_compare_views\.rs|src/bin/cir_materialize\.rs|src/bin/cir_extract_inputs\.rs|src/bin/cir_drop_report\.rs|src/bin/cir_ta_only_fixture\.rs|src/bin/cir_dump_reject_list\.rs|src/bin/rpki_object_parse\.rs|src/bin/triage_ccr_cir_pair\.rs|src/bin/rpki_artifact_metrics\.rs|src/bin/rpki_daemon\.rs|src/bin/sequence_triage_ccr_cir\.rs|src/tools/rpki_artifact_metrics\.rs|src/ccr/compare_view\.rs|src/progress_log\.rs|src/cli\.rs|src/validation/run_tree_from_tal\.rs|src/validation/tree_parallel\.rs|src/validation/tree_runner\.rs|src/validation/from_tal\.rs|src/sync/store_projection\.rs|src/sync/repo\.rs|src/sync/rrdp\.rs|src/storage\.rs|src/cir/materialize\.rs' +IGNORE_REGEX='repository_view_stats\.rs|db_stats\.rs|rrdp_state_dump\.rs|ccr_dump\.rs|ccr_verify\.rs|ccr_to_routinator_csv\.rs|ccr_to_compare_views\.rs|cir_materialize\.rs|cir_extract_inputs\.rs|cir_drop_report\.rs|cir_ta_only_fixture\.rs|cir_dump_reject_list\.rs|rpki_object_parse\.rs|triage_ccr_cir_pair\.rs|rpki_artifact_metrics|rpki_daemon\.rs|sequence_triage_ccr_cir|ccr_state_compare\.rs|cir_state_compare\.rs|cir_probe_rpki_client_cache\.rs|ccr/compare_view\.rs|progress_log\.rs|cli\.rs|validation/run_tree_from_tal\.rs|validation/tree_parallel\.rs|validation/tree_runner|validation/from_tal\.rs|sync/store_projection\.rs|sync/repo\.rs|sync/rrdp|(^|/)storage(/|\.rs$)|cir/materialize\.rs' # Preserve colored output even though we post-process output by running under a pseudo-TTY. # We run tests only once, then generate both CLI text + HTML reports without rerunning tests. diff --git a/scripts/soak/portable-soak.env.example b/scripts/soak/portable-soak.env.example index bc388f9..e54f36a 100644 --- a/scripts/soak/portable-soak.env.example +++ b/scripts/soak/portable-soak.env.example @@ -25,9 +25,11 @@ OUTPUT_COMPACT_REPORT=1 # 是否复用持久 rsync mirror。1 表示跨 run 复用；失败隔离数据库时也不会清理 mirror。 ALLOW_RSYNC_MIRROR_REUSE=1 -# rsync 同步 scope。 -# publication-point 表示默认只同步当前发布点；module-root 表示扩大到 rsync module 根目录。 -RSYNC_SCOPE=publication-point +# rsync 同步/去重 scope。 +# host 表示按 rsync host 做失败短路，但实际拉取仍限定当前发布点，避免同一不可达 host 重复等待超时； +# publication-point 表示只按当前发布点去重； +# module-root 表示扩大实际拉取到 rsync module 根目录。 +RSYNC_SCOPE=host # 前一轮失败或不完整时，是否隔离旧数据库和运行态目录后强制下一轮 snapshot。 # 建议保持 1；设置为 0 时，检测到前一轮失败会直接停止。 @@ -45,6 +47,14 @@ RPKI_PROGRESS_SLOW_SECS=10 # 是否在运行前尝试禁用 rpki-client timer 并杀掉竞争 RP 进程。 DISABLE_COMPETING_RPS=1 +# 传给 rpki 子进程的额外参数。多个参数用空格分隔。 +# 示例：RPKI_EXTRA_ARGS="--enable-roa-validation-cache" +RPKI_EXTRA_ARGS="" + +# 是否为每轮输出 timing profile 到 runs/run_xxxx/analyze/timing.json。 +# 性能 profile 或打点验证时设置为 1；普通 soak 建议保持 0，避免额外开销。 +RPKI_ANALYZE=0 + # 可选覆盖路径；默认由 package 自动推导。 # BIN_DIR="${PACKAGE_ROOT}/bin" # FIXTURE_DIR="${PACKAGE_ROOT}/fixtures" diff --git a/scripts/soak/run_soak.sh b/scripts/soak/run_soak.sh index df771bb..658ce14 100755 --- a/scripts/soak/run_soak.sh +++ b/scripts/soak/run_soak.sh @@ -17,12 +17,14 @@ RUN_ROOT="${RUN_ROOT:-$PACKAGE_ROOT}" RETAIN_RUNS="${RETAIN_RUNS:-10}" OUTPUT_COMPACT_REPORT="${OUTPUT_COMPACT_REPORT:-1}" ALLOW_RSYNC_MIRROR_REUSE="${ALLOW_RSYNC_MIRROR_REUSE:-1}" -RSYNC_SCOPE="${RSYNC_SCOPE:-publication-point}" +RSYNC_SCOPE="${RSYNC_SCOPE:-host}" FAILURE_SNAPSHOT_RESET="${FAILURE_SNAPSHOT_RESET:-1}" DB_STATS_EXACT_EVERY="${DB_STATS_EXACT_EVERY:-3}" RPKI_PROGRESS_LOG="${RPKI_PROGRESS_LOG:-1}" RPKI_PROGRESS_SLOW_SECS="${RPKI_PROGRESS_SLOW_SECS:-10}" DISABLE_COMPETING_RPS="${DISABLE_COMPETING_RPS:-1}" +RPKI_EXTRA_ARGS="${RPKI_EXTRA_ARGS:-}" +RPKI_ANALYZE="${RPKI_ANALYZE:-0}" BIN_DIR="${BIN_DIR:-$PACKAGE_ROOT/bin}" FIXTURE_DIR="${FIXTURE_DIR:-$PACKAGE_ROOT/fixtures}" @@ -84,10 +86,10 @@ validate_max_runs() { validate_rsync_scope() { case "$RSYNC_SCOPE" in - publication-point|module-root) + host|publication-point|module-root) ;; *) - die "RSYNC_SCOPE must be publication-point or module-root: $RSYNC_SCOPE" + die "RSYNC_SCOPE must be host, publication-point, or module-root: $RSYNC_SCOPE" ;; esac } @@ -394,6 +396,11 @@ build_child_args() { --vaps-csv-out "{run_out}/vaps.csv" --compare-view-trust-anchor "$(compare_view_trust_anchor)" ) + if [[ -n "$RPKI_EXTRA_ARGS" ]]; then + # shellcheck disable=SC2206 + local extra_args=( $RPKI_EXTRA_ARGS ) + CHILD_ARGS+=("${extra_args[@]}") + fi } copy_inner_run_outputs() { @@ -541,6 +548,9 @@ run_one_round() { "$INVALID_DB_PATH" "$INVALID_STATE_PATH" "$INVALID_TMP_PATH" "" "$PACKAGE_ROOT" "$ENV_FILE" build_child_args + if is_true "$RPKI_ANALYZE"; then + CHILD_ARGS+=(--analysis-out "$run_dir/analyze") + fi local daemon_args=( --state-root "$daemon_state_root" --rpki-bin "$RPKI_BIN" diff --git a/src/analysis/timing.rs b/src/analysis/timing.rs index 6125bb5..a158f4a 100644 --- a/src/analysis/timing.rs +++ b/src/analysis/timing.rs @@ -71,6 +71,14 @@ impl TimingHandle { .or_insert(inc); } + pub fn counts_snapshot(&self) -> HashMap { + let g = self.inner.lock().expect("timing lock"); + g.counts + .iter() + .map(|(key, value)| ((*key).to_string(), *value)) + .collect() + } + /// Record a phase duration directly in nanoseconds. /// /// This is useful when aggregating sub-phase timings locally (to reduce lock contention) diff --git a/src/cli.rs b/src/cli.rs index f66e344..e80afc1 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -26,12 +26,15 @@ use crate::policy::{Policy, StrictPolicy}; use crate::storage::{RocksStore, VcirStorageSummary}; use crate::validation::run_tree_from_tal::{ RunTreeFromTalAuditOutput, run_tree_from_multiple_tals_parallel_phase2_audit, + run_tree_from_multiple_tals_parallel_phase2_audit_with_timing, run_tree_from_tal_and_ta_der_parallel_phase2_audit, + run_tree_from_tal_and_ta_der_parallel_phase2_audit_with_timing, run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit, run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit_with_timing, run_tree_from_tal_and_ta_der_payload_replay_serial_audit, run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing, run_tree_from_tal_url_parallel_phase2_audit, + run_tree_from_tal_url_parallel_phase2_audit_with_timing, }; use crate::validation::tree::TreeRunConfig; #[cfg(test)] @@ -40,11 +43,13 @@ use output::{ ReportJsonFormat, run_compare_view_task, write_report_json_from_shared, write_stage_timing, }; use serde::Serialize; +use std::collections::HashMap; use std::sync::Arc; #[derive(Clone, Debug, PartialEq, Eq, Serialize)] struct RunStageTiming { validation_ms: u64, + enable_roa_validation_cache: bool, report_build_ms: u64, report_write_ms: Option, ccr_build_ms: Option, @@ -63,6 +68,8 @@ struct RunStageTiming { rrdp_download_ms_total: u64, rsync_download_ms_total: u64, download_bytes_total: u64, + roa_validation_cache: crate::validation::objects::RoaValidationCacheStats, + analysis_counts: HashMap, vcir_storage_summary_ms: Option, vcir_storage: Option, memory_telemetry: Option, @@ -115,6 +122,7 @@ pub struct CliArgs { pub report_json_compact: bool, pub skip_report_build: bool, pub skip_vcir_persist: bool, + pub enable_roa_validation_cache: bool, pub ccr_out_path: Option, pub vrps_csv_out_path: Option, pub vaps_csv_out_path: Option, @@ -147,6 +155,7 @@ pub struct CliArgs { pub validation_time: Option, pub analyze: bool, + pub analysis_out_path: Option, pub profile_cpu: bool, } @@ -168,6 +177,8 @@ Options: --report-json-compact Write report JSON without pretty-printing (requires --report-json) --skip-report-build Skip full audit report construction when --report-json is not requested --skip-vcir-persist Skip VCIR persistence/projection building for compare-only runs + --enable-roa-validation-cache + Reuse accepted ROA validation outputs from previous VCIR records (default: off) --ccr-out Write CCR DER ContentInfo to this path (optional) --vrps-csv-out Write VRP compare-view CSV directly from validation output (optional; requires --vaps-csv-out) --vaps-csv-out Write VAP compare-view CSV directly from validation output (optional; requires --vrps-csv-out) @@ -218,11 +229,12 @@ Options: --http-timeout-secs HTTP fetch timeout seconds (default: 20) --rsync-timeout-secs rsync I/O timeout seconds (default: 60) --rsync-mirror-root Persist rsync mirrors under this directory (default: disabled) - --rsync-scope rsync scope policy: publication-point or module-root (default: publication-point) + --rsync-scope rsync scope policy: host, publication-point, or module-root (default: host) --max-depth Max CA instance depth (0 = root only) --max-instances Max number of CA instances to process --validation-time Validation time in RFC3339 (default: now UTC) --analyze Write timing analysis JSON under target/live/analyze// + --analysis-out Write timing analysis JSON under this directory (implies --analyze) --profile-cpu (Requires build feature 'profile') Write CPU flamegraph under analyze dir --help Show this help @@ -246,6 +258,7 @@ pub fn parse_args(argv: &[String]) -> Result { let mut report_json_compact: bool = false; let mut skip_report_build: bool = false; let mut skip_vcir_persist: bool = false; + let mut enable_roa_validation_cache: bool = false; let mut ccr_out_path: Option = None; let mut vrps_csv_out_path: Option = None; let mut vaps_csv_out_path: Option = None; @@ -275,6 +288,7 @@ pub fn parse_args(argv: &[String]) -> Result { let mut max_instances: Option = None; let mut validation_time: Option = None; let mut analyze: bool = false; + let mut analysis_out_path: Option = None; let mut profile_cpu: bool = false; let mut i = 1usize; @@ -447,6 +461,9 @@ pub fn parse_args(argv: &[String]) -> Result { "--skip-vcir-persist" => { skip_vcir_persist = true; } + "--enable-roa-validation-cache" => { + enable_roa_validation_cache = true; + } "--ccr-out" => { i += 1; let v = argv.get(i).ok_or("--ccr-out requires a value")?; @@ -606,6 +623,12 @@ pub fn parse_args(argv: &[String]) -> Result { "--analyze" => { analyze = true; } + "--analysis-out" => { + i += 1; + let v = argv.get(i).ok_or("--analysis-out requires a value")?; + analyze = true; + analysis_out_path = Some(PathBuf::from(v)); + } "--profile-cpu" => { profile_cpu = true; } @@ -870,6 +893,7 @@ pub fn parse_args(argv: &[String]) -> Result { report_json_compact, skip_report_build, skip_vcir_persist, + enable_roa_validation_cache, ccr_out_path, vrps_csv_out_path, vaps_csv_out_path, @@ -898,6 +922,7 @@ pub fn parse_args(argv: &[String]) -> Result { max_instances, validation_time, analyze, + analysis_out_path, profile_cpu, }) } @@ -1004,6 +1029,7 @@ struct PostValidationShared { aspas: Arc<[crate::validation::objects::AspaAttestation]>, router_keys: Arc<[crate::validation::objects::RouterKeyPayload]>, publication_points: Arc<[crate::audit::PublicationPointAudit]>, + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats, downloads: Arc<[crate::audit::AuditDownloadEvent]>, download_stats: crate::audit::AuditDownloadStats, current_repo_objects: Arc<[crate::current_repo_index::CurrentRepoObject]>, @@ -1018,6 +1044,7 @@ impl PostValidationShared { successful_tal_inputs, tree, publication_points, + roa_cache_stats, downloads, download_stats, current_repo_objects, @@ -1043,6 +1070,7 @@ impl PostValidationShared { aspas: aspas.into(), router_keys: router_keys.into(), publication_points: publication_points.into(), + roa_cache_stats, downloads: downloads.into(), download_stats, current_repo_objects: current_repo_objects.into(), @@ -1752,50 +1780,25 @@ where R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, { if args.tal_inputs.len() > 1 { - return run_tree_from_multiple_tals_parallel_phase2_audit( - store, - policy, - args.tal_inputs.clone(), - http, - rsync, - validation_time, - config, - args.parallel_phase1_config.clone(), - args.parallel_phase2_config.clone(), - collect_current_repo_objects, - ) - .map_err(|e| e.to_string()); - } - - match ( - args.tal_url.as_ref(), - args.tal_path.as_ref(), - args.ta_path.as_ref(), - ) { - (Some(url), _, _) => run_tree_from_tal_url_parallel_phase2_audit( - store, - policy, - url, - http, - rsync, - validation_time, - config, - args.parallel_phase1_config.clone(), - args.parallel_phase2_config.clone(), - collect_current_repo_objects, - ) - .map_err(|e| e.to_string()), - (None, Some(tal_path), Some(ta_path)) => { - let tal_bytes = std::fs::read(tal_path) - .map_err(|e| format!("read tal failed: {}: {e}", tal_path.display()))?; - let ta_der = std::fs::read(ta_path) - .map_err(|e| format!("read ta failed: {}: {e}", ta_path.display()))?; - run_tree_from_tal_and_ta_der_parallel_phase2_audit( + return if let Some(t) = timing { + run_tree_from_multiple_tals_parallel_phase2_audit_with_timing( store, policy, - &tal_bytes, - &ta_der, - None, + args.tal_inputs.clone(), + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + t, + ) + } else { + run_tree_from_multiple_tals_parallel_phase2_audit( + store, + policy, + args.tal_inputs.clone(), http, rsync, validation_time, @@ -1804,6 +1807,81 @@ where args.parallel_phase2_config.clone(), collect_current_repo_objects, ) + } + .map_err(|e| e.to_string()); + } + + match ( + args.tal_url.as_ref(), + args.tal_path.as_ref(), + args.ta_path.as_ref(), + ) { + (Some(url), _, _) => if let Some(t) = timing { + run_tree_from_tal_url_parallel_phase2_audit_with_timing( + store, + policy, + url, + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + t, + ) + } else { + run_tree_from_tal_url_parallel_phase2_audit( + store, + policy, + url, + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + ) + } + .map_err(|e| e.to_string()), + (None, Some(tal_path), Some(ta_path)) => { + let tal_bytes = std::fs::read(tal_path) + .map_err(|e| format!("read tal failed: {}: {e}", tal_path.display()))?; + let ta_der = std::fs::read(ta_path) + .map_err(|e| format!("read ta failed: {}: {e}", ta_path.display()))?; + if let Some(t) = timing { + run_tree_from_tal_and_ta_der_parallel_phase2_audit_with_timing( + store, + policy, + &tal_bytes, + &ta_der, + None, + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + t, + ) + } else { + run_tree_from_tal_and_ta_der_parallel_phase2_audit( + store, + policy, + &tal_bytes, + &ta_der, + None, + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + ) + } .map_err(|e| e.to_string()) } (None, Some(tal_path), None) => { @@ -1875,6 +1953,7 @@ pub fn run(argv: &[String]) -> Result<(), String> { && !args.cir_enabled, persist_vcir: !args.skip_vcir_persist, build_ccr_accumulator: args.ccr_out_path.is_some(), + enable_roa_validation_cache: args.enable_roa_validation_cache, }; let replay_mode = args.payload_replay_archive.is_some(); let delta_replay_mode = args.payload_base_archive.is_some(); @@ -1899,11 +1978,13 @@ pub fn run(argv: &[String]) -> Result<(), String> { .map_err(|e| format!("format timestamp failed: {e}"))? }; - let out_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("target") - .join("live") - .join("analyze") - .join(ts_compact); + let out_dir = args.analysis_out_path.clone().unwrap_or_else(|| { + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("target") + .join("live") + .join("analyze") + .join(ts_compact) + }); std::fs::create_dir_all(&out_dir) .map_err(|e| format!("create analyze out dir failed: {}: {e}", out_dir.display()))?; @@ -2344,6 +2425,7 @@ pub fn run(argv: &[String]) -> Result<(), String> { ); let stage_timing = RunStageTiming { validation_ms, + enable_roa_validation_cache: args.enable_roa_validation_cache, report_build_ms, report_write_ms, ccr_build_ms, @@ -2362,6 +2444,11 @@ pub fn run(argv: &[String]) -> Result<(), String> { rrdp_download_ms_total, rsync_download_ms_total, download_bytes_total, + roa_validation_cache: shared.roa_cache_stats.clone(), + analysis_counts: timing + .as_ref() + .map(|(_, handle)| handle.counts_snapshot()) + .unwrap_or_default(), vcir_storage_summary_ms, vcir_storage, memory_telemetry: Some(MemoryTelemetrySummary { diff --git a/src/cli/tests.rs b/src/cli/tests.rs index 5ae734d..5a8a20e 100644 --- a/src/cli/tests.rs +++ b/src/cli/tests.rs @@ -18,6 +18,7 @@ fn parse_help_returns_usage() { assert!(err.contains("--rsync-scope"), "{err}"); assert!(err.contains("--parallel-phase2-object-workers"), "{err}"); assert!(err.contains("--memory-trim-after-validation"), "{err}"); + assert!(err.contains("--enable-roa-validation-cache"), "{err}"); assert!(!err.contains("--parallel-phase1"), "{err}"); assert!(!err.contains("--parallel-phase2 "), "{err}"); } @@ -122,6 +123,58 @@ fn parse_disables_memory_trim_after_validation_by_default() { assert!(!args.memory_trim_after_validation); } +#[test] +fn parse_accepts_enable_roa_validation_cache() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--enable-roa-validation-cache".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.enable_roa_validation_cache); +} + +#[test] +fn parse_disables_roa_validation_cache_by_default() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(!args.enable_roa_validation_cache); +} + +#[test] +fn parse_accepts_analysis_out_and_implies_analyze() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--analysis-out".to_string(), + "run/analyze".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.analyze); + assert_eq!( + args.analysis_out_path.as_deref(), + Some(std::path::Path::new("run/analyze")) + ); +} + #[test] fn parse_accepts_report_json_compact_when_report_json_is_set() { let argv = vec![ @@ -157,6 +210,21 @@ fn parse_accepts_rsync_scope_policy() { assert_eq!(args.rsync_scope_policy, RsyncScopePolicy::ModuleRoot); } +#[test] +fn parse_accepts_host_rsync_scope_policy() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--rsync-scope".to_string(), + "host".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!(args.rsync_scope_policy, RsyncScopePolicy::Host); +} + #[test] fn parse_rejects_invalid_rsync_scope_policy() { let argv = vec![ @@ -1380,6 +1448,7 @@ fn synthetic_post_validation_shared() -> PostValidationShared { successful_tal_inputs: Vec::new(), tree, publication_points: vec![pp1, pp2, pp3], + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), downloads: Vec::new(), download_stats: crate::audit::AuditDownloadStats::default(), current_repo_objects: Vec::new(), @@ -1465,6 +1534,7 @@ fn run_report_task_and_stage_timing_work() { let stage_timing = RunStageTiming { validation_ms: 1, + enable_roa_validation_cache: false, report_build_ms: report_output.report_build_ms, report_write_ms: report_output.report_write_ms, ccr_build_ms: Some(2), @@ -1483,6 +1553,8 @@ fn run_report_task_and_stage_timing_work() { rrdp_download_ms_total: 13, rsync_download_ms_total: 14, download_bytes_total: 15, + roa_validation_cache: crate::validation::objects::RoaValidationCacheStats::default(), + analysis_counts: std::collections::HashMap::new(), vcir_storage_summary_ms: Some(16), vcir_storage: Some(VcirStorageSummary { entry_count: 2, @@ -1558,6 +1630,7 @@ fn stage_timing_serializes_memory_telemetry() { let report_path = dir.path().join("report.json"); let stage_timing = RunStageTiming { validation_ms: 1, + enable_roa_validation_cache: true, report_build_ms: 2, report_write_ms: None, ccr_build_ms: None, @@ -1576,6 +1649,14 @@ fn stage_timing_serializes_memory_telemetry() { rrdp_download_ms_total: 8, rsync_download_ms_total: 9, download_bytes_total: 10, + roa_validation_cache: crate::validation::objects::RoaValidationCacheStats { + hit_roas: 2, + ..crate::validation::objects::RoaValidationCacheStats::default() + }, + analysis_counts: std::collections::HashMap::from([( + "roa_validation_cache_hit_roas".to_string(), + 2, + )]), vcir_storage_summary_ms: None, vcir_storage: None, memory_telemetry: Some(MemoryTelemetrySummary { @@ -1626,6 +1707,8 @@ fn stage_timing_serializes_memory_telemetry() { checkpoint["rocksdb"]["totals"]["cur_size_all_mem_tables"], 16 ); + assert_eq!(value["analysis_counts"]["roa_validation_cache_hit_roas"], 2); + assert_eq!(value["roa_validation_cache"]["hit_roas"], 2); assert!( value["memory_telemetry"] .as_object() diff --git a/src/fetch/rsync.rs b/src/fetch/rsync.rs index 07df48d..146644e 100644 --- a/src/fetch/rsync.rs +++ b/src/fetch/rsync.rs @@ -50,6 +50,16 @@ pub trait RsyncFetcher: Send + Sync { fn dedup_key(&self, rsync_base_uri: &str) -> String { normalize_rsync_base_uri(rsync_base_uri) } + + /// Return an optional failure-only deduplication key. + /// + /// This key is only used to short-circuit repeated failed rsync fallbacks. It must + /// not be used to reuse successful fetch results unless it is identical to + /// `dedup_key`, because a successful fetch for one publication point does not imply + /// that another publication point under the same host has been fetched. + fn failure_dedup_key(&self, _rsync_base_uri: &str) -> Option { + None + } } /// A simple "rsync" implementation backed by a local directory. diff --git a/src/fetch/rsync_system.rs b/src/fetch/rsync_system.rs index 8433465..f2da3f6 100644 --- a/src/fetch/rsync_system.rs +++ b/src/fetch/rsync_system.rs @@ -15,23 +15,25 @@ use crate::fetch::rsync::{ #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum RsyncScopePolicy { + Host, PublicationPoint, ModuleRoot, } impl Default for RsyncScopePolicy { fn default() -> Self { - Self::PublicationPoint + Self::Host } } impl RsyncScopePolicy { pub fn parse_cli_value(value: &str) -> Result { match value { + "host" => Ok(Self::Host), "publication-point" => Ok(Self::PublicationPoint), "module-root" => Ok(Self::ModuleRoot), _ => Err(format!( - "invalid --rsync-scope: {value}; expected publication-point or module-root" + "invalid --rsync-scope: {value}; expected host, publication-point, or module-root" )), } } @@ -296,7 +298,9 @@ impl SystemRsyncFetcher { fn scope_fetch_uri(&self, rsync_base_uri: &str) -> String { match self.config.scope_policy { - RsyncScopePolicy::PublicationPoint => normalize_rsync_base_uri(rsync_base_uri), + RsyncScopePolicy::Host | RsyncScopePolicy::PublicationPoint => { + normalize_rsync_base_uri(rsync_base_uri) + } RsyncScopePolicy::ModuleRoot => rsync_module_root_uri(rsync_base_uri) .unwrap_or_else(|| normalize_rsync_base_uri(rsync_base_uri)), } @@ -348,6 +352,21 @@ impl RsyncFetcher for SystemRsyncFetcher { fn dedup_key(&self, rsync_base_uri: &str) -> String { self.scope_fetch_uri(rsync_base_uri) } + + fn failure_dedup_key(&self, rsync_base_uri: &str) -> Option { + match self.config.scope_policy { + RsyncScopePolicy::Host => rsync_host_scope_uri(rsync_base_uri), + RsyncScopePolicy::PublicationPoint | RsyncScopePolicy::ModuleRoot => None, + } + } +} + +fn rsync_host_scope_uri(rsync_base_uri: &str) -> Option { + let parsed = url::Url::parse(rsync_base_uri).ok()?; + if parsed.scheme() != "rsync" { + return None; + } + Some(format!("rsync://{}/", parsed.host_str()?)) } struct TempDir { @@ -535,12 +554,57 @@ mod tests { } #[test] - fn system_rsync_dedup_key_uses_publication_point_scope_by_default() { + fn rsync_host_scope_uri_returns_host_only() { + assert_eq!( + rsync_host_scope_uri("rsync://example.net/repo/ta/ca/publication-point/"), + Some("rsync://example.net/".to_string()) + ); + assert_eq!(rsync_host_scope_uri("https://example.net/repo"), None); + } + + #[test] + fn system_rsync_dedup_key_uses_host_scope_by_default() { let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig::default()); assert_eq!( fetcher.dedup_key("rsync://example.net/repo/ta/ca/publication-point/"), "rsync://example.net/repo/ta/ca/publication-point/" ); + assert_eq!( + fetcher.failure_dedup_key("rsync://example.net/repo/ta/ca/publication-point/"), + Some("rsync://example.net/".to_string()) + ); + } + + #[test] + fn system_rsync_host_scope_does_not_widen_success_fetch_scope() { + let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig { + scope_policy: RsyncScopePolicy::Host, + ..SystemRsyncConfig::default() + }); + assert_eq!( + fetcher.dedup_key("rsync://example.net/repo/ta/ca/publication-point/"), + "rsync://example.net/repo/ta/ca/publication-point/" + ); + assert_eq!( + fetcher.scope_fetch_uri("rsync://example.net/repo/ta/ca/publication-point/"), + "rsync://example.net/repo/ta/ca/publication-point/" + ); + assert_eq!( + fetcher.failure_dedup_key("rsync://example.net/repo/ta/ca/publication-point/"), + Some("rsync://example.net/".to_string()) + ); + } + + #[test] + fn system_rsync_dedup_key_uses_publication_point_scope_when_configured() { + let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig { + scope_policy: RsyncScopePolicy::PublicationPoint, + ..SystemRsyncConfig::default() + }); + assert_eq!( + fetcher.dedup_key("rsync://example.net/repo/ta/ca/publication-point/"), + "rsync://example.net/repo/ta/ca/publication-point/" + ); } #[test] diff --git a/src/parallel/repo_runtime.rs b/src/parallel/repo_runtime.rs index 9584d01..9644add 100644 --- a/src/parallel/repo_runtime.rs +++ b/src/parallel/repo_runtime.rs @@ -76,6 +76,7 @@ pub struct Phase1RepoSyncRuntime { coordinator: Mutex, worker_pool: Mutex>, rsync_scope_resolver: Arc String + Send + Sync>, + rsync_failure_scope_resolver: Arc Option + Send + Sync>, sync_preference: SyncPreference, } @@ -85,11 +86,28 @@ impl Phase1RepoSyncRuntime { worker_pool: RepoTransportWorkerPool, rsync_scope_resolver: Arc String + Send + Sync>, sync_preference: SyncPreference, + ) -> Self { + Self::new_with_failure_scope( + coordinator, + worker_pool, + rsync_scope_resolver, + Arc::new(|_base: &str| None), + sync_preference, + ) + } + + pub fn new_with_failure_scope( + coordinator: GlobalRunCoordinator, + worker_pool: RepoTransportWorkerPool, + rsync_scope_resolver: Arc String + Send + Sync>, + rsync_failure_scope_resolver: Arc Option + Send + Sync>, + sync_preference: SyncPreference, ) -> Self { Self { coordinator: Mutex::new(coordinator), worker_pool: Mutex::new(worker_pool), rsync_scope_resolver, + rsync_failure_scope_resolver, sync_preference, } } @@ -117,6 +135,7 @@ impl Phase1RepoSyncRuntime { let identity = Self::build_identity(ca); let requester = Self::build_requester(ca); let rsync_scope_uri = (self.rsync_scope_resolver)(&identity.rsync_base_uri); + let rsync_failure_scope_uri = (self.rsync_failure_scope_resolver)(&identity.rsync_base_uri); let action = { let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); coordinator.register_transport_request( @@ -125,6 +144,7 @@ impl Phase1RepoSyncRuntime { time::OffsetDateTime::now_utc(), priority, rsync_scope_uri, + rsync_failure_scope_uri, self.sync_preference, ) }; @@ -137,6 +157,7 @@ impl Phase1RepoSyncRuntime { "manifest_rsync_uri": ca.manifest_rsync_uri, "publication_point_rsync_uri": ca.publication_point_rsync_uri, "repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri, + "rsync_failure_scope_uri": task.rsync_failure_scope_uri, "repo_key_notification_uri": task.repo_identity.notification_uri, "priority": priority, "transport_mode": match task.mode { @@ -160,6 +181,7 @@ impl Phase1RepoSyncRuntime { "manifest_rsync_uri": ca.manifest_rsync_uri, "publication_point_rsync_uri": ca.publication_point_rsync_uri, "repo_key_rsync_base_uri": identity.rsync_base_uri, + "rsync_failure_scope_uri": (self.rsync_failure_scope_resolver)(&identity.rsync_base_uri), "repo_key_notification_uri": identity.notification_uri, "priority": priority, "runtime_state": format!("{state:?}"), @@ -175,6 +197,7 @@ impl Phase1RepoSyncRuntime { "manifest_rsync_uri": ca.manifest_rsync_uri, "publication_point_rsync_uri": ca.publication_point_rsync_uri, "repo_key_rsync_base_uri": identity.rsync_base_uri, + "rsync_failure_scope_uri": result.rsync_failure_scope_uri, "repo_key_notification_uri": identity.notification_uri, "priority": priority, "transport_mode": match result.mode { @@ -213,6 +236,7 @@ impl Phase1RepoSyncRuntime { "phase1_repo_task_dispatched", serde_json::json!({ "repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri, + "rsync_failure_scope_uri": task.rsync_failure_scope_uri, "repo_key_notification_uri": task.repo_identity.notification_uri, "requester_count": task.requesters.len(), "priority": task.priority, @@ -240,11 +264,12 @@ impl Phase1RepoSyncRuntime { return Ok(None); }; let transport_identity = envelope.repo_identity.clone(); - let completed_dedup_key = envelope.dedup_key.clone(); + let completed_envelope = envelope.clone(); crate::progress_log::emit( "phase1_repo_task_result", serde_json::json!({ "repo_key_rsync_base_uri": envelope.repo_identity.rsync_base_uri, + "rsync_failure_scope_uri": envelope.rsync_failure_scope_uri, "repo_key_notification_uri": envelope.repo_identity.notification_uri, "timing_ms": envelope.timing_ms, "transport_mode": match envelope.mode { @@ -283,7 +308,7 @@ impl Phase1RepoSyncRuntime { let completions = { let coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); coordinator - .finalized_runtime_records_for_transport(&completed_dedup_key) + .finalized_runtime_records_for_transport_result(&completed_envelope) .into_iter() .filter_map(|record| { let outcome = match record.state { @@ -506,6 +531,7 @@ mod tests { fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { RepoTransportResultEnvelope { dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), repo_identity: task.repo_identity, mode: task.mode, tal_id: task.tal_id, @@ -531,6 +557,7 @@ mod tests { self.count.fetch_add(1, Ordering::SeqCst); RepoTransportResultEnvelope { dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), repo_identity: task.repo_identity, mode: task.mode, tal_id: task.tal_id, @@ -559,6 +586,7 @@ mod tests { self.rrdp_count.fetch_add(1, Ordering::SeqCst); RepoTransportResultEnvelope { dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), repo_identity: task.repo_identity, mode: RepoTransportMode::Rrdp, tal_id: task.tal_id, @@ -574,6 +602,7 @@ mod tests { self.rsync_count.fetch_add(1, Ordering::SeqCst); RepoTransportResultEnvelope { dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), repo_identity: task.repo_identity, mode: RepoTransportMode::Rsync, tal_id: task.tal_id, @@ -601,6 +630,7 @@ mod tests { self.rrdp_count.fetch_add(1, Ordering::SeqCst); RepoTransportResultEnvelope { dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), repo_identity: task.repo_identity, mode: RepoTransportMode::Rrdp, tal_id: task.tal_id, @@ -616,6 +646,7 @@ mod tests { self.rsync_count.fetch_add(1, Ordering::SeqCst); RepoTransportResultEnvelope { dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), repo_identity: task.repo_identity, mode: RepoTransportMode::Rsync, tal_id: task.tal_id, diff --git a/src/parallel/repo_scheduler.rs b/src/parallel/repo_scheduler.rs index 45760a7..62b5727 100644 --- a/src/parallel/repo_scheduler.rs +++ b/src/parallel/repo_scheduler.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use crate::parallel::types::{ InFlightRepoEntry, RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoRuntimeState, @@ -44,6 +44,7 @@ pub struct RepoRuntimeRecord { pub state: RepoRuntimeState, pub rrdp_notification_key: Option, pub rsync_scope_key: String, + pub rsync_failure_scope_key: Option, pub requesters: Vec, pub validation_time: time::OffsetDateTime, pub priority: u8, @@ -69,6 +70,9 @@ pub struct TransportCompletion { pub struct TransportStateTables { rrdp_inflight: HashMap, rsync_inflight: HashMap, + rsync_failure_by_scope: HashMap, + rsync_failure_probe_inflight: HashMap, + rsync_failure_scope_reachable: HashSet, runtime_records: HashMap, } @@ -107,9 +111,45 @@ impl TransportStateTables { .collect() } + pub fn finalized_runtime_records_for_transport_result( + &self, + result: &RepoTransportResultEnvelope, + ) -> Vec { + let failure_scope = reusable_rsync_failure_scope(result); + self.runtime_records + .values() + .filter(|record| { + let exact_match = match &result.dedup_key { + RepoDedupKey::RrdpNotify { notification_uri } => { + record.rrdp_notification_key.as_deref() == Some(notification_uri.as_str()) + } + RepoDedupKey::RsyncScope { rsync_scope_uri } => { + record.rsync_scope_key == *rsync_scope_uri + } + }; + let failure_scope_match = failure_scope + .map(|scope| record.rsync_failure_scope_key.as_deref() == Some(scope)) + .unwrap_or(false); + exact_match || failure_scope_match + }) + .filter(|record| { + matches!( + record.state, + RepoRuntimeState::RrdpOk + | RepoRuntimeState::RsyncOk + | RepoRuntimeState::FailedTerminal + ) + }) + .cloned() + .collect() + } + pub fn reset_run_state(&mut self) { self.rrdp_inflight.clear(); self.rsync_inflight.clear(); + self.rsync_failure_by_scope.clear(); + self.rsync_failure_probe_inflight.clear(); + self.rsync_failure_scope_reachable.clear(); self.runtime_records.clear(); } @@ -120,6 +160,7 @@ impl TransportStateTables { validation_time: time::OffsetDateTime, priority: u8, rsync_scope_uri: String, + rsync_failure_scope_uri: Option, sync_preference: SyncPreference, ) -> TransportRequestAction { if let Some(record) = self.runtime_records.get_mut(&identity) { @@ -177,6 +218,7 @@ impl TransportStateTables { state: RepoRuntimeState::RrdpOk, rrdp_notification_key: Some(notification_uri), rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), requesters: vec![requester], validation_time, priority, @@ -194,6 +236,7 @@ impl TransportStateTables { state: RepoRuntimeState::RrdpFailedPendingRsync, rrdp_notification_key: Some(notification_uri), rsync_scope_key: rsync_scope_uri.clone(), + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), requesters: vec![requester.clone()], validation_time, priority, @@ -207,6 +250,7 @@ impl TransportStateTables { validation_time, priority, rsync_scope_uri, + rsync_failure_scope_uri, ) } }; @@ -220,6 +264,7 @@ impl TransportStateTables { state: RepoRuntimeState::WaitingRrdp, rrdp_notification_key: Some(notification_uri), rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), requesters: vec![requester], validation_time, priority, @@ -236,6 +281,7 @@ impl TransportStateTables { dedup_key: RepoDedupKey::RrdpNotify { notification_uri: notification_uri.clone(), }, + rsync_failure_scope_uri: None, repo_identity: identity.clone(), mode: RepoTransportMode::Rrdp, tal_id: requester.tal_id.clone(), @@ -262,6 +308,7 @@ impl TransportStateTables { state: RepoRuntimeState::WaitingRrdp, rrdp_notification_key: Some(notification_uri), rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), requesters: vec![requester], validation_time, priority, @@ -279,6 +326,7 @@ impl TransportStateTables { validation_time, priority, rsync_scope_uri, + rsync_failure_scope_uri, ) } @@ -289,6 +337,7 @@ impl TransportStateTables { validation_time: time::OffsetDateTime, priority: u8, rsync_scope_uri: String, + rsync_failure_scope_uri: Option, ) -> TransportRequestAction { if let Some(entry) = self.rsync_inflight.get_mut(&rsync_scope_uri) { if let Some(result) = entry.last_result.clone() { @@ -301,6 +350,7 @@ impl TransportStateTables { state: RepoRuntimeState::RsyncOk, rrdp_notification_key: None, rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), requesters: vec![requester], validation_time, priority, @@ -318,6 +368,7 @@ impl TransportStateTables { state: RepoRuntimeState::FailedTerminal, rrdp_notification_key: None, rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), requesters: vec![requester], validation_time, priority, @@ -338,6 +389,7 @@ impl TransportStateTables { state: RepoRuntimeState::WaitingRsync, rrdp_notification_key: None, rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), requesters: vec![requester], validation_time, priority, @@ -350,10 +402,62 @@ impl TransportStateTables { }; } + if let Some(failure_scope_uri) = rsync_failure_scope_uri.as_ref() { + if let Some(result) = self.rsync_failure_by_scope.get(failure_scope_uri).cloned() { + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::FailedTerminal, + rrdp_notification_key: None, + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: Some(result.clone()), + }, + ); + return TransportRequestAction::ReusedTerminalFailure(result); + } + + if !self + .rsync_failure_scope_reachable + .contains(failure_scope_uri) + { + if self + .rsync_failure_probe_inflight + .get(failure_scope_uri) + .is_some() + { + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::WaitingRsync, + rrdp_notification_key: None, + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: None, + }, + ); + return TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync, + }; + } + } + } + let task = RepoTransportTask { dedup_key: RepoDedupKey::RsyncScope { rsync_scope_uri: rsync_scope_uri.clone(), }, + rsync_failure_scope_uri: rsync_failure_scope_uri.clone(), repo_identity: identity.clone(), mode: RepoTransportMode::Rsync, tal_id: requester.tal_id.clone(), @@ -373,6 +477,15 @@ impl TransportStateTables { finished_at: None, }, ); + if let Some(failure_scope_uri) = rsync_failure_scope_uri.as_ref() { + if !self + .rsync_failure_scope_reachable + .contains(failure_scope_uri) + { + self.rsync_failure_probe_inflight + .insert(failure_scope_uri.clone(), rsync_scope_uri.clone()); + } + } self.runtime_records.insert( identity.clone(), RepoRuntimeRecord { @@ -380,6 +493,7 @@ impl TransportStateTables { state: RepoRuntimeState::WaitingRsync, rrdp_notification_key: None, rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), requesters: vec![requester], validation_time, priority, @@ -390,6 +504,85 @@ impl TransportStateTables { TransportRequestAction::Enqueue(task) } + fn schedule_rsync_for_record( + record: &mut RepoRuntimeRecord, + rsync_inflight: &mut HashMap, + rsync_failure_by_scope: &HashMap, + rsync_failure_probe_inflight: &mut HashMap, + rsync_failure_scope_reachable: &HashSet, + follow_up_tasks: &mut Vec, + ) { + let rsync_scope_uri = record.rsync_scope_key.clone(); + if let Some(entry) = rsync_inflight.get_mut(&rsync_scope_uri) { + if let Some(result) = entry.last_result.clone() { + match result.result { + RepoTransportResultKind::Success { .. } => { + record.state = RepoRuntimeState::RsyncOk; + record.last_success = Some(result); + } + RepoTransportResultKind::Failed { .. } => { + record.state = RepoRuntimeState::FailedTerminal; + record.terminal_failure = Some(result); + } + } + return; + } + entry.waiting_requesters.extend(record.requesters.clone()); + record.state = RepoRuntimeState::WaitingRsync; + return; + } + + if let Some(failure_scope_uri) = record.rsync_failure_scope_key.as_ref() { + if let Some(result) = rsync_failure_by_scope.get(failure_scope_uri).cloned() { + record.state = RepoRuntimeState::FailedTerminal; + record.terminal_failure = Some(result); + return; + } + if !rsync_failure_scope_reachable.contains(failure_scope_uri) + && rsync_failure_probe_inflight.contains_key(failure_scope_uri) + { + record.state = RepoRuntimeState::WaitingRsync; + return; + } + } + + let first_requester = record + .requesters + .first() + .expect("rsync record must keep at least one requester"); + let task = RepoTransportTask { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: rsync_scope_uri.clone(), + }, + rsync_failure_scope_uri: record.rsync_failure_scope_key.clone(), + repo_identity: record.identity.clone(), + mode: RepoTransportMode::Rsync, + tal_id: first_requester.tal_id.clone(), + rir_id: first_requester.rir_id.clone(), + validation_time: record.validation_time, + priority: record.priority, + requesters: record.requesters.clone(), + }; + rsync_inflight.insert( + rsync_scope_uri.clone(), + TransportInFlightEntry { + state: TransportTaskState::Pending, + task: task.clone(), + waiting_requesters: Vec::new(), + last_result: None, + started_at: None, + finished_at: None, + }, + ); + if let Some(failure_scope_uri) = record.rsync_failure_scope_key.as_ref() { + if !rsync_failure_scope_reachable.contains(failure_scope_uri) { + rsync_failure_probe_inflight.insert(failure_scope_uri.clone(), rsync_scope_uri); + } + } + record.state = RepoRuntimeState::WaitingRsync; + follow_up_tasks.push(task); + } + pub fn mark_transport_running( &mut self, dedup_key: &RepoDedupKey, @@ -464,39 +657,14 @@ impl TransportStateTables { && record.state == RepoRuntimeState::WaitingRrdp { record.state = RepoRuntimeState::RrdpFailedPendingRsync; - let rsync_scope_uri = record.rsync_scope_key.clone(); - if let Some(existing) = self.rsync_inflight.get_mut(&rsync_scope_uri) { - existing - .waiting_requesters - .extend(record.requesters.clone()); - record.state = RepoRuntimeState::WaitingRsync; - } else { - let task = RepoTransportTask { - dedup_key: RepoDedupKey::RsyncScope { - rsync_scope_uri: rsync_scope_uri.clone(), - }, - repo_identity: record.identity.clone(), - mode: RepoTransportMode::Rsync, - tal_id: record.requesters[0].tal_id.clone(), - rir_id: record.requesters[0].rir_id.clone(), - validation_time: record.validation_time, - priority: record.priority, - requesters: record.requesters.clone(), - }; - self.rsync_inflight.insert( - rsync_scope_uri.clone(), - TransportInFlightEntry { - state: TransportTaskState::Pending, - task: task.clone(), - waiting_requesters: Vec::new(), - last_result: None, - started_at: None, - finished_at: None, - }, - ); - record.state = RepoRuntimeState::WaitingRsync; - follow_up_tasks.push(task); - } + Self::schedule_rsync_for_record( + record, + &mut self.rsync_inflight, + &self.rsync_failure_by_scope, + &mut self.rsync_failure_probe_inflight, + &self.rsync_failure_scope_reachable, + &mut follow_up_tasks, + ); } } Ok(TransportCompletion { @@ -508,6 +676,7 @@ impl TransportStateTables { RepoDedupKey::RsyncScope { rsync_scope_uri }, RepoTransportResultKind::Success { .. }, ) => { + let mut follow_up_tasks = Vec::new(); let entry = self .rsync_inflight .get_mut(rsync_scope_uri) @@ -515,6 +684,11 @@ impl TransportStateTables { entry.state = TransportTaskState::Finished; entry.finished_at = Some(finished_at); entry.last_result = Some(result.clone()); + if let Some(failure_scope_uri) = result.rsync_failure_scope_uri.as_ref() { + self.rsync_failure_probe_inflight.remove(failure_scope_uri); + self.rsync_failure_scope_reachable + .insert(failure_scope_uri.clone()); + } let released_requesters = std::mem::take(&mut entry.waiting_requesters); for record in self.runtime_records.values_mut() { if record.rsync_scope_key == *rsync_scope_uri @@ -528,9 +702,27 @@ impl TransportStateTables { record.last_success = Some(result.clone()); } } + if let Some(failure_scope_uri) = result.rsync_failure_scope_uri.as_ref() { + for record in self.runtime_records.values_mut() { + if record.rsync_scope_key != *rsync_scope_uri + && record.rsync_failure_scope_key.as_deref() + == Some(failure_scope_uri.as_str()) + && matches!(record.state, RepoRuntimeState::WaitingRsync) + { + Self::schedule_rsync_for_record( + record, + &mut self.rsync_inflight, + &self.rsync_failure_by_scope, + &mut self.rsync_failure_probe_inflight, + &self.rsync_failure_scope_reachable, + &mut follow_up_tasks, + ); + } + } + } Ok(TransportCompletion { released_requesters, - follow_up_tasks: Vec::new(), + follow_up_tasks, }) } ( @@ -544,9 +736,25 @@ impl TransportStateTables { entry.state = TransportTaskState::Finished; entry.finished_at = Some(finished_at); entry.last_result = Some(result.clone()); + let reusable_failure_scope = + reusable_rsync_failure_scope(&result).map(str::to_string); + if let Some(failure_scope_uri) = result.rsync_failure_scope_uri.as_ref() { + self.rsync_failure_probe_inflight.remove(failure_scope_uri); + if reusable_failure_scope.as_deref() == Some(failure_scope_uri.as_str()) { + self.rsync_failure_by_scope + .insert(failure_scope_uri.clone(), result.clone()); + } else { + self.rsync_failure_scope_reachable + .insert(failure_scope_uri.clone()); + } + } let released_requesters = std::mem::take(&mut entry.waiting_requesters); for record in self.runtime_records.values_mut() { - if record.rsync_scope_key == *rsync_scope_uri + if (record.rsync_scope_key == *rsync_scope_uri + || reusable_failure_scope + .as_deref() + .map(|scope| record.rsync_failure_scope_key.as_deref() == Some(scope)) + .unwrap_or(false)) && matches!( record.state, RepoRuntimeState::WaitingRsync @@ -557,6 +765,31 @@ impl TransportStateTables { record.terminal_failure = Some(result.clone()); } } + if let Some(failure_scope_uri) = result.rsync_failure_scope_uri.as_ref() { + if reusable_failure_scope.as_deref() != Some(failure_scope_uri.as_str()) { + let mut follow_up_tasks = Vec::new(); + for record in self.runtime_records.values_mut() { + if record.rsync_scope_key != *rsync_scope_uri + && record.rsync_failure_scope_key.as_deref() + == Some(failure_scope_uri.as_str()) + && matches!(record.state, RepoRuntimeState::WaitingRsync) + { + Self::schedule_rsync_for_record( + record, + &mut self.rsync_inflight, + &self.rsync_failure_by_scope, + &mut self.rsync_failure_probe_inflight, + &self.rsync_failure_scope_reachable, + &mut follow_up_tasks, + ); + } + } + return Ok(TransportCompletion { + released_requesters, + follow_up_tasks, + }); + } + } Ok(TransportCompletion { released_requesters, follow_up_tasks: Vec::new(), @@ -566,6 +799,29 @@ impl TransportStateTables { } } +fn reusable_rsync_failure_scope(result: &RepoTransportResultEnvelope) -> Option<&str> { + let scope = result.rsync_failure_scope_uri.as_deref()?; + match (&result.mode, &result.result) { + (RepoTransportMode::Rsync, RepoTransportResultKind::Failed { detail, .. }) + if is_host_level_rsync_failure(detail) => + { + Some(scope) + } + _ => None, + } +} + +fn is_host_level_rsync_failure(detail: &str) -> bool { + let lower = detail.to_ascii_lowercase(); + lower.contains("timeout waiting for daemon connection") + || lower.contains("failed to connect") + || lower.contains("no route to host") + || lower.contains("network is unreachable") + || lower.contains("connection refused") + || lower.contains("name or service not known") + || lower.contains("temporary failure in name resolution") +} + #[derive(Default)] pub struct InFlightRepoTable { entries: HashMap, @@ -1005,6 +1261,7 @@ mod transport_tests { time::OffsetDateTime::UNIX_EPOCH, 0, "rsync://example.test/module/".to_string(), + None, SyncPreference::RrdpThenRsync, ); let task = match action { @@ -1042,6 +1299,7 @@ mod transport_tests { time::OffsetDateTime::UNIX_EPOCH, 0, "rsync://example.test/module/".to_string(), + None, SyncPreference::RrdpThenRsync, ); let other_identity = RepoIdentity::new( @@ -1054,6 +1312,7 @@ mod transport_tests { time::OffsetDateTime::UNIX_EPOCH, 0, "rsync://example.test/module/".to_string(), + None, SyncPreference::RrdpThenRsync, ); assert_eq!( @@ -1090,6 +1349,7 @@ mod transport_tests { time::OffsetDateTime::UNIX_EPOCH, 0, "rsync://example.test/module/".to_string(), + None, SyncPreference::RrdpThenRsync, ); tables @@ -1106,6 +1366,7 @@ mod transport_tests { dedup_key: RepoDedupKey::RrdpNotify { notification_uri: "https://example.test/notify.xml".to_string(), }, + rsync_failure_scope_uri: None, repo_identity: identity.clone(), mode: RepoTransportMode::Rrdp, tal_id: "apnic".to_string(), @@ -1130,6 +1391,7 @@ mod transport_tests { time::OffsetDateTime::UNIX_EPOCH, 0, "rsync://example.test/module/".to_string(), + None, SyncPreference::RrdpThenRsync, ); assert!(matches!(action, TransportRequestAction::ReusedSuccess(_))); @@ -1137,6 +1399,25 @@ mod transport_tests { tables.runtime_records.get(&later_identity).unwrap().state, RepoRuntimeState::RrdpOk ); + + let same_identity_action = tables.register_transport_request( + later_identity.clone(), + requester("b-again"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + assert!(matches!( + same_identity_action, + TransportRequestAction::ReusedSuccess(_) + )); + + let finalized = tables.finalized_runtime_records_for_transport(&RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }); + assert_eq!(finalized.len(), 2); } #[test] @@ -1152,6 +1433,7 @@ mod transport_tests { time::OffsetDateTime::UNIX_EPOCH, 0, "rsync://example.test/module/".to_string(), + None, SyncPreference::RrdpThenRsync, ); tables @@ -1168,6 +1450,7 @@ mod transport_tests { dedup_key: RepoDedupKey::RrdpNotify { notification_uri: "https://example.test/notify.xml".to_string(), }, + rsync_failure_scope_uri: None, repo_identity: identity.clone(), mode: RepoTransportMode::Rrdp, tal_id: "apnic".to_string(), @@ -1187,6 +1470,154 @@ mod transport_tests { tables.runtime_records.get(&identity).unwrap().state, RepoRuntimeState::WaitingRsync ); + + let same_identity_action = tables.register_transport_request( + identity.clone(), + requester("a-again"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + assert_eq!( + same_identity_action, + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync + } + ); + + let later_identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/b/", + ); + let later_action = tables.register_transport_request( + later_identity.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + assert_eq!( + later_action, + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync + } + ); + assert_eq!( + tables.runtime_records.get(&later_identity).unwrap().state, + RepoRuntimeState::WaitingRsync + ); + } + + #[test] + fn host_failure_scope_probes_once_then_reuses_terminal_failure() { + let mut tables = TransportStateTables::new(); + let identity_a = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/a/", + ); + let identity_b = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/b/", + ); + let _ = tables.register_transport_request( + identity_a.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/a/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RrdpThenRsync, + ); + let _ = tables.register_transport_request( + identity_b.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/b/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RrdpThenRsync, + ); + tables + .mark_transport_running( + &RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("mark running"); + let completion = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: identity_a.clone(), + mode: RepoTransportMode::Rrdp, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 10, + result: RepoTransportResultKind::Failed { + detail: "rrdp timeout".to_string(), + warnings: Vec::new(), + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rrdp failure"); + assert_eq!(completion.follow_up_tasks.len(), 1); + assert_eq!( + completion.follow_up_tasks[0] + .rsync_failure_scope_uri + .as_deref(), + Some("rsync://example.test/") + ); + let probe_task = completion.follow_up_tasks[0].clone(); + + tables + .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark rsync running"); + let completion = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: probe_task.dedup_key, + rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, + repo_identity: probe_task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 15_000, + result: RepoTransportResultKind::Failed { + detail: "rsync error: timeout waiting for daemon connection".to_string(), + warnings: Vec::new(), + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rsync failure"); + assert!(completion.follow_up_tasks.is_empty()); + assert_eq!( + tables.runtime_records.get(&identity_a).unwrap().state, + RepoRuntimeState::FailedTerminal + ); + assert_eq!( + tables.runtime_records.get(&identity_b).unwrap().state, + RepoRuntimeState::FailedTerminal + ); + let finalized = tables.finalized_runtime_records_for_transport_result( + tables + .runtime_records + .get(&identity_a) + .unwrap() + .terminal_failure + .as_ref() + .unwrap(), + ); + assert_eq!(finalized.len(), 2); } #[test] @@ -1199,6 +1630,7 @@ mod transport_tests { time::OffsetDateTime::UNIX_EPOCH, 0, "rsync://example.test/module/".to_string(), + None, SyncPreference::RsyncOnly, ); tables @@ -1215,6 +1647,7 @@ mod transport_tests { dedup_key: RepoDedupKey::RsyncScope { rsync_scope_uri: "rsync://example.test/module/".to_string(), }, + rsync_failure_scope_uri: None, repo_identity: identity.clone(), mode: RepoTransportMode::Rsync, tal_id: "apnic".to_string(), @@ -1235,12 +1668,384 @@ mod transport_tests { time::OffsetDateTime::UNIX_EPOCH, 0, "rsync://example.test/module/".to_string(), + None, SyncPreference::RsyncOnly, ); assert!(matches!( action, TransportRequestAction::ReusedTerminalFailure(_) )); + + let finalized = tables.finalized_runtime_records_for_transport(&RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string(), + }); + assert_eq!(finalized.len(), 2); + + let same_identity_action = tables.register_transport_request( + identity, + requester("a-again"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + assert!(matches!( + same_identity_action, + TransportRequestAction::ReusedTerminalFailure(_) + )); + } + + #[test] + fn complete_rsync_success_reuses_for_later_identity_requests() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let action = tables.register_transport_request( + identity.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + assert!(matches!(action, TransportRequestAction::Enqueue(_))); + tables + .mark_transport_running( + &RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("mark running"); + tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: identity.clone(), + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 20, + result: RepoTransportResultKind::Success { + source: "rsync".to_string(), + warnings: Vec::new(), + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rsync success"); + + let later_identity = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let action = tables.register_transport_request( + later_identity.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + assert!(matches!(action, TransportRequestAction::ReusedSuccess(_))); + assert_eq!( + tables.runtime_records.get(&later_identity).unwrap().state, + RepoRuntimeState::RsyncOk + ); + } + + #[test] + fn register_rsync_request_waits_on_existing_rsync_task() { + let mut tables = TransportStateTables::new(); + let identity_a = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let identity_b = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let _ = tables.register_transport_request( + identity_a, + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + let action = tables.register_transport_request( + identity_b.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + assert_eq!( + action, + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync + } + ); + assert_eq!( + tables.runtime_records.get(&identity_b).unwrap().state, + RepoRuntimeState::WaitingRsync + ); + assert_eq!( + tables + .rsync_inflight + .get("rsync://example.test/module/") + .unwrap() + .waiting_requesters + .len(), + 1 + ); + } + + #[test] + fn host_failure_scope_success_marks_host_reachable_and_schedules_waiters() { + let mut tables = TransportStateTables::new(); + let identity_a = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let identity_b = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let action_a = tables.register_transport_request( + identity_a.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/a/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + let probe_task = match action_a { + TransportRequestAction::Enqueue(task) => task, + other => panic!("expected first host probe enqueue, got {other:?}"), + }; + let action_b = tables.register_transport_request( + identity_b.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/b/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + assert_eq!( + action_b, + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync + } + ); + + tables + .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark rsync running"); + let completion = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: probe_task.dedup_key, + rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, + repo_identity: probe_task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 5, + result: RepoTransportResultKind::Success { + source: "rsync".to_string(), + warnings: Vec::new(), + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rsync success"); + + assert_eq!( + tables.runtime_records.get(&identity_a).unwrap().state, + RepoRuntimeState::RsyncOk + ); + assert_eq!(completion.follow_up_tasks.len(), 1); + assert_eq!( + completion.follow_up_tasks[0].dedup_key, + RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/repo/b/".to_string() + } + ); + assert_eq!( + tables.runtime_records.get(&identity_b).unwrap().state, + RepoRuntimeState::WaitingRsync + ); + } + + #[test] + fn non_host_level_rsync_failure_does_not_poison_host_scope() { + let mut tables = TransportStateTables::new(); + let identity_a = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let identity_b = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let action_a = tables.register_transport_request( + identity_a.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/a/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + let probe_task = match action_a { + TransportRequestAction::Enqueue(task) => task, + other => panic!("expected first host probe enqueue, got {other:?}"), + }; + let _ = tables.register_transport_request( + identity_b.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/b/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + + tables + .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark rsync running"); + let completion = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: probe_task.dedup_key, + rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, + repo_identity: probe_task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 5, + result: RepoTransportResultKind::Failed { + detail: "rsync file digest mismatch after download".to_string(), + warnings: Vec::new(), + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rsync failure"); + + assert_eq!( + tables.runtime_records.get(&identity_a).unwrap().state, + RepoRuntimeState::FailedTerminal + ); + assert_eq!(completion.follow_up_tasks.len(), 1); + assert_eq!( + completion.follow_up_tasks[0].dedup_key, + RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/repo/b/".to_string() + } + ); + } + + #[test] + fn cached_host_level_failure_reuses_for_later_rsync_only_requests() { + let mut tables = TransportStateTables::new(); + let identity_a = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let action_a = tables.register_transport_request( + identity_a, + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/a/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + let probe_task = match action_a { + TransportRequestAction::Enqueue(task) => task, + other => panic!("expected first host probe enqueue, got {other:?}"), + }; + tables + .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark rsync running"); + let _ = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: probe_task.dedup_key, + rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, + repo_identity: probe_task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 15_000, + result: RepoTransportResultKind::Failed { + detail: "rsync error: failed to connect to daemon".to_string(), + warnings: Vec::new(), + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete host failure"); + + let later_identity = RepoIdentity::new(None, "rsync://example.test/repo/c/"); + let action = tables.register_transport_request( + later_identity.clone(), + requester("c"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/c/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + assert!(matches!( + action, + TransportRequestAction::ReusedTerminalFailure(_) + )); + assert_eq!( + tables.runtime_records.get(&later_identity).unwrap().state, + RepoRuntimeState::FailedTerminal + ); + } + + #[test] + fn reset_run_state_clears_host_failure_scope_cache() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let action = tables.register_transport_request( + identity, + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/a/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + let probe_task = match action { + TransportRequestAction::Enqueue(task) => task, + other => panic!("expected host probe enqueue, got {other:?}"), + }; + tables + .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark rsync running"); + let _ = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: probe_task.dedup_key, + rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, + repo_identity: probe_task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 15_000, + result: RepoTransportResultKind::Failed { + detail: "temporary failure in name resolution".to_string(), + warnings: Vec::new(), + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete host failure"); + + tables.reset_run_state(); + let later_identity = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let action = tables.register_transport_request( + later_identity, + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/b/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + assert!(matches!(action, TransportRequestAction::Enqueue(_))); } #[test] @@ -1256,6 +2061,7 @@ mod transport_tests { time::OffsetDateTime::UNIX_EPOCH, 0, "rsync://example.test/module/".to_string(), + None, SyncPreference::RsyncOnly, ); let task = match action { diff --git a/src/parallel/repo_worker.rs b/src/parallel/repo_worker.rs index 7d1346b..efbd347 100644 --- a/src/parallel/repo_worker.rs +++ b/src/parallel/repo_worker.rs @@ -85,6 +85,7 @@ impl RepoTransportExecutor for LiveRrdpTransportExecutor RepoTransportResultEnvelope { dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri, repo_identity: task.repo_identity, mode: RepoTransportMode::Rrdp, tal_id: task.tal_id, @@ -97,6 +98,7 @@ impl RepoTransportExecutor for LiveRrdpTransportExecutor RepoTransportResultEnvelope { dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri, repo_identity: task.repo_identity, mode: RepoTransportMode::Rrdp, tal_id: task.tal_id, @@ -151,6 +153,7 @@ impl RepoTransportExecutor for LiveRsyncTransportExec ) { Ok(_) => RepoTransportResultEnvelope { dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri, repo_identity: task.repo_identity, mode: RepoTransportMode::Rsync, tal_id: task.tal_id, @@ -163,6 +166,7 @@ impl RepoTransportExecutor for LiveRsyncTransportExec }, Err(err) => RepoTransportResultEnvelope { dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri, repo_identity: task.repo_identity, mode: RepoTransportMode::Rsync, tal_id: task.tal_id, @@ -660,6 +664,7 @@ mod tests { dedup_key: RepoDedupKey::RrdpNotify { notification_uri: notification_uri.to_string(), }, + rsync_failure_scope_uri: None, repo_identity: RepoIdentity::new(Some(notification_uri.to_string()), rsync_base_uri), mode: RepoTransportMode::Rrdp, tal_id: "arin".to_string(), @@ -684,6 +689,7 @@ mod tests { dedup_key: RepoDedupKey::RsyncScope { rsync_scope_uri: rsync_scope_uri.to_string(), }, + rsync_failure_scope_uri: None, repo_identity: RepoIdentity::new(None, rsync_base_uri), mode: RepoTransportMode::Rsync, tal_id: "arin".to_string(), @@ -873,6 +879,7 @@ mod tests { fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { RepoTransportResultEnvelope { dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri, repo_identity: task.repo_identity, mode: task.mode, tal_id: task.tal_id, diff --git a/src/parallel/run_coordinator.rs b/src/parallel/run_coordinator.rs index 2b365aa..38ce112 100644 --- a/src/parallel/run_coordinator.rs +++ b/src/parallel/run_coordinator.rs @@ -123,6 +123,7 @@ impl GlobalRunCoordinator { validation_time: time::OffsetDateTime, priority: u8, rsync_scope_uri: String, + rsync_failure_scope_uri: Option, sync_preference: SyncPreference, ) -> TransportRequestAction { let action = self.transport_tables.register_transport_request( @@ -131,6 +132,7 @@ impl GlobalRunCoordinator { validation_time, priority, rsync_scope_uri, + rsync_failure_scope_uri, sync_preference, ); match &action { @@ -205,6 +207,14 @@ impl GlobalRunCoordinator { .finalized_runtime_records_for_transport(dedup_key) } + pub fn finalized_runtime_records_for_transport_result( + &self, + result: &RepoTransportResultEnvelope, + ) -> Vec { + self.transport_tables + .finalized_runtime_records_for_transport_result(result) + } + pub fn reset_run_state(&mut self) { self.in_flight_repos.reset_run_state(); self.transport_tables.reset_run_state(); @@ -296,6 +306,7 @@ mod tests { time::OffsetDateTime::UNIX_EPOCH, 0, "rsync://example.test/repo/".to_string(), + None, SyncPreference::RrdpThenRsync, ); assert!(matches!( diff --git a/src/parallel/types.rs b/src/parallel/types.rs index ad146e3..ce278d9 100644 --- a/src/parallel/types.rs +++ b/src/parallel/types.rs @@ -105,6 +105,7 @@ pub enum RepoTransportMode { #[derive(Clone, Debug, PartialEq, Eq)] pub struct RepoTransportTask { pub dedup_key: RepoDedupKey, + pub rsync_failure_scope_uri: Option, pub repo_identity: RepoIdentity, pub mode: RepoTransportMode, pub tal_id: String, @@ -129,6 +130,7 @@ pub enum RepoTransportResultKind { #[derive(Clone, Debug, PartialEq, Eq)] pub struct RepoTransportResultEnvelope { pub dedup_key: RepoDedupKey, + pub rsync_failure_scope_uri: Option, pub repo_identity: RepoIdentity, pub mode: RepoTransportMode, pub tal_id: String, @@ -218,6 +220,7 @@ impl RepoSyncTask { ) -> RepoTransportTask { RepoTransportTask { dedup_key, + rsync_failure_scope_uri: None, repo_identity: self.repo_key.as_identity(), mode, tal_id: self.tal_id.clone(), @@ -450,6 +453,7 @@ mod tests { dedup_key: RepoDedupKey::RsyncScope { rsync_scope_uri: "rsync://example.test/module/".to_string(), }, + rsync_failure_scope_uri: None, repo_identity: identity.clone(), mode: RepoTransportMode::Rsync, tal_id: "arin".to_string(), @@ -464,6 +468,7 @@ mod tests { dedup_key: RepoDedupKey::RsyncScope { rsync_scope_uri: "rsync://example.test/module/".to_string(), }, + rsync_failure_scope_uri: None, repo_identity: identity, mode: RepoTransportMode::Rsync, tal_id: "arin".to_string(), diff --git a/src/storage.rs b/src/storage.rs index 43aa3f4..f1d7f07 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -1191,8 +1191,7 @@ impl VcirStorageEntrySummary { child_resources: VcirChildResourceSizeBreakdown::from_vcir(vcir), local_output_old_projection_bytes: field_sizes.local_output_old_projection_bytes(), local_output_typed_projection_bytes: field_sizes.local_output_typed_projection_bytes(), - local_output_projection_saved_bytes: field_sizes - .local_output_projection_saved_bytes(), + local_output_projection_saved_bytes: field_sizes.local_output_projection_saved_bytes(), field_sizes, } } @@ -2113,10 +2112,11 @@ impl RocksStore { summary .child_resources .add_assign(&entry_summary.child_resources); - summary - .field_sizes - .add_assign(&entry_summary.field_sizes); - push_top_vcir_storage_entry(&mut summary.top_entries_by_vcir_value_bytes, entry_summary); + summary.field_sizes.add_assign(&entry_summary.field_sizes); + push_top_vcir_storage_entry( + &mut summary.top_entries_by_vcir_value_bytes, + entry_summary, + ); } summary.local_output_old_projection_bytes = summary.field_sizes.local_output_old_projection_bytes(); diff --git a/src/validation/objects.rs b/src/validation/objects.rs index 8222c8a..4647524 100644 --- a/src/validation/objects.rs +++ b/src/validation/objects.rs @@ -15,12 +15,15 @@ use crate::parallel::object_worker::{ use crate::policy::{Policy, SignedObjectFailurePolicy}; use crate::report::{RfcRef, Warning}; use crate::storage::{ - PackFile, PackTime, VcirLocalOutput, VcirLocalOutputPayload, VcirOutputType, + PackFile, PackTime, ValidatedCaInstanceResult, VcirArtifactKind, VcirArtifactRole, + VcirArtifactValidationStatus, VcirLocalOutput, VcirLocalOutputPayload, VcirOutputType, VcirSourceObjectType, }; use crate::validation::cert_path::{CertPathError, validate_signed_object_ee_cert_path_fast}; use crate::validation::manifest::PublicationPointData; use crate::validation::publication_point::PublicationPointSnapshot; +use sha2::Digest; +use std::collections::HashMap; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use x509_parser::prelude::FromDer; @@ -38,6 +41,10 @@ fn sha256_hex_to_32(hex_value: &str) -> [u8; 32] { out } +fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(sha2::Sha256::digest(bytes)) +} + fn decode_resource_certificate_with_policy( der: &[u8], policy: &Policy, @@ -110,6 +117,7 @@ pub struct ObjectsOutput { pub warnings: Vec, pub stats: ObjectsStats, pub audit: Vec, + pub roa_cache_stats: RoaValidationCacheStats, } #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -123,6 +131,315 @@ pub struct ObjectsStats { pub publication_point_dropped: bool, } +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)] +pub struct RoaValidationCacheStats { + pub enabled_publication_points: usize, + pub vcir_hit_publication_points: usize, + pub vcir_miss_publication_points: usize, + pub hit_roas: usize, + pub miss_roas: usize, + pub blocked_roas: usize, + pub fresh_roas: usize, + pub context_gate_nanos: u64, + pub lookup_nanos: u64, +} + +#[derive(Clone, Copy, Debug)] +pub struct RoaValidationCacheInput<'a> { + enabled: bool, + view: Option<&'a RoaValidationCacheView>, +} + +impl<'a> RoaValidationCacheInput<'a> { + pub fn disabled() -> Self { + Self { + enabled: false, + view: None, + } + } + + pub fn enabled(view: Option<&'a RoaValidationCacheView>) -> Self { + Self { + enabled: true, + view, + } + } +} + +impl RoaValidationCacheStats { + pub fn add_assign(&mut self, other: &Self) { + self.enabled_publication_points += other.enabled_publication_points; + self.vcir_hit_publication_points += other.vcir_hit_publication_points; + self.vcir_miss_publication_points += other.vcir_miss_publication_points; + self.hit_roas += other.hit_roas; + self.miss_roas += other.miss_roas; + self.blocked_roas += other.blocked_roas; + self.fresh_roas += other.fresh_roas; + self.context_gate_nanos = self + .context_gate_nanos + .saturating_add(other.context_gate_nanos); + self.lookup_nanos = self.lookup_nanos.saturating_add(other.lookup_nanos); + } + + fn for_input(input: RoaValidationCacheInput<'_>, roa_total: usize) -> Self { + let mut stats = Self::default(); + if !input.enabled { + return stats; + } + + stats.enabled_publication_points = 1; + if input.view.is_some() { + stats.vcir_hit_publication_points = 1; + } else { + stats.vcir_miss_publication_points = 1; + stats.miss_roas = roa_total; + stats.fresh_roas = roa_total; + } + stats + } + + fn record_to_timing(&self, timing: Option<&TimingHandle>) { + let Some(timing) = timing else { + return; + }; + record_non_zero( + timing, + "roa_validation_cache_enabled_publication_points", + self.enabled_publication_points, + ); + record_non_zero( + timing, + "roa_validation_cache_vcir_hit_publication_points", + self.vcir_hit_publication_points, + ); + record_non_zero( + timing, + "roa_validation_cache_vcir_miss_publication_points", + self.vcir_miss_publication_points, + ); + record_non_zero(timing, "roa_validation_cache_hit_roas", self.hit_roas); + record_non_zero(timing, "roa_validation_cache_miss_roas", self.miss_roas); + record_non_zero( + timing, + "roa_validation_cache_blocked_roas", + self.blocked_roas, + ); + record_non_zero(timing, "roa_validation_cache_fresh_roas", self.fresh_roas); + record_non_zero( + timing, + "roa_validation_cache_context_gate_nanos", + self.context_gate_nanos as usize, + ); + record_non_zero( + timing, + "roa_validation_cache_lookup_nanos", + self.lookup_nanos as usize, + ); + timing.record_phase_nanos( + "roa_validation_cache_context_gate_total", + self.context_gate_nanos, + ); + timing.record_phase_nanos("roa_validation_cache_lookup_total", self.lookup_nanos); + } +} + +fn record_non_zero(timing: &TimingHandle, key: &'static str, value: usize) { + if value > 0 { + timing.record_count(key, value as u64); + } +} + +#[derive(Clone, Debug)] +pub struct RoaValidationCacheView { + entries_by_uri: HashMap, + issuer_ca_sha256_hex: Option, + crl_sha256_by_uri: HashMap, + blocked: bool, +} + +#[derive(Clone, Debug)] +pub struct CachedRoaValidationResult { + source_object_hash: [u8; 32], + outputs: Vec, +} + +impl RoaValidationCacheView { + pub fn from_vcir( + vcir: &ValidatedCaInstanceResult, + validation_time: time::OffsetDateTime, + ) -> Self { + let mut entries_by_uri: HashMap = HashMap::new(); + let mut issuer_ca_sha256_hex: Option = None; + let mut crl_sha256_by_uri: HashMap = HashMap::new(); + let blocked = vcir + .instance_gate + .instance_effective_until + .parse() + .map(|effective_until| effective_until <= validation_time) + .unwrap_or(true); + + if blocked { + return Self { + entries_by_uri, + issuer_ca_sha256_hex, + crl_sha256_by_uri, + blocked, + }; + } + + for artifact in &vcir.related_artifacts { + if artifact.validation_status != VcirArtifactValidationStatus::Accepted { + continue; + } + match (artifact.artifact_role, artifact.artifact_kind) { + ( + VcirArtifactRole::IssuerCert | VcirArtifactRole::TrustAnchorCert, + VcirArtifactKind::Cer, + ) => { + issuer_ca_sha256_hex = Some(artifact.sha256.clone()); + } + (_, VcirArtifactKind::Crl) => { + if let Some(uri) = artifact.uri.as_ref() { + crl_sha256_by_uri.insert(uri.clone(), artifact.sha256.clone()); + } + } + _ => {} + } + } + + for output in &vcir.local_outputs { + if output.output_type != VcirOutputType::Vrp + || output.source_object_type != VcirSourceObjectType::Roa + { + continue; + } + entries_by_uri + .entry(output.source_object_uri.clone()) + .or_insert_with(|| CachedRoaValidationResult { + source_object_hash: output.source_object_hash, + outputs: Vec::new(), + }) + .outputs + .push(output.clone()); + } + + Self { + entries_by_uri, + issuer_ca_sha256_hex, + crl_sha256_by_uri, + blocked, + } + } + + fn matches_current_context(&self, issuer_ca_der: &[u8], locked_files: &[PackFile]) -> bool { + if self.blocked { + return false; + } + + let Some(expected_issuer_hash) = self.issuer_ca_sha256_hex.as_ref() else { + return false; + }; + if expected_issuer_hash != &sha256_hex(issuer_ca_der) { + return false; + } + + let current_crl_hashes = locked_files + .iter() + .filter(|file| file.rsync_uri.ends_with(".crl")) + .map(|file| (file.rsync_uri.clone(), sha256_hex_from_32(&file.sha256))) + .collect::>(); + self.crl_sha256_by_uri == current_crl_hashes + } + + fn lookup( + &self, + file: &PackFile, + validation_time: time::OffsetDateTime, + ) -> RoaCacheLookupResult { + if self.blocked { + return RoaCacheLookupResult::Blocked; + } + + let Some(cached) = self.entries_by_uri.get(file.rsync_uri.as_str()) else { + return RoaCacheLookupResult::Miss; + }; + if cached.source_object_hash != file.sha256 { + return RoaCacheLookupResult::Blocked; + } + if cached.outputs.is_empty() { + return RoaCacheLookupResult::Miss; + } + if cached.outputs.iter().any(|output| { + output + .item_effective_until + .parse() + .map_or(true, |t| t <= validation_time) + }) { + return RoaCacheLookupResult::Blocked; + } + + let mut vrps = Vec::with_capacity(cached.outputs.len()); + for output in &cached.outputs { + let VcirLocalOutputPayload::Vrp { + asn, + afi, + prefix_len, + addr, + max_length, + } = &output.payload + else { + return RoaCacheLookupResult::Blocked; + }; + vrps.push(Vrp { + asn: *asn, + prefix: IpPrefix { + afi: *afi, + prefix_len: *prefix_len, + addr: *addr, + }, + max_length: *max_length, + }); + } + + RoaCacheLookupResult::Hit(RoaTaskOk { + vrps, + local_outputs: cached.outputs.clone(), + reused_from_cache: true, + }) + } +} + +fn active_roa_cache_view<'a>( + roa_cache: RoaValidationCacheInput<'a>, + issuer_ca_der: &[u8], + locked_files: &[PackFile], + stats: &mut RoaValidationCacheStats, + roa_total: usize, +) -> Option<&'a RoaValidationCacheView> { + let view = roa_cache.view?; + let gate_started = Instant::now(); + if view.matches_current_context(issuer_ca_der, locked_files) { + stats.context_gate_nanos = stats + .context_gate_nanos + .saturating_add(gate_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + Some(view) + } else { + stats.context_gate_nanos = stats + .context_gate_nanos + .saturating_add(gate_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + stats.blocked_roas += roa_total; + stats.fresh_roas += roa_total; + None + } +} + +#[derive(Debug)] +enum RoaCacheLookupResult { + Hit(RoaTaskOk), + Miss, + Blocked, +} + #[derive(Clone, Copy)] pub(crate) struct RoaTask<'a> { pub(crate) index: usize, @@ -133,6 +450,7 @@ pub(crate) struct RoaTask<'a> { pub(crate) struct RoaTaskOk { pub(crate) vrps: Vec, pub(crate) local_outputs: Vec, + pub(crate) reused_from_cache: bool, } #[derive(Debug)] @@ -180,6 +498,32 @@ pub fn process_publication_point_for_issuer_with_options, collect_vcir_local_outputs: bool, +) -> ObjectsOutput { + process_publication_point_for_issuer_with_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + RoaValidationCacheInput::disabled(), + ) +} + +pub fn process_publication_point_for_issuer_with_cache_options( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, ) -> ObjectsOutput { let manifest_rsync_uri = publication_point.manifest_rsync_uri(); let manifest_bytes = publication_point.manifest_bytes(); @@ -194,6 +538,7 @@ pub fn process_publication_point_for_issuer_with_options = Vec::new(); // Enforce that `manifest_bytes` is actually a manifest object. @@ -243,6 +588,7 @@ pub fn process_publication_point_for_issuer_with_options { @@ -330,6 +678,7 @@ pub fn process_publication_point_for_issuer_with_options = Vec::new(); let mut aspas: Vec = Vec::new(); let mut local_outputs_cache: Vec = Vec::new(); + let active_cache_view = active_roa_cache_view( + roa_cache, + issuer_ca_der, + locked_files, + &mut roa_cache_stats, + stats.roa_total, + ); for (idx, file) in locked_files.iter().enumerate() { if file.rsync_uri.ends_with(".roa") { - let task = RoaTask { index: idx, file }; - let _t = timing.as_ref().map(|t| t.span_phase("objects_roa_total")); - let result = validate_roa_task_serial( - task, - manifest_rsync_uri, - issuer_ca_der, - &issuer_ca, - &issuer_spki, - issuer_ca_rsync_uri, - &mut crl_cache, - &issuer_resources_index, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - collect_vcir_local_outputs, - policy.strict.cms_der, - policy.strict.name, - ); + let result = if let Some(cache_view) = active_cache_view { + let lookup_started = Instant::now(); + let lookup_result = cache_view.lookup(file, validation_time); + roa_cache_stats.lookup_nanos = roa_cache_stats.lookup_nanos.saturating_add( + lookup_started + .elapsed() + .as_nanos() + .min(u128::from(u64::MAX)) as u64, + ); + match lookup_result { + RoaCacheLookupResult::Hit(ok) => { + roa_cache_stats.hit_roas += 1; + RoaTaskResult { + publication_point_id: 0, + index: idx, + worker_index: 0, + queue_wait_ms: 0, + worker_ms: 0, + outcome: Ok(ok), + } + } + RoaCacheLookupResult::Miss => { + roa_cache_stats.miss_roas += 1; + roa_cache_stats.fresh_roas += 1; + let task = RoaTask { index: idx, file }; + let _t = timing.as_ref().map(|t| t.span_phase("objects_roa_total")); + validate_roa_task_serial( + task, + manifest_rsync_uri, + issuer_ca_der, + &issuer_ca, + &issuer_spki, + issuer_ca_rsync_uri, + &mut crl_cache, + &issuer_resources_index, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + policy.strict.cms_der, + policy.strict.name, + ) + } + RoaCacheLookupResult::Blocked => { + roa_cache_stats.blocked_roas += 1; + roa_cache_stats.fresh_roas += 1; + let task = RoaTask { index: idx, file }; + let _t = timing.as_ref().map(|t| t.span_phase("objects_roa_total")); + validate_roa_task_serial( + task, + manifest_rsync_uri, + issuer_ca_der, + &issuer_ca, + &issuer_spki, + issuer_ca_rsync_uri, + &mut crl_cache, + &issuer_resources_index, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + policy.strict.cms_der, + policy.strict.name, + ) + } + } + } else { + let task = RoaTask { index: idx, file }; + let _t = timing.as_ref().map(|t| t.span_phase("objects_roa_total")); + validate_roa_task_serial( + task, + manifest_rsync_uri, + issuer_ca_der, + &issuer_ca, + &issuer_spki, + issuer_ca_rsync_uri, + &mut crl_cache, + &issuer_resources_index, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + policy.strict.cms_der, + policy.strict.name, + ) + }; match result.outcome { Ok(mut ok) => { stats.roa_ok += 1; vrps.append(&mut ok.vrps); - if collect_vcir_local_outputs { + if collect_vcir_local_outputs || ok.reused_from_cache { local_outputs_cache.extend(ok.local_outputs); } audit.push(ObjectAuditEntry { @@ -502,6 +929,7 @@ pub fn process_publication_point_for_issuer_with_options, config: &ParallelPhase2Config, collect_vcir_local_outputs: bool, +) -> ObjectsOutput { + process_publication_point_for_issuer_parallel_roa_with_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + config, + collect_vcir_local_outputs, + RoaValidationCacheInput::disabled(), + ) +} + +pub fn process_publication_point_for_issuer_parallel_roa_with_cache_options< + P: PublicationPointData, +>( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + config: &ParallelPhase2Config, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, ) -> ObjectsOutput { if config.object_workers <= 1 || policy.signed_object_failure_policy == SignedObjectFailurePolicy::DropPublicationPoint { - return process_publication_point_for_issuer_with_options( + return process_publication_point_for_issuer_with_cache_options( publication_point, policy, issuer_ca_der, @@ -676,13 +1138,14 @@ pub fn process_publication_point_for_issuer_parallel_roa_with_options pool, Err(_) => { - return process_publication_point_for_issuer_with_options( + return process_publication_point_for_issuer_with_cache_options( publication_point, policy, issuer_ca_der, @@ -692,11 +1155,12 @@ pub fn process_publication_point_for_issuer_parallel_roa_with_options, pool: &ParallelRoaWorkerPool, collect_vcir_local_outputs: bool, +) -> ObjectsOutput { + process_publication_point_for_issuer_parallel_roa_with_pool_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + pool, + collect_vcir_local_outputs, + RoaValidationCacheInput::disabled(), + ) +} + +pub fn process_publication_point_for_issuer_parallel_roa_with_pool_cache_options< + P: PublicationPointData, +>( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + pool: &ParallelRoaWorkerPool, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, ) -> ObjectsOutput { if policy.signed_object_failure_policy == SignedObjectFailurePolicy::DropPublicationPoint { return process_publication_point_for_issuer_with_options( @@ -774,9 +1269,10 @@ pub fn process_publication_point_for_issuer_parallel_roa_with_pool_options< timing, pool, collect_vcir_local_outputs, + roa_cache, ) .unwrap_or_else(|_| { - process_publication_point_for_issuer_with_options( + process_publication_point_for_issuer_with_cache_options( publication_point, policy, issuer_ca_der, @@ -786,6 +1282,7 @@ pub fn process_publication_point_for_issuer_parallel_roa_with_pool_options< validation_time, timing, collect_vcir_local_outputs, + roa_cache, ) }) } @@ -925,6 +1422,7 @@ fn validate_owned_roa_task(worker_index: usize, task: OwnedRoaTask) -> RoaTaskRe .map(|(vrps, local_outputs)| RoaTaskOk { vrps, local_outputs, + reused_from_cache: false, }); RoaTaskResult { @@ -949,6 +1447,9 @@ pub(crate) struct ParallelObjectsStage { collect_vcir_local_outputs: bool, strict_cms_der: bool, strict_name: bool, + roa_task_indices: Vec, + cached_roa_results: Vec, + roa_cache_stats: RoaValidationCacheStats, warnings: Vec, stats: ObjectsStats, audit: Vec, @@ -974,26 +1475,22 @@ impl ParallelObjectsStage { F: FnMut(OwnedRoaTask), { let shared = self.shared.clone(); - self.locked_files() - .iter() - .enumerate() - .filter(|(_, file)| file.rsync_uri.ends_with(".roa")) - .for_each(|(index, _)| { - push(OwnedRoaTask { - publication_point_id: self.publication_point_id, - index, - shared: shared.clone(), - validation_time: self.validation_time, - collect_vcir_local_outputs: self.collect_vcir_local_outputs, - strict_cms_der: self.strict_cms_der, - strict_name: self.strict_name, - submitted_at: None, - }); + self.roa_task_indices.iter().for_each(|index| { + push(OwnedRoaTask { + publication_point_id: self.publication_point_id, + index: *index, + shared: shared.clone(), + validation_time: self.validation_time, + collect_vcir_local_outputs: self.collect_vcir_local_outputs, + strict_cms_der: self.strict_cms_der, + strict_name: self.strict_name, + submitted_at: None, }); + }); } pub(crate) fn roa_task_count(&self) -> usize { - self.stats.roa_total + self.roa_task_indices.len() } pub(crate) fn aspa_task_count(&self) -> usize { @@ -1003,13 +1500,9 @@ impl ParallelObjectsStage { pub(crate) fn locked_file_count(&self) -> usize { self.shared.locked_files.len() } - - fn locked_files(&self) -> &[PackFile] { - self.shared.locked_files.as_ref() - } } -pub(crate) fn prepare_publication_point_for_parallel_roa( +pub(crate) fn prepare_publication_point_for_parallel_roa_with_cache( publication_point_id: u64, publication_point: &P, policy: &Policy, @@ -1019,6 +1512,7 @@ pub(crate) fn prepare_publication_point_for_parallel_roa, validation_time: time::OffsetDateTime, collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, ) -> ParallelObjectsPrepare { let manifest_rsync_uri = publication_point.manifest_rsync_uri(); let manifest_bytes = publication_point.manifest_bytes(); @@ -1033,6 +1527,7 @@ pub(crate) fn prepare_publication_point_for_parallel_roa = Vec::new(); let _manifest = match ManifestObject::decode_der_with_strict_options( @@ -1081,6 +1576,7 @@ pub(crate) fn prepare_publication_point_for_parallel_roa { @@ -1166,6 +1664,7 @@ pub(crate) fn prepare_publication_point_for_parallel_roa { + roa_cache_stats.hit_roas += 1; + cached_roa_results.push(RoaTaskResult { + publication_point_id, + index, + worker_index: usize::MAX, + queue_wait_ms: 0, + worker_ms: 0, + outcome: Ok(ok), + }); + } + RoaCacheLookupResult::Miss => { + roa_cache_stats.miss_roas += 1; + roa_cache_stats.fresh_roas += 1; + roa_task_indices.push(index); + } + RoaCacheLookupResult::Blocked => { + roa_cache_stats.blocked_roas += 1; + roa_cache_stats.fresh_roas += 1; + roa_task_indices.push(index); + } + } + } else { + roa_task_indices.push(index); + } + } + ParallelObjectsPrepare::Staged(ParallelObjectsStage { publication_point_id, shared: Arc::new(RoaTaskShared { @@ -1245,6 +1795,9 @@ pub(crate) fn prepare_publication_point_for_parallel_roa, timing: Option<&TimingHandle>, ) -> Result { + roa_results.extend(stage.cached_roa_results); roa_results.sort_by_key(|result| result.index); let mut roa_results = roa_results.into_iter().peekable(); let shared = stage.shared.clone(); @@ -1271,6 +1825,7 @@ pub(crate) fn reduce_parallel_roa_stage( let validation_time = stage.validation_time; let strict_cms_der = stage.strict_cms_der; let strict_name = stage.strict_name; + let roa_cache_stats = stage.roa_cache_stats; let mut stats = stage.stats; let mut warnings = stage.warnings; let mut audit = stage.audit; @@ -1298,7 +1853,7 @@ pub(crate) fn reduce_parallel_roa_stage( Ok(mut ok) => { stats.roa_ok += 1; vrps.append(&mut ok.vrps); - if collect_vcir_local_outputs { + if collect_vcir_local_outputs || ok.reused_from_cache { local_outputs_cache.extend(ok.local_outputs); } audit.push(ObjectAuditEntry { @@ -1385,6 +1940,8 @@ pub(crate) fn reduce_parallel_roa_stage( )); } + roa_cache_stats.record_to_timing(timing); + Ok(ObjectsOutput { vrps, aspas, @@ -1393,6 +1950,7 @@ pub(crate) fn reduce_parallel_roa_stage( warnings, stats, audit, + roa_cache_stats, }) } @@ -1407,8 +1965,9 @@ fn process_publication_point_for_issuer_parallel_roa_inner, pool: &ParallelRoaWorkerPool, collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, ) -> Result { - let stage = match prepare_publication_point_for_parallel_roa( + let stage = match prepare_publication_point_for_parallel_roa_with_cache( 0, publication_point, _policy, @@ -1418,6 +1977,7 @@ fn process_publication_point_for_issuer_parallel_roa_inner return Ok(out), ParallelObjectsPrepare::Staged(stage) => stage, @@ -1593,6 +2153,7 @@ pub(crate) fn validate_roa_task_serial( .map(|(vrps, local_outputs)| RoaTaskOk { vrps, local_outputs, + reused_from_cache: false, }); RoaTaskResult { @@ -2499,21 +3060,519 @@ fn roa_afi_to_string(afi: RoaAfi) -> &'static str { #[cfg(test)] mod tests { use super::*; + use crate::analysis::timing::{TimingHandle, TimingMeta}; use crate::data_model::rc::{ Afi, AsIdOrRange, AsIdentifierChoice, IpAddressFamily, IpAddressOrRange, IpAddressRange, IpPrefix, IpResourceSet, }; use crate::policy::Policy; - use crate::storage::PackTime; + use crate::storage::{ + PackTime, ValidatedManifestMeta, VcirAuditSummary, VcirCcrManifestProjection, + VcirInstanceGate, VcirRelatedArtifact, VcirSummary, + }; use crate::validation::publication_point::PublicationPointSnapshot; use std::collections::HashMap; use time::OffsetDateTime; + use time::format_description::well_known::Rfc3339; fn fixture_bytes(path: &str) -> Vec { std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(path)) .unwrap_or_else(|e| panic!("read fixture {path}: {e}")) } + fn fixed_time(value: &str) -> OffsetDateTime { + OffsetDateTime::parse(value, &Rfc3339).expect("parse fixed test time") + } + + fn sample_roa_cache_vcir( + issuer_der: &[u8], + crl_hash: [u8; 32], + roa_hash: [u8; 32], + item_effective_until: OffsetDateTime, + instance_effective_until: OffsetDateTime, + ) -> ValidatedCaInstanceResult { + let manifest_time = PackTime::from_utc_offset_datetime(fixed_time("2026-06-04T00:00:00Z")); + let effective_until = PackTime::from_utc_offset_datetime(instance_effective_until); + ValidatedCaInstanceResult { + manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), + parent_manifest_rsync_uri: Some("rsync://example.test/repo/parent.mft".to_string()), + tal_id: "test-tal".to_string(), + ca_subject_name: "CN=example".to_string(), + ca_ski: "001122".to_string(), + issuer_ski: "334455".to_string(), + last_successful_validation_time: manifest_time.clone(), + current_manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), + current_crl_rsync_uri: "rsync://example.test/repo/current.crl".to_string(), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: vec![1], + validated_manifest_this_update: manifest_time.clone(), + validated_manifest_next_update: effective_until.clone(), + }, + ccr_manifest_projection: VcirCcrManifestProjection { + manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), + manifest_sha256: vec![0xaa; 32], + manifest_size: 2048, + manifest_ee_aki: vec![0xbb; 20], + manifest_number_be: vec![1], + manifest_this_update: manifest_time.clone(), + manifest_sia_locations_der: vec![vec![0x30, 0x00]], + subordinate_skis: Vec::new(), + }, + instance_gate: VcirInstanceGate { + manifest_next_update: effective_until.clone(), + current_crl_next_update: effective_until.clone(), + self_ca_not_after: effective_until.clone(), + instance_effective_until: effective_until, + }, + child_entries: Vec::new(), + local_outputs: vec![VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: PackTime::from_utc_offset_datetime(item_effective_until), + source_object_uri: "rsync://example.test/repo/a.roa".to_string(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: roa_hash, + source_ee_cert_hash: [0xcc; 32], + payload: VcirLocalOutputPayload::Vrp { + asn: 64500, + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + max_length: 24, + }, + rule_hash: [0xdd; 32], + }], + related_artifacts: vec![ + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::IssuerCert, + artifact_kind: VcirArtifactKind::Cer, + uri: Some("rsync://example.test/repo/ca.cer".to_string()), + sha256: sha256_hex(issuer_der), + object_type: Some("cer".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::CurrentCrl, + artifact_kind: VcirArtifactKind::Crl, + uri: Some("rsync://example.test/repo/current.crl".to_string()), + sha256: sha256_hex_from_32(&crl_hash), + object_type: Some("crl".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + }, + ], + summary: VcirSummary { + local_vrp_count: 1, + local_aspa_count: 0, + local_router_key_count: 0, + child_count: 0, + accepted_object_count: 2, + rejected_object_count: 0, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible: true, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: Vec::new(), + }, + } + } + + #[test] + fn roa_validation_cache_view_hits_when_context_and_hash_match() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_hash = [0x22; 32]; + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let view = RoaValidationCacheView::from_vcir(&vcir, validation_time); + let files = vec![ + PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/a.roa", + vec![0x01], + roa_hash, + ), + PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/current.crl", + vec![0x02], + crl_hash, + ), + ]; + + assert!(view.matches_current_context(issuer_der, &files)); + let hit = view.lookup(&files[0], validation_time); + let RoaCacheLookupResult::Hit(ok) = hit else { + panic!("expected cache hit, got {hit:?}"); + }; + assert!(ok.reused_from_cache); + assert_eq!(ok.vrps.len(), 1); + assert_eq!(ok.vrps[0].asn, 64500); + assert_eq!(ok.local_outputs.len(), 1); + } + + #[test] + fn roa_validation_cache_view_blocks_when_context_changes() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_hash = [0x22; 32]; + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let view = RoaValidationCacheView::from_vcir(&vcir, validation_time); + let files = vec![ + PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/a.roa", + vec![0x01], + roa_hash, + ), + PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/current.crl", + vec![0x02], + [0x33; 32], + ), + ]; + let mut stats = + RoaValidationCacheStats::for_input(RoaValidationCacheInput::enabled(Some(&view)), 1); + + assert!(!view.matches_current_context(issuer_der, &files)); + assert!( + active_roa_cache_view( + RoaValidationCacheInput::enabled(Some(&view)), + issuer_der, + &files, + &mut stats, + 1, + ) + .is_none() + ); + assert_eq!(stats.blocked_roas, 1); + assert_eq!(stats.fresh_roas, 1); + } + + #[test] + fn roa_validation_cache_view_blocks_expired_output() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_hash = [0x22; 32]; + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-04T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let view = RoaValidationCacheView::from_vcir(&vcir, validation_time); + let file = PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/a.roa", + vec![0x01], + roa_hash, + ); + + assert!(matches!( + view.lookup(&file, validation_time), + RoaCacheLookupResult::Blocked + )); + } + + #[test] + fn roa_validation_cache_stats_records_vcir_miss_to_timing() { + let stats = RoaValidationCacheStats::for_input(RoaValidationCacheInput::enabled(None), 3); + assert_eq!(stats.enabled_publication_points, 1); + assert_eq!(stats.vcir_miss_publication_points, 1); + assert_eq!(stats.miss_roas, 3); + assert_eq!(stats.fresh_roas, 3); + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + stats.record_to_timing(Some(&timing)); + let dir = tempfile::tempdir().expect("timing dir"); + let path = dir.path().join("timing.json"); + timing.write_json(&path, 10).expect("write timing"); + let report: serde_json::Value = + serde_json::from_slice(&std::fs::read(path).expect("read timing")) + .expect("parse timing"); + + assert_eq!( + report["counts"]["roa_validation_cache_enabled_publication_points"], + 1 + ); + assert_eq!( + report["counts"]["roa_validation_cache_vcir_miss_publication_points"], + 1 + ); + assert_eq!(report["counts"]["roa_validation_cache_miss_roas"], 3); + assert_eq!(report["counts"]["roa_validation_cache_fresh_roas"], 3); + } + + #[test] + fn roa_validation_cache_view_blocks_expired_instance_gate() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_hash = [0x22; 32]; + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-04T00:00:00Z"), + ); + let view = RoaValidationCacheView::from_vcir(&vcir, validation_time); + let file = PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/a.roa", + vec![0x01], + roa_hash, + ); + + assert!(!view.matches_current_context(issuer_der, &[])); + assert!(matches!( + view.lookup(&file, validation_time), + RoaCacheLookupResult::Blocked + )); + } + + #[test] + fn roa_validation_cache_view_ignores_rejected_artifacts_and_non_roa_outputs() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_hash = [0x22; 32]; + let roa_hash = [0x11; 32]; + let mut vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + vcir.related_artifacts.insert( + 0, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::IssuerCert, + artifact_kind: VcirArtifactKind::Cer, + uri: Some("rsync://example.test/repo/rejected.cer".to_string()), + sha256: "00".repeat(32), + object_type: Some("cer".to_string()), + validation_status: VcirArtifactValidationStatus::Rejected, + }, + ); + vcir.local_outputs.push(VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until: PackTime::from_utc_offset_datetime(fixed_time( + "2026-06-07T00:00:00Z", + )), + source_object_uri: "rsync://example.test/repo/a.asa".to_string(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: [0x44; 32], + source_ee_cert_hash: [0x55; 32], + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64500, + provider_as_ids: vec![64501], + }, + rule_hash: [0x66; 32], + }); + let view = RoaValidationCacheView::from_vcir(&vcir, validation_time); + let files = vec![ + PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/a.roa", + vec![0x01], + roa_hash, + ), + PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/current.crl", + vec![0x02], + crl_hash, + ), + ]; + + assert!(view.matches_current_context(issuer_der, &files)); + assert!(matches!( + view.lookup(&files[0], validation_time), + RoaCacheLookupResult::Hit(_) + )); + assert!(matches!( + view.lookup( + &PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/a.asa", + vec![0x03], + [0x44; 32], + ), + validation_time + ), + RoaCacheLookupResult::Miss + )); + } + + #[test] + fn roa_validation_cache_context_rejects_missing_and_changed_issuer() { + let mut view = RoaValidationCacheView { + entries_by_uri: HashMap::new(), + issuer_ca_sha256_hex: None, + crl_sha256_by_uri: HashMap::new(), + blocked: false, + }; + assert!(!view.matches_current_context(b"issuer-ca", &[])); + + view.issuer_ca_sha256_hex = Some("00".repeat(32)); + assert!(!view.matches_current_context(b"issuer-ca", &[])); + + view.issuer_ca_sha256_hex = Some(sha256_hex(b"issuer-ca")); + assert!(view.matches_current_context(b"issuer-ca", &[])); + } + + #[test] + fn roa_validation_cache_lookup_classifies_hash_empty_payload_and_missing_uri() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let roa_hash = [0x11; 32]; + let file = PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/a.roa", + vec![0x01], + roa_hash, + ); + let mut output = sample_roa_cache_vcir( + b"issuer-ca", + [0x22; 32], + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ) + .local_outputs + .remove(0); + let mut view = RoaValidationCacheView { + entries_by_uri: HashMap::new(), + issuer_ca_sha256_hex: Some(sha256_hex(b"issuer-ca")), + crl_sha256_by_uri: HashMap::new(), + blocked: false, + }; + + assert!(matches!( + view.lookup(&file, validation_time), + RoaCacheLookupResult::Miss + )); + + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: [0xff; 32], + outputs: vec![output.clone()], + }, + ); + assert!(matches!( + view.lookup(&file, validation_time), + RoaCacheLookupResult::Blocked + )); + + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: roa_hash, + outputs: Vec::new(), + }, + ); + assert!(matches!( + view.lookup(&file, validation_time), + RoaCacheLookupResult::Miss + )); + + output.payload = VcirLocalOutputPayload::Aspa { + customer_as_id: 64500, + provider_as_ids: vec![64501], + }; + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: roa_hash, + outputs: vec![output], + }, + ); + assert!(matches!( + view.lookup(&file, validation_time), + RoaCacheLookupResult::Blocked + )); + } + + #[test] + fn parallel_roa_cache_blocked_falls_back_to_single_fresh_task() { + let manifest_bytes = fixture_bytes( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft", + ); + let issuer_ca_der = fixture_bytes( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ); + let crl_hash = [0x22; 32]; + let actual_roa_hash = [0x11; 32]; + let cached_roa_hash = [0x33; 32]; + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let publication_point = PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: + "rsync://rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft" + .to_string(), + publication_point_rsync_uri: "rsync://rpki.cernet.net/repo/cernet/0/".to_string(), + manifest_number_be: vec![1], + this_update: PackTime::from_utc_offset_datetime(validation_time), + next_update: PackTime::from_utc_offset_datetime( + validation_time + time::Duration::days(1), + ), + verified_at: PackTime::from_utc_offset_datetime(validation_time), + manifest_bytes, + files: vec![ + PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/current.crl", + vec![0x02], + crl_hash, + ), + PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/a.roa", + vec![0x01], + actual_roa_hash, + ), + ], + }; + let vcir = sample_roa_cache_vcir( + &issuer_ca_der, + crl_hash, + cached_roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let view = RoaValidationCacheView::from_vcir(&vcir, validation_time); + + let stage = match prepare_publication_point_for_parallel_roa_with_cache( + 7, + &publication_point, + &Policy::default(), + &issuer_ca_der, + None, + None, + None, + validation_time, + false, + RoaValidationCacheInput::enabled(Some(&view)), + ) { + ParallelObjectsPrepare::Staged(stage) => stage, + ParallelObjectsPrepare::Complete(_) => panic!("expected staged ROA fallback"), + }; + + assert_eq!(stage.roa_cache_stats.blocked_roas, 1); + assert_eq!(stage.roa_cache_stats.fresh_roas, 1); + assert_eq!(stage.roa_task_indices, vec![1]); + assert_eq!(stage.roa_task_count(), 1); + } + #[test] fn merge_as_intervals_merges_overlapping_and_adjacent() { let v = vec![(1, 2), (3, 5), (10, 10), (11, 12)]; @@ -3015,6 +4074,9 @@ mod tests { collect_vcir_local_outputs: false, strict_cms_der: false, strict_name: false, + roa_task_indices: vec![0, 1], + cached_roa_results: Vec::new(), + roa_cache_stats: RoaValidationCacheStats::default(), warnings: Vec::new(), stats: ObjectsStats { roa_total: 2, diff --git a/src/validation/run.rs b/src/validation/run.rs index 62804cb..fa3c65c 100644 --- a/src/validation/run.rs +++ b/src/validation/run.rs @@ -79,6 +79,7 @@ pub fn run_publication_point_once( parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: false, }; let result = runner diff --git a/src/validation/run_tree_from_tal.rs b/src/validation/run_tree_from_tal.rs index ea4ccf9..8c9e749 100644 --- a/src/validation/run_tree_from_tal.rs +++ b/src/validation/run_tree_from_tal.rs @@ -94,6 +94,7 @@ pub struct RunTreeFromTalAuditOutput { pub successful_tal_inputs: Vec, pub tree: TreeRunOutput, pub publication_points: Vec, + pub roa_cache_stats: crate::validation::objects::RoaValidationCacheStats, pub downloads: Vec, pub download_stats: crate::audit::AuditDownloadStats, pub current_repo_objects: Vec, @@ -132,6 +133,7 @@ fn make_live_runner<'a>( parallel_phase2_config: Option, ccr_accumulator: Option, persist_vcir: bool, + enable_roa_validation_cache: bool, ) -> Rpkiv1PublicationPointRunner<'a> { let parallel_roa_worker_pool = parallel_phase2_config .as_ref() @@ -156,6 +158,7 @@ fn make_live_runner<'a>( parallel_roa_worker_pool, ccr_accumulator: ccr_accumulator.map(Mutex::new), persist_vcir, + enable_roa_validation_cache, } } @@ -186,13 +189,18 @@ where ); let pool = RepoTransportWorkerPool::new(RepoWorkerPoolConfig::from(¶llel_config), executor) .map_err(RunTreeFromTalError::Replay)?; + let resolver_rsync_fetcher_arc = Arc::clone(&rsync_fetcher_arc); let resolver: Arc String + Send + Sync> = - Arc::new(move |base: &str| rsync_fetcher_arc.dedup_key(base)); + Arc::new(move |base: &str| resolver_rsync_fetcher_arc.dedup_key(base)); + let failure_rsync_fetcher_arc = Arc::clone(&rsync_fetcher_arc); + let failure_resolver: Arc Option + Send + Sync> = + Arc::new(move |base: &str| failure_rsync_fetcher_arc.failure_dedup_key(base)); let _ = policy; // policy reserved for later runtime-level decisions - let runtime = Arc::new(Phase1RepoSyncRuntime::new( + let runtime = Arc::new(Phase1RepoSyncRuntime::new_with_failure_scope( coordinator, pool, resolver, + failure_resolver, policy.sync_preference, )); Ok((runtime, current_repo_index)) @@ -419,6 +427,7 @@ pub fn run_tree_from_tal_url_serial( None, None, config.persist_vcir, + config.enable_roa_validation_cache, ); let root = root_handle_from_trust_anchor( @@ -458,6 +467,7 @@ pub fn run_tree_from_tal_url_serial_audit( None, None, config.persist_vcir, + config.enable_roa_validation_cache, ); let root = root_handle_from_trust_anchor( @@ -469,6 +479,7 @@ pub fn run_tree_from_tal_url_serial_audit( let TreeRunAuditOutput { tree, publication_points, + roa_cache_stats, } = run_tree_serial_audit(root, &runner, config)?; let downloads = download_log.snapshot_events(); @@ -479,6 +490,7 @@ pub fn run_tree_from_tal_url_serial_audit( successful_tal_inputs: Vec::new(), tree, publication_points, + roa_cache_stats, downloads, download_stats, current_repo_objects: Vec::new(), @@ -515,6 +527,7 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing( None, None, config.persist_vcir, + config.enable_roa_validation_cache, ); let root = root_handle_from_trust_anchor( @@ -527,6 +540,7 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing( let TreeRunAuditOutput { tree, publication_points, + roa_cache_stats, } = run_tree_serial_audit(root, &runner, config)?; let downloads = download_log.snapshot_events(); @@ -537,6 +551,7 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing( successful_tal_inputs: Vec::new(), tree, publication_points, + roa_cache_stats, downloads, download_stats, current_repo_objects: Vec::new(), @@ -556,6 +571,7 @@ fn run_single_root_parallel_audit_inner( parallel_config: ParallelPhase1Config, phase2_config: Option, collect_current_repo_objects: bool, + timing: Option, ) -> Result where H: Fetcher + Clone + 'static, @@ -569,7 +585,7 @@ where http_fetcher, rsync_fetcher, parallel_config, - None, + timing.clone(), Some(download_log.clone()), tal_inputs, )?; @@ -580,7 +596,7 @@ where http_fetcher, rsync_fetcher, validation_time, - None, + timing, Some(download_log.clone()), Some(current_repo_index), Some(runtime), @@ -588,6 +604,7 @@ where (phase2_enabled && config.build_ccr_accumulator) .then(|| CcrAccumulator::new(vec![discovery.trust_anchor.clone()])), config.persist_vcir, + config.enable_roa_validation_cache, ); let root = root_handle_from_trust_anchor( @@ -599,6 +616,7 @@ where let TreeRunAuditOutput { tree, publication_points, + roa_cache_stats, } = if phase2_enabled { run_tree_parallel_phase2_audit(root, &runner, config)? } else { @@ -612,6 +630,7 @@ where successful_tal_inputs: Vec::new(), tree, publication_points, + roa_cache_stats, downloads, download_stats, current_repo_objects: snapshot_current_repo_objects( @@ -633,6 +652,7 @@ fn run_multi_root_parallel_audit_inner( parallel_config: ParallelPhase1Config, phase2_config: Option, collect_current_repo_objects: bool, + timing: Option, ) -> Result where H: Fetcher + Clone + 'static, @@ -673,7 +693,7 @@ where http_fetcher, rsync_fetcher, parallel_config, - None, + timing.clone(), Some(download_log.clone()), successful_tal_inputs.clone(), )?; @@ -684,7 +704,7 @@ where http_fetcher, rsync_fetcher, validation_time, - None, + timing, Some(download_log.clone()), Some(current_repo_index), Some(runtime), @@ -698,11 +718,13 @@ where ) }), config.persist_vcir, + config.enable_roa_validation_cache, ); let TreeRunAuditOutput { tree, publication_points, + roa_cache_stats, } = if phase2_enabled { run_tree_parallel_phase2_audit_multi_root(root_handles, &runner, config)? } else { @@ -716,6 +738,7 @@ where successful_tal_inputs, tree, publication_points, + roa_cache_stats, downloads, download_stats, current_repo_objects: snapshot_current_repo_objects( @@ -755,6 +778,7 @@ where parallel_config, None, collect_current_repo_objects, + None, ) } @@ -806,6 +830,7 @@ where parallel_config, None, collect_current_repo_objects, + None, ) } @@ -835,6 +860,7 @@ where parallel_config, None, collect_current_repo_objects, + None, ) } @@ -868,6 +894,42 @@ where parallel_config, Some(phase2_config), collect_current_repo_objects, + None, + ) +} + +pub fn run_tree_from_tal_url_parallel_phase2_audit_with_timing( + store: Arc, + policy: &crate::policy::Policy, + tal_url: &str, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: ParallelPhase2Config, + collect_current_repo_objects: bool, + timing: &TimingHandle, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let discovery = + discover_root_ca_instance_from_tal_url_with_policy(policy, http_fetcher, tal_url)?; + run_single_root_parallel_audit_inner( + store, + policy, + discovery, + vec![TalInputSpec::from_url(tal_url.to_string())], + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + Some(phase2_config), + collect_current_repo_objects, + Some(timing.clone()), ) } @@ -920,6 +982,61 @@ where parallel_config, Some(phase2_config), collect_current_repo_objects, + None, + ) +} + +pub fn run_tree_from_tal_and_ta_der_parallel_phase2_audit_with_timing( + store: Arc, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&url::Url>, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: ParallelPhase2Config, + collect_current_repo_objects: bool, + timing: &TimingHandle, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + let derived_tal_id = derive_tal_id(&discovery); + let tal_inputs = vec![TalInputSpec { + tal_id: derived_tal_id.clone(), + rir_id: derived_tal_id, + source: TalSource::DerBytes { + tal_url: discovery + .tal_url + .clone() + .unwrap_or_else(|| "embedded-tal".to_string()), + tal_bytes: tal_bytes.to_vec(), + ta_der: ta_der.to_vec(), + }, + }]; + run_single_root_parallel_audit_inner( + store, + policy, + discovery, + tal_inputs, + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + Some(phase2_config), + collect_current_repo_objects, + Some(timing.clone()), ) } @@ -950,6 +1067,39 @@ where parallel_config, Some(phase2_config), collect_current_repo_objects, + None, + ) +} + +pub fn run_tree_from_multiple_tals_parallel_phase2_audit_with_timing( + store: Arc, + policy: &crate::policy::Policy, + tal_inputs: Vec, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: ParallelPhase2Config, + collect_current_repo_objects: bool, + timing: &TimingHandle, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + run_multi_root_parallel_audit_inner( + store, + policy, + tal_inputs, + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + Some(phase2_config), + collect_current_repo_objects, + Some(timing.clone()), ) } @@ -991,6 +1141,7 @@ pub fn run_tree_from_tal_and_ta_der_serial( parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, }; let root = root_handle_from_trust_anchor( @@ -1043,6 +1194,7 @@ pub fn run_tree_from_tal_bytes_serial_audit( parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, }; let root = root_handle_from_trust_anchor( @@ -1054,6 +1206,7 @@ pub fn run_tree_from_tal_bytes_serial_audit( let TreeRunAuditOutput { tree, publication_points, + roa_cache_stats, } = run_tree_serial_audit(root, &runner, config)?; let downloads = download_log.snapshot_events(); @@ -1064,6 +1217,7 @@ pub fn run_tree_from_tal_bytes_serial_audit( successful_tal_inputs: Vec::new(), tree, publication_points, + roa_cache_stats, downloads, download_stats, current_repo_objects: Vec::new(), @@ -1113,6 +1267,7 @@ pub fn run_tree_from_tal_bytes_serial_audit_with_timing( parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, }; let root = root_handle_from_trust_anchor( @@ -1125,6 +1280,7 @@ pub fn run_tree_from_tal_bytes_serial_audit_with_timing( let TreeRunAuditOutput { tree, publication_points, + roa_cache_stats, } = run_tree_serial_audit(root, &runner, config)?; drop(_tree); @@ -1136,6 +1292,7 @@ pub fn run_tree_from_tal_bytes_serial_audit_with_timing( successful_tal_inputs: Vec::new(), tree, publication_points, + roa_cache_stats, downloads, download_stats, current_repo_objects: Vec::new(), @@ -1182,6 +1339,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit( parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, }; let root = root_handle_from_trust_anchor( @@ -1193,6 +1351,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit( let TreeRunAuditOutput { tree, publication_points, + roa_cache_stats, } = run_tree_serial_audit(root, &runner, config)?; let downloads = download_log.snapshot_events(); @@ -1203,6 +1362,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit( successful_tal_inputs: Vec::new(), tree, publication_points, + roa_cache_stats, downloads, download_stats, current_repo_objects: Vec::new(), @@ -1252,6 +1412,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing( parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, }; let root = root_handle_from_trust_anchor( @@ -1264,6 +1425,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing( let TreeRunAuditOutput { tree, publication_points, + roa_cache_stats, } = run_tree_serial_audit(root, &runner, config)?; let downloads = download_log.snapshot_events(); @@ -1274,6 +1436,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing( successful_tal_inputs: Vec::new(), tree, publication_points, + roa_cache_stats, downloads, download_stats, current_repo_objects: Vec::new(), @@ -1329,6 +1492,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial( parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, }; let root = root_handle_from_trust_anchor( @@ -1391,6 +1555,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit( parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, }; let root = root_handle_from_trust_anchor( @@ -1402,6 +1567,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit( let TreeRunAuditOutput { tree, publication_points, + roa_cache_stats, } = run_tree_serial_audit(root, &runner, config)?; let downloads = download_log.snapshot_events(); @@ -1412,6 +1578,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit( successful_tal_inputs: Vec::new(), tree, publication_points, + roa_cache_stats, downloads, download_stats, current_repo_objects: Vec::new(), @@ -1471,6 +1638,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing( parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, }; let root = root_handle_from_trust_anchor( @@ -1483,6 +1651,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing( let TreeRunAuditOutput { tree, publication_points, + roa_cache_stats, } = run_tree_serial_audit(root, &runner, config)?; let downloads = download_log.snapshot_events(); @@ -1493,6 +1662,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing( successful_tal_inputs: Vec::new(), tree, publication_points, + roa_cache_stats, downloads, download_stats, current_repo_objects: Vec::new(), @@ -1509,6 +1679,7 @@ fn build_payload_replay_runner<'a>( validation_time: time::OffsetDateTime, timing: Option, download_log: Option, + enable_roa_validation_cache: bool, ) -> Rpkiv1PublicationPointRunner<'a> { Rpkiv1PublicationPointRunner { store, @@ -1530,6 +1701,7 @@ fn build_payload_replay_runner<'a>( parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache, } } @@ -1542,6 +1714,7 @@ fn build_payload_delta_replay_runner<'a>( validation_time: time::OffsetDateTime, timing: Option, download_log: Option, + enable_roa_validation_cache: bool, ) -> Rpkiv1PublicationPointRunner<'a> { Rpkiv1PublicationPointRunner { store, @@ -1563,6 +1736,7 @@ fn build_payload_delta_replay_runner<'a>( parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache, } } @@ -1575,6 +1749,7 @@ fn build_payload_delta_replay_current_store_runner<'a>( validation_time: time::OffsetDateTime, timing: Option, download_log: Option, + enable_roa_validation_cache: bool, ) -> Rpkiv1PublicationPointRunner<'a> { Rpkiv1PublicationPointRunner { store, @@ -1596,6 +1771,7 @@ fn build_payload_delta_replay_current_store_runner<'a>( parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache, } } @@ -1648,6 +1824,7 @@ fn run_payload_delta_replay_audit_inner( base_validation_time, Some(t.clone()), None, + config.enable_roa_validation_cache, ); let _base = run_tree_serial(root.clone(), &base_runner, config)?; } else { @@ -1660,6 +1837,7 @@ fn run_payload_delta_replay_audit_inner( base_validation_time, None, None, + config.enable_roa_validation_cache, ); let _base = run_tree_serial(root.clone(), &base_runner, config)?; } @@ -1668,7 +1846,7 @@ fn run_payload_delta_replay_audit_inner( .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; let delta_rsync_fetcher = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); let download_log = DownloadLogHandle::new(); - let (tree, publication_points) = if let Some(t) = timing.as_ref() { + let (tree, publication_points, roa_cache_stats) = if let Some(t) = timing.as_ref() { let _phase = t.span_phase("payload_delta_replay_target_total"); let delta_runner = build_payload_delta_replay_runner( store, @@ -1679,12 +1857,14 @@ fn run_payload_delta_replay_audit_inner( base_validation_time, Some(t.clone()), Some(download_log.clone()), + config.enable_roa_validation_cache, ); let TreeRunAuditOutput { tree, publication_points, + roa_cache_stats, } = run_tree_serial_audit(root, &delta_runner, config)?; - (tree, publication_points) + (tree, publication_points, roa_cache_stats) } else { let delta_runner = build_payload_delta_replay_runner( store, @@ -1695,12 +1875,14 @@ fn run_payload_delta_replay_audit_inner( validation_time, None, Some(download_log.clone()), + config.enable_roa_validation_cache, ); let TreeRunAuditOutput { tree, publication_points, + roa_cache_stats, } = run_tree_serial_audit(root, &delta_runner, config)?; - (tree, publication_points) + (tree, publication_points, roa_cache_stats) }; let downloads = download_log.snapshot_events(); let download_stats = DownloadLogHandle::stats_from_events(&downloads); @@ -1710,6 +1892,7 @@ fn run_payload_delta_replay_audit_inner( successful_tal_inputs: Vec::new(), tree, publication_points, + roa_cache_stats, downloads, download_stats, current_repo_objects: Vec::new(), @@ -1822,7 +2005,7 @@ fn run_payload_delta_replay_step_audit_inner( PayloadDeltaReplayCurrentStoreRsyncFetcher::new(store, delta_index.clone()); let download_log = DownloadLogHandle::new(); - let (tree, publication_points) = if let Some(t) = timing.as_ref() { + let (tree, publication_points, roa_cache_stats) = if let Some(t) = timing.as_ref() { let _phase = t.span_phase("payload_delta_replay_step_total"); let delta_runner = build_payload_delta_replay_current_store_runner( store, @@ -1833,12 +2016,14 @@ fn run_payload_delta_replay_step_audit_inner( validation_time, Some(t.clone()), Some(download_log.clone()), + config.enable_roa_validation_cache, ); let TreeRunAuditOutput { tree, publication_points, + roa_cache_stats, } = run_tree_serial_audit(root, &delta_runner, config)?; - (tree, publication_points) + (tree, publication_points, roa_cache_stats) } else { let delta_runner = build_payload_delta_replay_current_store_runner( store, @@ -1849,12 +2034,14 @@ fn run_payload_delta_replay_step_audit_inner( validation_time, None, Some(download_log.clone()), + config.enable_roa_validation_cache, ); let TreeRunAuditOutput { tree, publication_points, + roa_cache_stats, } = run_tree_serial_audit(root, &delta_runner, config)?; - (tree, publication_points) + (tree, publication_points, roa_cache_stats) }; let downloads = download_log.snapshot_events(); let download_stats = DownloadLogHandle::stats_from_events(&downloads); @@ -1864,6 +2051,7 @@ fn run_payload_delta_replay_step_audit_inner( successful_tal_inputs: Vec::new(), tree, publication_points, + roa_cache_stats, downloads, download_stats, current_repo_objects: Vec::new(), @@ -2140,6 +2328,7 @@ mod replay_api_tests { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, ) .unwrap_err(); @@ -2175,6 +2364,7 @@ mod replay_api_tests { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, ) .expect("run replay root-only audit"); @@ -2220,6 +2410,7 @@ mod replay_api_tests { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, ) .expect("run replay root-only audit"); @@ -2266,6 +2457,7 @@ mod replay_api_tests { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, &timing, ) @@ -2322,6 +2514,7 @@ mod replay_api_tests { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, ) .unwrap_err(); @@ -2354,6 +2547,7 @@ mod replay_api_tests { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, ) .unwrap_err(); @@ -2406,6 +2600,7 @@ mod replay_api_tests { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, ) .expect("run delta replay root-only audit"); @@ -2467,6 +2662,7 @@ mod replay_api_tests { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, &timing, ) diff --git a/src/validation/tree.rs b/src/validation/tree.rs index d66beea..ab4bf33 100644 --- a/src/validation/tree.rs +++ b/src/validation/tree.rs @@ -3,7 +3,9 @@ use crate::audit::PublicationPointAudit; use crate::data_model::rc::{AsResourceSet, IpResourceSet}; use crate::report::Warning; use crate::validation::manifest::PublicationPointSource; -use crate::validation::objects::{AspaAttestation, ObjectsOutput, RouterKeyPayload, Vrp}; +use crate::validation::objects::{ + AspaAttestation, ObjectsOutput, RoaValidationCacheStats, RouterKeyPayload, Vrp, +}; use crate::validation::publication_point::PublicationPointSnapshot; #[derive(Clone, Debug, PartialEq, Eq)] @@ -18,6 +20,8 @@ pub struct TreeRunConfig { pub persist_vcir: bool, /// Build online CCR manifest projections during phase2 validation. pub build_ccr_accumulator: bool, + /// Reuse accepted ROA validation outputs from previous VCIR when explicitly enabled. + pub enable_roa_validation_cache: bool, } impl Default for TreeRunConfig { @@ -28,6 +32,7 @@ impl Default for TreeRunConfig { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, } } } @@ -117,6 +122,7 @@ pub trait PublicationPointRunner { pub struct TreeRunAuditOutput { pub tree: TreeRunOutput, pub publication_points: Vec, + pub roa_cache_stats: RoaValidationCacheStats, } pub fn run_tree_serial( @@ -169,6 +175,7 @@ pub fn run_tree_serial_audit_multi_root( let mut aspas: Vec = Vec::new(); let mut router_keys: Vec = Vec::new(); let mut publication_points: Vec = Vec::new(); + let mut roa_cache_stats = RoaValidationCacheStats::default(); while let Some(node) = queue.pop_front() { let ca = &node.handle; @@ -203,6 +210,7 @@ pub fn run_tree_serial_audit_multi_root( instances_processed += 1; warnings.extend(res.warnings); warnings.extend(res.objects.warnings); + roa_cache_stats.add_assign(&res.objects.roa_cache_stats); vrps.extend(res.objects.vrps); aspas.extend(res.objects.aspas); router_keys.extend(res.objects.router_keys); @@ -255,6 +263,7 @@ pub fn run_tree_serial_audit_multi_root( router_keys, }, publication_points, + roa_cache_stats, }) } @@ -338,6 +347,7 @@ mod tests { warnings: Vec::new(), stats: ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), }, audit: PublicationPointAudit::default(), discovered_children: children, diff --git a/src/validation/tree_parallel.rs b/src/validation/tree_parallel.rs index 3617949..c5fd34c 100644 --- a/src/validation/tree_parallel.rs +++ b/src/validation/tree_parallel.rs @@ -8,9 +8,11 @@ use crate::parallel::repo_runtime::{RepoSyncRequestStatus, RepoSyncRuntimeOutcom use crate::parallel::types::RepoIdentity; use crate::policy::SignedObjectFailurePolicy; use crate::report::Warning; +use crate::validation::manifest::PublicationPointData; use crate::validation::objects::{ ObjectsOutput, OwnedRoaTask, ParallelObjectsPrepare, ParallelObjectsStage, - prepare_publication_point_for_parallel_roa, reduce_parallel_roa_stage, + RoaValidationCacheInput, prepare_publication_point_for_parallel_roa_with_cache, + reduce_parallel_roa_stage, }; use crate::validation::tree::{ CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult, PublicationPointRunner, @@ -888,7 +890,32 @@ fn stage_ready_publication_point( metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); let prepare_started = Instant::now(); - match prepare_publication_point_for_parallel_roa( + let has_roa = fresh_stage + .fresh_point + .files() + .iter() + .any(|file| file.rsync_uri.ends_with(".roa")); + if runner.enable_roa_validation_cache { + if let Some(timing) = runner.timing.as_ref() { + if has_roa { + timing.record_count("roa_validation_cache_roa_candidate_publication_points", 1); + } else { + timing.record_count("roa_validation_cache_skipped_no_roa_publication_points", 1); + } + } + } + let roa_cache_view = if has_roa { + runner + .roa_validation_cache_view_for_fresh_point(&fresh_stage.fresh_point.manifest_rsync_uri) + } else { + None + }; + let roa_cache = if runner.enable_roa_validation_cache && has_roa { + RoaValidationCacheInput::enabled(roa_cache_view.as_ref()) + } else { + RoaValidationCacheInput::disabled() + }; + match prepare_publication_point_for_parallel_roa_with_cache( ready.node.id, &fresh_stage.fresh_point, runner.policy, @@ -898,6 +925,7 @@ fn stage_ready_publication_point( ready.node.handle.effective_as_resources.as_ref(), runner.validation_time, runner.persist_vcir, + roa_cache, ) { ParallelObjectsPrepare::Complete(mut objects) => { metrics.prepare_ms = elapsed_ms(prepare_started); @@ -1571,6 +1599,7 @@ fn build_tree_output(mut finished: Vec) -> TreeRunAudi let mut aspas = Vec::new(); let mut router_keys = Vec::new(); let mut publication_points = Vec::new(); + let mut roa_cache_stats = crate::validation::objects::RoaValidationCacheStats::default(); for item in finished { match item.result { @@ -1582,6 +1611,7 @@ fn build_tree_output(mut finished: Vec) -> TreeRunAudi instances_processed += 1; warnings.extend(result_warnings); warnings.extend(objects.warnings); + roa_cache_stats.add_assign(&objects.roa_cache_stats); vrps.extend(objects.vrps); aspas.extend(objects.aspas); router_keys.extend(objects.router_keys); @@ -1612,6 +1642,7 @@ fn build_tree_output(mut finished: Vec) -> TreeRunAudi router_keys, }, publication_points, + roa_cache_stats, } } @@ -1669,6 +1700,7 @@ mod tests { warnings: Vec::new(), stats: ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), }, audit: PublicationPointAudit::default(), discovered_children: Vec::new(), diff --git a/src/validation/tree_runner.rs b/src/validation/tree_runner.rs index 4a6c4cc..0c36782 100644 --- a/src/validation/tree_runner.rs +++ b/src/validation/tree_runner.rs @@ -45,9 +45,9 @@ use crate::validation::manifest::{ process_manifest_publication_point_fresh_after_repo_sync_with_timing, }; use crate::validation::objects::{ - AspaAttestation, ParallelRoaWorkerPool, RouterKeyPayload, Vrp, - process_publication_point_for_issuer_parallel_roa_with_options, - process_publication_point_for_issuer_parallel_roa_with_pool_options, + AspaAttestation, ParallelRoaWorkerPool, RoaValidationCacheInput, RoaValidationCacheView, + RouterKeyPayload, Vrp, process_publication_point_for_issuer_parallel_roa_with_cache_options, + process_publication_point_for_issuer_parallel_roa_with_pool_cache_options, }; use crate::validation::publication_point::PublicationPointSnapshot; use crate::validation::tree::{ @@ -151,9 +151,59 @@ pub struct Rpkiv1PublicationPointRunner<'a> { /// This is intended for replay/compare-only runs where the caller does not need /// the resulting DB to be reused by a later delta run. pub persist_vcir: bool, + pub enable_roa_validation_cache: bool, } impl<'a> Rpkiv1PublicationPointRunner<'a> { + pub(crate) fn roa_validation_cache_view_for_fresh_point( + &self, + manifest_rsync_uri: &str, + ) -> Option { + if !self.enable_roa_validation_cache { + return None; + } + let load_started = std::time::Instant::now(); + let loaded_vcir = self.store.get_vcir(manifest_rsync_uri); + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "roa_validation_cache_vcir_load_total", + load_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64, + ); + } + match loaded_vcir { + Ok(Some(vcir)) => { + let view_started = std::time::Instant::now(); + let view = RoaValidationCacheView::from_vcir(&vcir, self.validation_time); + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "roa_validation_cache_view_build_total", + view_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64, + ); + } + Some(view) + } + Ok(None) => { + if let Some(timing) = self.timing.as_ref() { + timing.record_count("roa_validation_cache_vcir_missing_publication_points", 1); + } + None + } + Err(err) => { + if let Some(timing) = self.timing.as_ref() { + timing.record_count("roa_validation_cache_vcir_load_errors", 1); + } + crate::progress_log::emit( + "roa_validation_cache_vcir_load_error", + serde_json::json!({ + "manifest_rsync_uri": manifest_rsync_uri, + "error": err.to_string(), + }), + ); + None + } + } + } + pub(crate) fn ccr_accumulator_snapshot(&self) -> Option { self.ccr_accumulator .as_ref() @@ -653,6 +703,35 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> { } = stage; warnings.extend(stage_warnings); + let has_roa = fresh_point + .files() + .iter() + .any(|file| file.rsync_uri.ends_with(".roa")); + if self.enable_roa_validation_cache { + if let Some(timing) = self.timing.as_ref() { + if has_roa { + timing.record_count( + "roa_validation_cache_roa_candidate_publication_points", + 1, + ); + } else { + timing.record_count( + "roa_validation_cache_skipped_no_roa_publication_points", + 1, + ); + } + } + } + let roa_cache_view = if has_roa { + self.roa_validation_cache_view_for_fresh_point(fresh_point.manifest_rsync_uri()) + } else { + None + }; + let roa_cache = if self.enable_roa_validation_cache && has_roa { + RoaValidationCacheInput::enabled(roa_cache_view.as_ref()) + } else { + RoaValidationCacheInput::disabled() + }; let objects_processing_started = std::time::Instant::now(); let mut objects = { let _objects_total = self @@ -660,7 +739,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> { .as_ref() .map(|t| t.span_phase("objects_processing_total")); if let Some(phase2_pool) = self.parallel_roa_worker_pool.as_ref() { - process_publication_point_for_issuer_parallel_roa_with_pool_options( + process_publication_point_for_issuer_parallel_roa_with_pool_cache_options( &fresh_point, self.policy, &ca.ca_certificate_der, @@ -671,9 +750,10 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> { self.timing.as_ref(), phase2_pool, false, + roa_cache, ) } else if let Some(phase2_config) = self.parallel_phase2_config.as_ref() { - process_publication_point_for_issuer_parallel_roa_with_options( + process_publication_point_for_issuer_parallel_roa_with_cache_options( &fresh_point, self.policy, &ca.ca_certificate_der, @@ -684,9 +764,10 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> { self.timing.as_ref(), phase2_config, false, + roa_cache, ) } else { - crate::validation::objects::process_publication_point_for_issuer_with_options( + crate::validation::objects::process_publication_point_for_issuer_with_cache_options( &fresh_point, self.policy, &ca.ca_certificate_der, @@ -696,6 +777,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> { self.validation_time, self.timing.as_ref(), false, + roa_cache, ) } }; @@ -2017,6 +2099,7 @@ fn empty_objects_output() -> crate::validation::objects::ObjectsOutput { warnings: Vec::new(), stats: crate::validation::objects::ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), } } @@ -2549,29 +2632,6 @@ fn restore_children_from_vcir( (children, audits) } -fn persist_vcir_for_fresh_result( - store: &RocksStore, - ca: &CaInstanceHandle, - pack: &PublicationPointSnapshot, - objects: &mut crate::validation::objects::ObjectsOutput, - warnings: &[Warning], - child_audits: &[ObjectAuditEntry], - discovered_children: &[DiscoveredChildCaInstance], - validation_time: time::OffsetDateTime, -) -> Result<(), String> { - persist_vcir_for_fresh_result_with_timing( - store, - ca, - pack, - objects, - warnings, - child_audits, - discovered_children, - validation_time, - ) - .map(|_timing| ()) -} - fn persist_vcir_for_fresh_result_with_timing( store: &RocksStore, ca: &CaInstanceHandle, @@ -2616,27 +2676,6 @@ fn persist_vcir_for_fresh_result_with_timing( Ok(timing) } -fn build_vcir_from_fresh_result( - ca: &CaInstanceHandle, - pack: &PublicationPointSnapshot, - objects: &mut crate::validation::objects::ObjectsOutput, - warnings: &[Warning], - child_audits: &[ObjectAuditEntry], - discovered_children: &[DiscoveredChildCaInstance], - validation_time: time::OffsetDateTime, -) -> Result { - build_vcir_from_fresh_result_with_timing( - ca, - pack, - objects, - warnings, - child_audits, - discovered_children, - validation_time, - ) - .map(|(vcir, _timing)| vcir) -} - fn build_vcir_from_fresh_result_with_timing( ca: &CaInstanceHandle, pack: &PublicationPointSnapshot, diff --git a/src/validation/tree_runner/tests.rs b/src/validation/tree_runner/tests.rs index 41c35a9..8d9c32d 100644 --- a/src/validation/tree_runner/tests.rs +++ b/src/validation/tree_runner/tests.rs @@ -68,6 +68,7 @@ fn sample_runner_with_ccr_accumulator<'a>( parallel_roa_worker_pool: None, ccr_accumulator: Some(Mutex::new(CcrAccumulator::new(Vec::new()))), persist_vcir: true, + enable_roa_validation_cache: false, } } @@ -762,6 +763,7 @@ fn build_vcir_local_outputs_prefers_cached_outputs() { warnings: Vec::new(), stats: crate::validation::objects::ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), }, ) .expect("reuse cached outputs"); @@ -957,6 +959,7 @@ fn finalize_fresh_publication_point_releases_local_outputs_cache_after_persist() parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: false, }; let ca = CaInstanceHandle { depth: 0, @@ -1057,7 +1060,7 @@ fn persist_vcir_for_fresh_result_stores_vcir_and_replay_meta_for_real_snapshot() }; let mut objects = objects; - persist_vcir_for_fresh_result( + persist_vcir_for_fresh_result_with_timing( &store, &ca, &pack, @@ -1067,6 +1070,7 @@ fn persist_vcir_for_fresh_result_stores_vcir_and_replay_meta_for_real_snapshot() &[], validation_time, ) + .map(|_timing| ()) .expect("persist vcir for fresh result"); let vcir = store @@ -1265,6 +1269,7 @@ fn build_vcir_related_artifacts_classifies_snapshot_files_and_audit_statuses() { detail: Some("skipped aspa".to_string()), }, ], + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), }; let artifacts = build_vcir_related_artifacts( &ca, @@ -1516,6 +1521,7 @@ fn runner_offline_rsync_fixture_produces_pack_and_warnings() { parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: false, }; // For this fixture-driven smoke, we provide the correct issuer CA certificate (the CA for @@ -1577,10 +1583,123 @@ fn runner_offline_rsync_fixture_produces_pack_and_warnings() { .get_manifest_replay_meta(&manifest_rsync_uri) .expect("get replay meta") .expect("replay meta exists"); + assert_eq!(replay_meta.manifest_rsync_uri, manifest_rsync_uri); +} + +#[test] +fn runner_roa_validation_cache_reuses_vcir_outputs_on_second_fixture_run() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate_der: issuer_ca_der, + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri, + rrdp_notification_uri: None, + }; + + let first_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + }; + let first = first_runner + .run_publication_point(&handle) + .expect("first fresh run"); + assert!(first.objects.vrps.len() > 1); + assert_eq!(first.objects.roa_cache_stats.hit_roas, 0); + + let second_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: true, + }; + let second = second_runner + .run_publication_point(&handle) + .expect("second cache-enabled run"); + + assert_eq!(second.objects.vrps, first.objects.vrps); + assert_eq!(second.objects.roa_cache_stats.enabled_publication_points, 1); assert_eq!( - replay_meta.manifest_rsync_uri, - manifest_rsync_uri + second.objects.roa_cache_stats.vcir_hit_publication_points, + 1 ); + assert_eq!( + second.objects.roa_cache_stats.vcir_miss_publication_points, + 0 + ); + assert!(second.objects.roa_cache_stats.hit_roas > 1); + assert_eq!(second.objects.roa_cache_stats.miss_roas, 0); + assert_eq!(second.objects.roa_cache_stats.blocked_roas, 0); + assert_eq!(second.objects.roa_cache_stats.fresh_roas, 0); } #[test] @@ -1669,6 +1788,7 @@ fn runner_rsync_dedup_skips_second_sync_for_same_base() { parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: false, }; let first = runner.run_publication_point(&handle).expect("first run ok"); @@ -1782,6 +1902,7 @@ fn runner_rsync_dedup_skips_second_sync_for_same_module_scope() { parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: false, }; let first = runner.run_publication_point(&handle).expect("first run ok"); @@ -1898,6 +2019,7 @@ fn runner_rsync_dedup_works_in_rsync_only_mode_even_when_rrdp_notify_exists() { parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: false, }; let first = runner.run_publication_point(&handle).expect("first run ok"); @@ -1985,6 +2107,7 @@ fn runner_when_repo_sync_fails_uses_current_instance_vcir_and_keeps_children_emp parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: false, }; let first = ok_runner .run_publication_point(&handle) @@ -2016,6 +2139,7 @@ fn runner_when_repo_sync_fails_uses_current_instance_vcir_and_keeps_children_emp parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: false, }; let second = bad_runner .run_publication_point(&handle) @@ -2063,6 +2187,7 @@ fn build_publication_point_audit_emits_no_audit_entry_for_duplicate_pack_uri() { warnings: Vec::new(), stats: crate::validation::objects::ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), }; let audit = build_publication_point_audit_from_snapshot( @@ -2131,6 +2256,7 @@ fn build_publication_point_audit_marks_invalid_crl_as_error_and_overlays_roa_aud result: AuditObjectResult::Ok, detail: None, }], + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), }; let audit = build_publication_point_audit_from_snapshot( @@ -2822,7 +2948,7 @@ fn fresh_and_reuse_paths_produce_equivalent_ccr_manifest_projection() { discover_children_from_fresh_snapshot_with_audit(&ca, &pack, validation_time, None) .expect("discover children"); let mut objects = empty_objects_output(); - let fresh_vcir = build_vcir_from_fresh_result( + let (fresh_vcir, _timing) = build_vcir_from_fresh_result_with_timing( &ca, &pack, &mut objects, @@ -3097,6 +3223,7 @@ fn build_publication_point_audit_from_vcir_uses_vcir_metadata_and_overlays_child result: AuditObjectResult::Error, detail: Some("overridden from object audit".to_string()), }], + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), }; let child_audits = vec![ObjectAuditEntry { rsync_uri: vcir.child_entries[0].child_cert_rsync_uri.clone(), @@ -3192,6 +3319,7 @@ fn build_publication_point_audit_from_vcir_failed_no_cache_keeps_current_reject_ result: AuditObjectResult::Error, detail: Some("manifest is not valid at validation_time".to_string()), }], + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), }; let audit = build_publication_point_audit_from_vcir( @@ -3327,6 +3455,7 @@ fn build_publication_point_audit_from_vcir_without_cached_inputs_returns_empty_l warnings: vec![Warning::new("object warning")], stats: crate::validation::objects::ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), }, &[], ); @@ -3609,6 +3738,7 @@ fn runner_dedup_paths_execute_with_timing_enabled() { parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: false, }; let first = runner_rrdp .run_publication_point(&handle) @@ -3643,6 +3773,7 @@ fn runner_dedup_paths_execute_with_timing_enabled() { parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: false, }; let third = runner_rsync .run_publication_point(&handle) diff --git a/tests/test_apnic_stats_live_stage2.rs b/tests/test_apnic_stats_live_stage2.rs index c0e004f..c7a3fab 100644 --- a/tests/test_apnic_stats_live_stage2.rs +++ b/tests/test_apnic_stats_live_stage2.rs @@ -186,6 +186,7 @@ fn apnic_tree_full_stats_serial() { parallel_roa_worker_pool: None, ccr_accumulator: None, persist_vcir: true, + enable_roa_validation_cache: false, }; let stats = RefCell::new(LiveStats::default()); @@ -217,6 +218,7 @@ fn apnic_tree_full_stats_serial() { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, ) .expect("run tree"); diff --git a/tests/test_apnic_tree_live_m15.rs b/tests/test_apnic_tree_live_m15.rs index 565fce2..609b5d2 100644 --- a/tests/test_apnic_tree_live_m15.rs +++ b/tests/test_apnic_tree_live_m15.rs @@ -39,6 +39,7 @@ fn apnic_tree_depth1_processes_more_than_root() { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, ) .expect("run tree from tal"); @@ -80,6 +81,7 @@ fn apnic_tree_root_only_processes_root_with_long_timeouts() { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, ) .expect("run APNIC root-only"); diff --git a/tests/test_deterministic_semantics_m4.rs b/tests/test_deterministic_semantics_m4.rs index b056052..6019778 100644 --- a/tests/test_deterministic_semantics_m4.rs +++ b/tests/test_deterministic_semantics_m4.rs @@ -111,6 +111,7 @@ fn crl_mismatch_drops_publication_point_and_cites_rfc_sections() { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, ) .expect("run tree audit"); diff --git a/tests/test_run_tree_from_tal_offline_m17.rs b/tests/test_run_tree_from_tal_offline_m17.rs index dad9f9c..77ff7be 100644 --- a/tests/test_run_tree_from_tal_offline_m17.rs +++ b/tests/test_run_tree_from_tal_offline_m17.rs @@ -117,6 +117,7 @@ fn run_tree_from_tal_url_entry_executes_and_records_failure_when_repo_empty() { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, ) .expect("run tree"); @@ -164,6 +165,7 @@ fn run_tree_from_tal_and_ta_der_entry_executes_and_records_failure_when_repo_emp compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, ) .expect("run tree"); @@ -219,6 +221,7 @@ fn run_tree_from_tal_url_audit_entry_collects_no_publication_points_when_repo_em compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, ) .expect("run tree audit"); @@ -262,6 +265,7 @@ fn run_tree_from_tal_and_ta_der_audit_entry_collects_no_publication_points_when_ compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, ) .expect("run tree audit"); @@ -313,6 +317,7 @@ fn run_tree_from_tal_url_audit_with_timing_records_phases_when_repo_empty() { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, &timing, ) @@ -363,6 +368,7 @@ fn run_tree_from_tal_and_ta_der_audit_with_timing_records_phases_when_repo_empty compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, &timing, ) diff --git a/tests/test_tree_failure_handling.rs b/tests/test_tree_failure_handling.rs index 8cba3ed..2b20415 100644 --- a/tests/test_tree_failure_handling.rs +++ b/tests/test_tree_failure_handling.rs @@ -117,6 +117,7 @@ fn tree_continues_when_a_publication_point_fails() { warnings: Vec::new(), stats: ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: Default::default(), }, audit: PublicationPointAudit::default(), discovered_children: vec![ @@ -143,6 +144,7 @@ fn tree_continues_when_a_publication_point_fails() { warnings: Vec::new(), stats: ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: Default::default(), }, audit: PublicationPointAudit::default(), discovered_children: Vec::new(), diff --git a/tests/test_tree_traversal_m14.rs b/tests/test_tree_traversal_m14.rs index 3f68d3d..4fd324d 100644 --- a/tests/test_tree_traversal_m14.rs +++ b/tests/test_tree_traversal_m14.rs @@ -127,6 +127,7 @@ fn tree_enqueues_children_for_fresh_and_current_instance_vcir_results() { warnings: Vec::new(), stats: ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: Default::default(), }, audit: PublicationPointAudit::default(), discovered_children: root_children, @@ -149,6 +150,7 @@ fn tree_enqueues_children_for_fresh_and_current_instance_vcir_results() { warnings: Vec::new(), stats: ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: Default::default(), }, audit: PublicationPointAudit::default(), discovered_children: child1_children, @@ -171,6 +173,7 @@ fn tree_enqueues_children_for_fresh_and_current_instance_vcir_results() { warnings: Vec::new(), stats: ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: Default::default(), }, audit: PublicationPointAudit::default(), discovered_children: Vec::new(), @@ -193,6 +196,7 @@ fn tree_enqueues_children_for_fresh_and_current_instance_vcir_results() { warnings: Vec::new(), stats: ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: Default::default(), }, audit: PublicationPointAudit::default(), discovered_children: Vec::new(), @@ -252,6 +256,7 @@ fn tree_respects_max_depth_and_max_instances() { warnings: Vec::new(), stats: ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: Default::default(), }, audit: PublicationPointAudit::default(), discovered_children: vec![discovered_child(root_manifest, child_manifest)], @@ -274,6 +279,7 @@ fn tree_respects_max_depth_and_max_instances() { warnings: Vec::new(), stats: ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: Default::default(), }, audit: PublicationPointAudit::default(), discovered_children: Vec::new(), @@ -289,6 +295,7 @@ fn tree_respects_max_depth_and_max_instances() { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, ) .expect("run tree depth-limited"); @@ -304,6 +311,7 @@ fn tree_respects_max_depth_and_max_instances() { compact_audit: false, persist_vcir: true, build_ccr_accumulator: true, + enable_roa_validation_cache: false, }, ) .expect("run tree instance-limited"); @@ -331,6 +339,7 @@ fn tree_audit_includes_parent_and_discovered_from_for_non_root_nodes() { warnings: Vec::new(), stats: ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: Default::default(), }, audit: PublicationPointAudit::default(), discovered_children: vec![discovered_child(root_manifest, child_manifest)], @@ -353,6 +362,7 @@ fn tree_audit_includes_parent_and_discovered_from_for_non_root_nodes() { warnings: Vec::new(), stats: ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: Default::default(), }, audit: PublicationPointAudit::default(), discovered_children: Vec::new(), @@ -421,6 +431,7 @@ fn tree_aggregates_router_keys_from_publication_point_results() { warnings: Vec::new(), stats: ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: Default::default(), }, audit: PublicationPointAudit::default(), discovered_children: Vec::new(), @@ -462,6 +473,7 @@ fn tree_prefers_lexicographically_first_discovery_when_duplicate_manifest_is_que warnings: Vec::new(), stats: ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: Default::default(), }, audit: PublicationPointAudit::default(), discovered_children: vec![first, second], @@ -484,6 +496,7 @@ fn tree_prefers_lexicographically_first_discovery_when_duplicate_manifest_is_que warnings: Vec::new(), stats: ObjectsStats::default(), audit: Vec::new(), + roa_cache_stats: Default::default(), }, audit: PublicationPointAudit::default(), discovered_children: Vec::new(),