diff --git a/scripts/soak/publish_remote231.sh b/scripts/soak/publish_remote231.sh index 119496e..37c34a6 100755 --- a/scripts/soak/publish_remote231.sh +++ b/scripts/soak/publish_remote231.sh @@ -5,11 +5,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" REMOTE_HOST="${REMOTE_HOST:-root@47.251.127.231}" -REMOTE_ROOT="${REMOTE_ROOT:-/root/rpki_20260608_2_feature062_24h_20260608T075547Z/portable-soak}" +REMOTE_ROOT="${REMOTE_ROOT:-/root/ours-rp-continuous/portable-soak}" PACKAGE_ARCHIVE="${PACKAGE_ARCHIVE:-}" MODE="${MODE:-dry-run}" RESTART_QUERY_SERVICE="${RESTART_QUERY_SERVICE:-0}" -QUERY_SERVICE_PID_PATTERN="${QUERY_SERVICE_PID_PATTERN:-rpki_query_service --query-db /root/rpki_20260616_query_service_deploy/query-db}" +QUERY_SERVICE_PID_PATTERN="${QUERY_SERVICE_PID_PATTERN:-$REMOTE_ROOT/bin/rpki_query_service}" usage() { cat <<'USAGE' @@ -27,7 +27,7 @@ Default mode is dry-run. Use --execute to apply changes. Environment overrides: REMOTE_HOST=root@47.251.127.231 - REMOTE_ROOT=/root/rpki_20260608_2_feature062_24h_20260608T075547Z/portable-soak + REMOTE_ROOT=/root/ours-rp-continuous/portable-soak RESTART_QUERY_SERVICE=0|1 USAGE } @@ -433,17 +433,18 @@ if [[ "$restart_query_service" == "1" ]]; then terminate_matching -TERM "$query_pattern" sleep 2 fi - nohup /root/rpki_20260616_query_service_deploy/bin/rpki_query_service \ - --query-db /root/rpki_20260616_query_service_deploy/query-db \ + nohup "$remote_root/bin/rpki_query_service" \ + --query-db "$remote_root/state/query-db" \ --repo-bytes-db "$remote_root/state/db/repo-bytes.db" \ - --export-root /root/rpki_20260616_query_service_deploy/query-exports \ + --export-root "$remote_root/state/query-exports" \ --listen 0.0.0.0:9560 \ --watch-run-root "$remote_root" \ --watch-interval-secs 60 \ --watch-min-run-seq "$next_index" \ --retain-indexed-runs 10 \ - --indexer-bin /root/rpki_20260616_query_service_deploy/bin/rpki_query_indexer \ - > /root/rpki_20260616_query_service_deploy/query-service.publish-${timestamp}.log 2>&1 & + --indexer-bin "$remote_root/bin/rpki_query_indexer" \ + --projection-entry-limit 20 \ + > "$remote_root/logs/query-service.publish-${timestamp}.log" 2>&1 & log "restarted query service" else log "would restart query service to reopen repo-bytes db" diff --git a/src/storage.rs b/src/storage.rs index 5ccaa54..4cd71b7 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -759,6 +759,94 @@ mod serde_byte_vec { } } +mod serde_optional_byte_vec { + pub(super) fn serialize(value: &Option>, serializer: S) -> Result + where + S: serde::Serializer, + { + match value { + Some(bytes) => serializer.serialize_some(bytes), + None => serializer.serialize_none(), + } + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> + where + D: serde::Deserializer<'de>, + { + struct OptionalByteVecVisitor; + + impl<'de> serde::de::Visitor<'de> for OptionalByteVecVisitor { + type Value = Option>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("optional byte vector") + } + + fn visit_none(self) -> Result + where + E: serde::de::Error, + { + Ok(None) + } + + fn visit_some(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer + .deserialize_bytes(super::ByteVecVisitor) + .map(Some) + } + } + + deserializer.deserialize_option(OptionalByteVecVisitor) + } +} + +mod serde_optional_bytes_32 { + pub(super) fn serialize(value: &Option<[u8; 32]>, serializer: S) -> Result + where + S: serde::Serializer, + { + match value { + Some(bytes) => serializer.serialize_some(bytes.as_slice()), + None => serializer.serialize_none(), + } + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: serde::Deserializer<'de>, + { + struct OptionalBytes32Visitor; + + impl<'de> serde::de::Visitor<'de> for OptionalBytes32Visitor { + type Value = Option<[u8; 32]>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("optional 32-byte array") + } + + fn visit_none(self) -> Result + where + E: serde::de::Error, + { + Ok(None) + } + + fn visit_some(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + super::deserialize_fixed_bytes::(deserializer).map(Some) + } + } + + deserializer.deserialize_option(OptionalBytes32Visitor) + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum VcirLocalOutputPayload { @@ -934,6 +1022,21 @@ impl RoaCacheCrlProjection { } } +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoaCacheObjectMeta { + pub source_object_uri: String, + pub source_object_hash: [u8; 32], + pub ee_serial: Vec, + pub crl_uri: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoaCacheProjectionContext { + pub parent_context_digest: [u8; 32], + pub policy_fingerprint: [u8; 32], + pub object_meta: Vec, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct RoaCacheLocalOutputProjection { #[serde(rename = "e")] @@ -985,6 +1088,11 @@ pub struct RoaCacheObjectProjection { #[serde(rename = "h")] #[serde(with = "serde_bytes_32")] pub source_object_hash: [u8; 32], + #[serde(rename = "s", default, skip_serializing_if = "Option::is_none")] + #[serde(with = "serde_optional_byte_vec")] + pub ee_serial: Option>, + #[serde(rename = "c", default, skip_serializing_if = "Option::is_none")] + pub crl_uri: Option, #[serde(rename = "o")] pub outputs: Vec, } @@ -995,6 +1103,17 @@ impl RoaCacheObjectProjection { "roa_cache_projection.entries[].source_object_uri", &self.source_object_uri, )?; + if let Some(serial) = &self.ee_serial { + if serial.is_empty() { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[].ee_serial", + detail: "must not be empty when present".to_string(), + }); + } + } + if let Some(crl_uri) = &self.crl_uri { + validate_non_empty("roa_cache_projection.entries[].crl_uri", crl_uri)?; + } if self.outputs.is_empty() { return Err(StorageError::InvalidData { entity: "roa_cache_projection.entries[]", @@ -1016,6 +1135,12 @@ pub struct RoaCacheProjection { pub instance_effective_until: PackTime, #[serde(rename = "i")] pub issuer_ca_sha256_hex: Option, + #[serde(rename = "p", default, skip_serializing_if = "Option::is_none")] + #[serde(with = "serde_optional_bytes_32")] + pub parent_context_digest: Option<[u8; 32]>, + #[serde(rename = "f", default, skip_serializing_if = "Option::is_none")] + #[serde(with = "serde_optional_bytes_32")] + pub policy_fingerprint: Option<[u8; 32]>, #[serde(rename = "c")] pub crl_sha256_by_uri: Vec, #[serde(rename = "r")] @@ -1466,6 +1591,13 @@ impl PublicationPointCacheProjection { impl RoaCacheProjection { pub fn from_vcir(vcir: &ValidatedCaInstanceResult) -> StorageResult> { + Self::from_vcir_with_context(vcir, None) + } + + pub fn from_vcir_with_context( + vcir: &ValidatedCaInstanceResult, + context: Option<&RoaCacheProjectionContext>, + ) -> StorageResult> { let mut issuer_ca_sha256_hex = None; let mut crl_sha256_by_uri = Vec::new(); for artifact in &vcir.related_artifacts { @@ -1492,6 +1624,13 @@ impl RoaCacheProjection { } crl_sha256_by_uri.sort_by(|left, right| left.uri.cmp(&right.uri)); + let meta_by_uri = context.map(|context| { + context + .object_meta + .iter() + .map(|meta| (meta.source_object_uri.as_str(), meta)) + .collect::>() + }); let mut entries: Vec = Vec::new(); let mut entry_index_by_uri: HashMap = HashMap::new(); for output in &vcir.local_outputs { @@ -1499,6 +1638,23 @@ impl RoaCacheProjection { else { continue; }; + let meta = meta_by_uri + .as_ref() + .and_then(|meta| meta.get(output.source_object_uri.as_str()).copied()); + if context.is_some() && meta.is_none() { + continue; + } + if let Some(meta) = meta { + if meta.source_object_hash != output.source_object_hash { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[]", + detail: format!( + "metadata source object hash mismatch for {}", + output.source_object_uri + ), + }); + } + } if let Some(entry_index) = entry_index_by_uri.get(output.source_object_uri.as_str()) { let entry = &mut entries[*entry_index]; if entry.source_object_hash != output.source_object_hash { @@ -1516,6 +1672,8 @@ impl RoaCacheProjection { entries.push(RoaCacheObjectProjection { source_object_uri: output.source_object_uri.clone(), source_object_hash: output.source_object_hash, + ee_serial: meta.map(|meta| meta.ee_serial.clone()), + crl_uri: meta.map(|meta| meta.crl_uri.clone()), outputs: vec![projected_output], }); } @@ -1529,6 +1687,8 @@ impl RoaCacheProjection { manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), instance_effective_until: vcir.instance_gate.instance_effective_until.clone(), issuer_ca_sha256_hex, + parent_context_digest: context.map(|context| context.parent_context_digest), + policy_fingerprint: context.map(|context| context.policy_fingerprint), crl_sha256_by_uri, entries, }; @@ -1548,6 +1708,13 @@ impl RoaCacheProjection { if let Some(hash) = &self.issuer_ca_sha256_hex { validate_sha256_hex("roa_cache_projection.issuer_ca_sha256_hex", hash)?; } + if self.parent_context_digest.is_some() != self.policy_fingerprint.is_some() { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.context", + detail: "parent_context_digest and policy_fingerprint must be both present or both absent" + .to_string(), + }); + } let mut seen_crls = HashSet::with_capacity(self.crl_sha256_by_uri.len()); for crl in &self.crl_sha256_by_uri { crl.validate_internal()?; @@ -2306,10 +2473,11 @@ fn write_roa_cache_projection_to_batch( projection_cf: &ColumnFamily, batch: &mut WriteBatch, vcir: &ValidatedCaInstanceResult, + context: Option<&RoaCacheProjectionContext>, timing: Option<&mut VcirReplaceTimingBreakdown>, ) -> StorageResult<()> { let projection_key = roa_cache_projection_key(&vcir.manifest_rsync_uri); - let projection = RoaCacheProjection::from_vcir(vcir)?; + let projection = RoaCacheProjection::from_vcir_with_context(vcir, context)?; match projection { Some(projection) => { let projection_value = encode_cbor(&projection, "roa_cache_projection")?; @@ -2761,6 +2929,15 @@ impl RocksStore { &self, vcir: &ValidatedCaInstanceResult, publication_point_projection: Option<&PublicationPointCacheProjection>, + ) -> StorageResult<()> { + self.put_vcir_with_projections(vcir, None, publication_point_projection) + } + + pub fn put_vcir_with_projections( + &self, + vcir: &ValidatedCaInstanceResult, + roa_cache_context: Option<&RoaCacheProjectionContext>, + publication_point_projection: Option<&PublicationPointCacheProjection>, ) -> StorageResult<()> { vcir.validate_internal()?; let vcir_cf = self.cf(CF_VCIR)?; @@ -2776,7 +2953,13 @@ impl RocksStore { let replay_key = manifest_replay_meta_key(&replay_meta.manifest_rsync_uri); let replay_value = encode_cbor(&replay_meta, "manifest_replay_meta")?; batch.put_cf(replay_cf, replay_key.as_bytes(), replay_value); - write_roa_cache_projection_to_batch(projection_cf, &mut batch, vcir, None)?; + write_roa_cache_projection_to_batch( + projection_cf, + &mut batch, + vcir, + roa_cache_context, + None, + )?; write_publication_point_cache_projection_to_batch( pp_projection_cf, &mut batch, @@ -2799,6 +2982,19 @@ impl RocksStore { &self, vcir: &ValidatedCaInstanceResult, publication_point_projection: Option<&PublicationPointCacheProjection>, + ) -> StorageResult { + self.replace_vcir_manifest_replay_meta_and_projections( + vcir, + None, + publication_point_projection, + ) + } + + pub fn replace_vcir_manifest_replay_meta_and_projections( + &self, + vcir: &ValidatedCaInstanceResult, + roa_cache_context: Option<&RoaCacheProjectionContext>, + publication_point_projection: Option<&PublicationPointCacheProjection>, ) -> StorageResult { let mut timing = VcirReplaceTimingBreakdown { rss_before_kb: process_vm_rss_kb(), @@ -2837,7 +3033,13 @@ impl RocksStore { timing.rss_after_replay_meta_encode_kb = process_vm_rss_kb(); let projection_encode_started = std::time::Instant::now(); - write_roa_cache_projection_to_batch(projection_cf, &mut batch, vcir, Some(&mut timing))?; + write_roa_cache_projection_to_batch( + projection_cf, + &mut batch, + vcir, + roa_cache_context, + Some(&mut timing), + )?; timing.roa_cache_projection_encode_ms = projection_encode_started.elapsed().as_millis() as u64; timing.rss_after_roa_cache_projection_encode_kb = process_vm_rss_kb(); diff --git a/src/validation/objects.rs b/src/validation/objects.rs index befb951..269b3a0 100644 --- a/src/validation/objects.rs +++ b/src/validation/objects.rs @@ -1,6 +1,7 @@ use crate::analysis::timing::TimingHandle; use crate::audit::{AuditObjectKind, AuditObjectResult, ObjectAuditEntry, sha256_hex_from_32}; use crate::data_model::aspa::{AspaDecodeError, AspaObject, AspaValidateError}; +use crate::data_model::common::BigUnsigned; use crate::data_model::manifest::ManifestObject; use crate::data_model::rc::{ AsIdentifierChoice, AsResourceSet, IpAddressChoice, IpAddressOrRange, IpPrefix as RcIpPrefix, @@ -15,9 +16,9 @@ use crate::parallel::object_worker::{ use crate::policy::{Policy, SignedObjectFailurePolicy}; use crate::report::{RfcRef, Warning}; use crate::storage::{ - PackFile, PackTime, RoaCacheProjection, ValidatedCaInstanceResult, VcirArtifactKind, - VcirArtifactRole, VcirArtifactValidationStatus, VcirLocalOutput, VcirLocalOutputPayload, - VcirOutputType, VcirSourceObjectType, + PackFile, PackTime, RoaCacheObjectMeta, RoaCacheProjection, ValidatedCaInstanceResult, + VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirLocalOutput, + VcirLocalOutputPayload, VcirOutputType, VcirSourceObjectType, }; use crate::validation::cert_path::{CertPathError, validate_signed_object_ee_cert_path_fast}; use crate::validation::manifest::PublicationPointData; @@ -60,14 +61,34 @@ fn decode_resource_certificate_with_policy( pub(crate) struct VerifiedIssuerCrl { crl: crate::data_model::crl::RpkixCrl, revoked_serials: std::collections::HashSet>, + sha256_hex: String, } #[derive(Clone, Debug)] pub(crate) enum CachedIssuerCrl { - Pending(Vec), + Pending { + bytes: Vec, + sha256_hex: Option, + }, Ok(Arc), } +impl CachedIssuerCrl { + fn current_sha256_hex(&mut self) -> &str { + match self { + CachedIssuerCrl::Pending { bytes, sha256_hex } => { + if sha256_hex.is_none() { + *sha256_hex = Some(crate::audit::sha256_hex(bytes)); + } + sha256_hex + .as_deref() + .expect("pending CRL sha256 must be populated") + } + CachedIssuerCrl::Ok(verified) => verified.sha256_hex.as_str(), + } + } +} + #[derive(Clone, Debug, Default)] pub(crate) struct IssuerResourcesIndex { ip_v4: Option, Vec)>>, @@ -118,6 +139,7 @@ pub struct ObjectsOutput { pub stats: ObjectsStats, pub audit: Vec, pub roa_cache_stats: RoaValidationCacheStats, + pub roa_cache_object_meta: Vec, } #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -140,6 +162,12 @@ pub struct RoaValidationCacheStats { pub miss_roas: usize, pub blocked_roas: usize, pub fresh_roas: usize, + pub context_blocked_roas: usize, + pub crl_recheck_hit_roas: usize, + pub hash_blocked_roas: usize, + pub expired_blocked_roas: usize, + pub revoked_blocked_roas: usize, + pub metadata_blocked_roas: usize, pub context_gate_nanos: u64, pub lookup_nanos: u64, } @@ -148,6 +176,8 @@ pub struct RoaValidationCacheStats { pub struct RoaValidationCacheInput<'a> { enabled: bool, view: Option<&'a RoaValidationCacheView>, + parent_context_digest: Option<[u8; 32]>, + policy_fingerprint: Option<[u8; 32]>, } impl<'a> RoaValidationCacheInput<'a> { @@ -155,6 +185,8 @@ impl<'a> RoaValidationCacheInput<'a> { Self { enabled: false, view: None, + parent_context_digest: None, + policy_fingerprint: None, } } @@ -162,6 +194,21 @@ impl<'a> RoaValidationCacheInput<'a> { Self { enabled: true, view, + parent_context_digest: None, + policy_fingerprint: None, + } + } + + pub fn enabled_with_context( + view: Option<&'a RoaValidationCacheView>, + parent_context_digest: [u8; 32], + policy_fingerprint: [u8; 32], + ) -> Self { + Self { + enabled: true, + view, + parent_context_digest: Some(parent_context_digest), + policy_fingerprint: Some(policy_fingerprint), } } } @@ -175,6 +222,12 @@ impl RoaValidationCacheStats { self.miss_roas += other.miss_roas; self.blocked_roas += other.blocked_roas; self.fresh_roas += other.fresh_roas; + self.context_blocked_roas += other.context_blocked_roas; + self.crl_recheck_hit_roas += other.crl_recheck_hit_roas; + self.hash_blocked_roas += other.hash_blocked_roas; + self.expired_blocked_roas += other.expired_blocked_roas; + self.revoked_blocked_roas += other.revoked_blocked_roas; + self.metadata_blocked_roas += other.metadata_blocked_roas; self.context_gate_nanos = self .context_gate_nanos .saturating_add(other.context_gate_nanos); @@ -225,6 +278,36 @@ impl RoaValidationCacheStats { self.blocked_roas, ); record_non_zero(timing, "roa_validation_cache_fresh_roas", self.fresh_roas); + record_non_zero( + timing, + "roa_validation_cache_context_blocked_roas", + self.context_blocked_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_crl_recheck_hit_roas", + self.crl_recheck_hit_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_hash_blocked_roas", + self.hash_blocked_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_expired_blocked_roas", + self.expired_blocked_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_revoked_blocked_roas", + self.revoked_blocked_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_metadata_blocked_roas", + self.metadata_blocked_roas, + ); record_non_zero( timing, "roa_validation_cache_context_gate_nanos", @@ -253,6 +336,8 @@ fn record_non_zero(timing: &TimingHandle, key: &'static str, value: usize) { pub struct RoaValidationCacheView { entries_by_uri: HashMap, issuer_ca_sha256_hex: Option, + parent_context_digest: Option<[u8; 32]>, + policy_fingerprint: Option<[u8; 32]>, crl_sha256_by_uri: HashMap, blocked: bool, } @@ -260,6 +345,8 @@ pub struct RoaValidationCacheView { #[derive(Clone, Debug)] pub struct CachedRoaValidationResult { source_object_hash: [u8; 32], + ee_serial: Option>, + crl_uri: Option, outputs: Vec, } @@ -271,6 +358,8 @@ impl RoaValidationCacheView { let mut entries_by_uri: HashMap = HashMap::with_capacity(projection.entries.len()); let issuer_ca_sha256_hex = projection.issuer_ca_sha256_hex.clone(); + let parent_context_digest = projection.parent_context_digest; + let policy_fingerprint = projection.policy_fingerprint; let crl_sha256_by_uri = projection .crl_sha256_by_uri .iter() @@ -286,6 +375,8 @@ impl RoaValidationCacheView { return Self { entries_by_uri, issuer_ca_sha256_hex, + parent_context_digest, + policy_fingerprint, crl_sha256_by_uri, blocked, }; @@ -310,6 +401,8 @@ impl RoaValidationCacheView { entry.source_object_uri.clone(), CachedRoaValidationResult { source_object_hash: entry.source_object_hash, + ee_serial: entry.ee_serial.clone(), + crl_uri: entry.crl_uri.clone(), outputs, }, ); @@ -318,79 +411,19 @@ impl RoaValidationCacheView { Self { entries_by_uri, issuer_ca_sha256_hex, + parent_context_digest, + policy_fingerprint, crl_sha256_by_uri, blocked, } } - pub fn from_vcir( - vcir: &ValidatedCaInstanceResult, - validation_time: time::OffsetDateTime, - ) -> Self { - let mut entries_by_uri: HashMap = HashMap::new(); - let mut issuer_ca_sha256_hex: Option = None; - let mut crl_sha256_by_uri: HashMap = HashMap::new(); - let blocked = vcir - .instance_gate - .instance_effective_until - .parse() - .map(|effective_until| effective_until <= validation_time) - .unwrap_or(true); - - if blocked { - return Self { - entries_by_uri, - issuer_ca_sha256_hex, - crl_sha256_by_uri, - blocked, - }; - } - - for artifact in &vcir.related_artifacts { - if artifact.validation_status != VcirArtifactValidationStatus::Accepted { - continue; - } - match (artifact.artifact_role, artifact.artifact_kind) { - ( - VcirArtifactRole::IssuerCert | VcirArtifactRole::TrustAnchorCert, - VcirArtifactKind::Cer, - ) => { - issuer_ca_sha256_hex = Some(artifact.sha256.clone()); - } - (_, VcirArtifactKind::Crl) => { - if let Some(uri) = artifact.uri.as_ref() { - crl_sha256_by_uri.insert(uri.clone(), artifact.sha256.clone()); - } - } - _ => {} - } - } - - for output in &vcir.local_outputs { - if output.output_type != VcirOutputType::Vrp - || output.source_object_type != VcirSourceObjectType::Roa - { - continue; - } - entries_by_uri - .entry(output.source_object_uri.clone()) - .or_insert_with(|| CachedRoaValidationResult { - source_object_hash: output.source_object_hash, - outputs: Vec::new(), - }) - .outputs - .push(output.clone()); - } - - Self { - entries_by_uri, - issuer_ca_sha256_hex, - crl_sha256_by_uri, - blocked, - } - } - - fn matches_current_context(&self, issuer_ca_der: &[u8], locked_files: &[PackFile]) -> bool { + fn matches_current_context( + &self, + issuer_ca_der: &[u8], + parent_context_digest: Option<[u8; 32]>, + policy_fingerprint: Option<[u8; 32]>, + ) -> bool { if self.blocked { return false; } @@ -402,28 +435,41 @@ impl RoaValidationCacheView { return false; } - let current_crl_hashes = locked_files - .iter() - .filter(|file| file.rsync_uri.ends_with(".crl")) - .map(|file| (file.rsync_uri.clone(), sha256_hex_from_32(&file.sha256))) - .collect::>(); - self.crl_sha256_by_uri == current_crl_hashes + if let Some(expected_parent_context) = self.parent_context_digest { + if Some(expected_parent_context) != parent_context_digest { + return false; + } + } else { + return false; + } + + if let Some(expected_policy) = self.policy_fingerprint { + if Some(expected_policy) != policy_fingerprint { + return false; + } + } else { + return false; + } + + true } fn lookup( &self, file: &PackFile, + crl_cache: &mut std::collections::HashMap, + issuer_ca_der: &[u8], validation_time: time::OffsetDateTime, ) -> RoaCacheLookupResult { if self.blocked { - return RoaCacheLookupResult::Blocked; + return RoaCacheLookupResult::ExpiredBlocked; } let Some(cached) = self.entries_by_uri.get(file.rsync_uri.as_str()) else { return RoaCacheLookupResult::Miss; }; if cached.source_object_hash != file.sha256 { - return RoaCacheLookupResult::Blocked; + return RoaCacheLookupResult::HashBlocked; } if cached.outputs.is_empty() { return RoaCacheLookupResult::Miss; @@ -434,7 +480,38 @@ impl RoaValidationCacheView { .parse() .map_or(true, |t| t <= validation_time) }) { - return RoaCacheLookupResult::Blocked; + return RoaCacheLookupResult::ExpiredBlocked; + } + + let Some(crl_uri) = cached.crl_uri.as_deref() else { + return RoaCacheLookupResult::MetadataBlocked; + }; + let Some(ee_serial) = cached.ee_serial.as_ref() else { + return RoaCacheLookupResult::MetadataBlocked; + }; + let crl_unchanged = { + let Some(current_crl_hash) = crl_cache + .get_mut(crl_uri) + .map(CachedIssuerCrl::current_sha256_hex) + else { + return RoaCacheLookupResult::MetadataBlocked; + }; + self.crl_sha256_by_uri + .get(crl_uri) + .map(|expected| expected == current_crl_hash) + .unwrap_or(false) + }; + if !crl_unchanged { + let verified_crl = match ensure_issuer_crl_verified(crl_uri, crl_cache, issuer_ca_der) { + Ok(verified_crl) => verified_crl, + Err(_) => return RoaCacheLookupResult::MetadataBlocked, + }; + if !crl_valid_at_time(&verified_crl.crl, validation_time) { + return RoaCacheLookupResult::ExpiredBlocked; + } + if verified_crl.revoked_serials.contains(ee_serial) { + return RoaCacheLookupResult::RevokedBlocked; + } } let mut vrps = Vec::with_capacity(cached.outputs.len()); @@ -447,7 +524,7 @@ impl RoaValidationCacheView { max_length, } = &output.payload else { - return RoaCacheLookupResult::Blocked; + return RoaCacheLookupResult::MetadataBlocked; }; vrps.push(Vrp { asn: *asn, @@ -460,24 +537,38 @@ impl RoaValidationCacheView { }); } - RoaCacheLookupResult::Hit(RoaTaskOk { + let ok = RoaTaskOk { vrps, local_outputs: cached.outputs.clone(), reused_from_cache: true, - }) + cache_object_meta: Some(RoaCacheObjectMeta { + source_object_uri: file.rsync_uri.clone(), + source_object_hash: file.sha256, + ee_serial: ee_serial.clone(), + crl_uri: crl_uri.to_string(), + }), + }; + if crl_unchanged { + RoaCacheLookupResult::Hit(ok) + } else { + RoaCacheLookupResult::CrlRecheckHit(ok) + } } } fn active_roa_cache_view<'a>( roa_cache: RoaValidationCacheInput<'a>, issuer_ca_der: &[u8], - locked_files: &[PackFile], stats: &mut RoaValidationCacheStats, roa_total: usize, ) -> Option<&'a RoaValidationCacheView> { let view = roa_cache.view?; let gate_started = Instant::now(); - if view.matches_current_context(issuer_ca_der, locked_files) { + if view.matches_current_context( + issuer_ca_der, + roa_cache.parent_context_digest, + roa_cache.policy_fingerprint, + ) { stats.context_gate_nanos = stats .context_gate_nanos .saturating_add(gate_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); @@ -487,6 +578,7 @@ fn active_roa_cache_view<'a>( .context_gate_nanos .saturating_add(gate_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); stats.blocked_roas += roa_total; + stats.context_blocked_roas += roa_total; stats.fresh_roas += roa_total; None } @@ -495,8 +587,38 @@ fn active_roa_cache_view<'a>( #[derive(Debug)] enum RoaCacheLookupResult { Hit(RoaTaskOk), + CrlRecheckHit(RoaTaskOk), Miss, - Blocked, + HashBlocked, + ExpiredBlocked, + RevokedBlocked, + MetadataBlocked, +} + +fn crl_valid_at_time( + crl: &crate::data_model::crl::RpkixCrl, + validation_time: time::OffsetDateTime, +) -> bool { + let this_update = crl.this_update.utc.to_offset(time::UtcOffset::UTC); + let next_update = crl.next_update.utc.to_offset(time::UtcOffset::UTC); + validation_time >= this_update && validation_time < next_update +} + +fn record_roa_cache_block( + stats: &mut RoaValidationCacheStats, + lookup_result: &RoaCacheLookupResult, +) { + stats.blocked_roas += 1; + stats.fresh_roas += 1; + match lookup_result { + RoaCacheLookupResult::HashBlocked => stats.hash_blocked_roas += 1, + RoaCacheLookupResult::ExpiredBlocked => stats.expired_blocked_roas += 1, + RoaCacheLookupResult::RevokedBlocked => stats.revoked_blocked_roas += 1, + RoaCacheLookupResult::MetadataBlocked => stats.metadata_blocked_roas += 1, + RoaCacheLookupResult::Hit(_) + | RoaCacheLookupResult::CrlRecheckHit(_) + | RoaCacheLookupResult::Miss => {} + } } #[derive(Clone, Copy)] @@ -510,6 +632,7 @@ pub(crate) struct RoaTaskOk { pub(crate) vrps: Vec, pub(crate) local_outputs: Vec, pub(crate) reused_from_cache: bool, + pub(crate) cache_object_meta: Option, } #[derive(Debug)] @@ -648,6 +771,7 @@ pub fn process_publication_point_for_issuer_with_cache_options { @@ -738,6 +864,7 @@ pub fn process_publication_point_for_issuer_with_cache_options = Vec::new(); let mut aspas: Vec = Vec::new(); let mut local_outputs_cache: Vec = Vec::new(); + let mut roa_cache_object_meta: Vec = Vec::new(); let active_cache_view = active_roa_cache_view( roa_cache, issuer_ca_der, - locked_files, &mut roa_cache_stats, stats.roa_total, ); @@ -817,7 +951,8 @@ pub fn process_publication_point_for_issuer_with_cache_options { + roa_cache_stats.hit_roas += 1; + roa_cache_stats.crl_recheck_hit_roas += 1; + RoaTaskResult { + publication_point_id: 0, + index: idx, + worker_index: 0, + queue_wait_ms: 0, + worker_ms: 0, + outcome: Ok(ok), + } + } RoaCacheLookupResult::Miss => { roa_cache_stats.miss_roas += 1; roa_cache_stats.fresh_roas += 1; @@ -859,9 +1006,11 @@ pub fn process_publication_point_for_issuer_with_cache_options { - roa_cache_stats.blocked_roas += 1; - roa_cache_stats.fresh_roas += 1; + blocked @ (RoaCacheLookupResult::HashBlocked + | RoaCacheLookupResult::ExpiredBlocked + | RoaCacheLookupResult::RevokedBlocked + | RoaCacheLookupResult::MetadataBlocked) => { + record_roa_cache_block(&mut roa_cache_stats, &blocked); let task = RoaTask { index: idx, file }; let _t = timing.as_ref().map(|t| t.span_phase("objects_roa_total")); validate_roa_task_serial( @@ -911,6 +1060,9 @@ pub fn process_publication_point_for_issuer_with_cache_options RoaTaskRe task.strict_cms_der, task.strict_name, ) - .map(|(vrps, local_outputs)| RoaTaskOk { + .map(|(vrps, local_outputs, cache_object_meta)| RoaTaskOk { vrps, local_outputs, reused_from_cache: false, + cache_object_meta, }); RoaTaskResult { @@ -1636,6 +1792,7 @@ pub(crate) fn prepare_publication_point_for_parallel_roa_with_cache { @@ -1724,18 +1883,25 @@ pub(crate) fn prepare_publication_point_for_parallel_roa_with_cache = locked_files + let mut crl_cache: std::collections::HashMap = locked_files .iter() .filter(|f| f.rsync_uri.ends_with(".crl")) .map(|f| { let bytes = f .bytes_cloned() .expect("snapshot CRL bytes must be loadable"); - (f.rsync_uri.clone(), CachedIssuerCrl::Pending(bytes)) + ( + f.rsync_uri.clone(), + CachedIssuerCrl::Pending { + bytes, + sha256_hex: None, + }, + ) }) .collect(); @@ -1780,13 +1946,13 @@ pub(crate) fn prepare_publication_point_for_parallel_roa_with_cache { + roa_cache_stats.hit_roas += 1; + roa_cache_stats.crl_recheck_hit_roas += 1; + cached_roa_results.push(RoaTaskResult { + publication_point_id, + index, + worker_index: usize::MAX, + queue_wait_ms: 0, + worker_ms: 0, + outcome: Ok(ok), + }); + } RoaCacheLookupResult::Miss => { roa_cache_stats.miss_roas += 1; roa_cache_stats.fresh_roas += 1; roa_task_indices.push(index); } - RoaCacheLookupResult::Blocked => { - roa_cache_stats.blocked_roas += 1; - roa_cache_stats.fresh_roas += 1; + blocked @ (RoaCacheLookupResult::HashBlocked + | RoaCacheLookupResult::ExpiredBlocked + | RoaCacheLookupResult::RevokedBlocked + | RoaCacheLookupResult::MetadataBlocked) => { + record_roa_cache_block(&mut roa_cache_stats, &blocked); roa_task_indices.push(index); } } @@ -1891,6 +2072,7 @@ pub(crate) fn reduce_parallel_roa_stage( let mut vrps: Vec = Vec::new(); let mut aspas: Vec = Vec::new(); let mut local_outputs_cache: Vec = Vec::new(); + let mut roa_cache_object_meta: Vec = Vec::new(); for (idx, file) in shared.locked_files.iter().enumerate() { if file.rsync_uri.ends_with(".roa") { @@ -1915,6 +2097,9 @@ pub(crate) fn reduce_parallel_roa_stage( if collect_vcir_local_outputs || ok.reused_from_cache { local_outputs_cache.extend(ok.local_outputs); } + if let Some(meta) = ok.cache_object_meta.take() { + roa_cache_object_meta.push(meta); + } audit.push(ObjectAuditEntry { rsync_uri: file.rsync_uri.clone(), sha256_hex: sha256_hex_from_32(&file.sha256), @@ -2010,6 +2195,7 @@ pub(crate) fn reduce_parallel_roa_stage( stats, audit, roa_cache_stats, + roa_cache_object_meta, }) } @@ -2209,10 +2395,11 @@ pub(crate) fn validate_roa_task_serial( strict_cms_der, strict_name, ) - .map(|(vrps, local_outputs)| RoaTaskOk { + .map(|(vrps, local_outputs, cache_object_meta)| RoaTaskOk { vrps, local_outputs, reused_from_cache: false, + cache_object_meta, }); RoaTaskResult { @@ -2241,7 +2428,7 @@ fn process_roa_with_issuer( collect_vcir_local_outputs: bool, strict_cms_der: bool, strict_name: bool, -) -> Result<(Vec, Vec), ObjectValidateError> { +) -> Result<(Vec, Vec, Option), ObjectValidateError> { let _decode = timing .as_ref() .map(|t| t.span_phase("objects_roa_decode_and_validate_total")); @@ -2301,8 +2488,14 @@ fn process_roa_with_issuer( drop(_subset); let vrps = roa_to_vrps(&roa); + let cache_object_meta = RoaCacheObjectMeta { + source_object_uri: file.rsync_uri.clone(), + source_object_hash: file.sha256, + ee_serial: BigUnsigned::from_biguint(&ee.resource_cert.tbs.serial_number).bytes_be, + crl_uri: issuer_crl_rsync_uri.to_string(), + }; if !collect_vcir_local_outputs { - return Ok((vrps, Vec::new())); + return Ok((vrps, Vec::new(), Some(cache_object_meta))); } let source_object_hash = sha256_hex_from_32(&file.sha256); let source_ee_cert_hash = crate::audit::sha256_hex(ee.raw_der.as_slice()); @@ -2338,7 +2531,7 @@ fn process_roa_with_issuer( }) .collect(); - Ok((vrps, local_outputs)) + Ok((vrps, local_outputs, Some(cache_object_meta))) } fn process_roa_with_issuer_parallel_cached( @@ -2357,7 +2550,7 @@ fn process_roa_with_issuer_parallel_cached( collect_vcir_local_outputs: bool, strict_cms_der: bool, strict_name: bool, -) -> Result<(Vec, Vec), ObjectValidateError> { +) -> Result<(Vec, Vec, Option), ObjectValidateError> { let _decode = timing .as_ref() .map(|t| t.span_phase("objects_roa_decode_and_validate_total")); @@ -2423,8 +2616,14 @@ fn process_roa_with_issuer_parallel_cached( drop(_subset); let vrps = roa_to_vrps(&roa); + let cache_object_meta = RoaCacheObjectMeta { + source_object_uri: file.rsync_uri.clone(), + source_object_hash: file.sha256, + ee_serial: BigUnsigned::from_biguint(&ee.resource_cert.tbs.serial_number).bytes_be, + crl_uri: issuer_crl_rsync_uri.clone(), + }; if !collect_vcir_local_outputs { - return Ok((vrps, Vec::new())); + return Ok((vrps, Vec::new(), Some(cache_object_meta))); } let source_object_hash = sha256_hex_from_32(&file.sha256); let source_ee_cert_hash = crate::audit::sha256_hex(ee.raw_der.as_slice()); @@ -2460,7 +2659,7 @@ fn process_roa_with_issuer_parallel_cached( }) .collect(); - Ok((vrps, local_outputs)) + Ok((vrps, local_outputs, Some(cache_object_meta))) } fn process_aspa_with_issuer( @@ -2637,8 +2836,11 @@ fn ensure_issuer_crl_verified<'a>( .expect("CRL must exist in cache"); match entry { CachedIssuerCrl::Ok(v) => Ok(Arc::clone(v)), - CachedIssuerCrl::Pending(bytes) => { + CachedIssuerCrl::Pending { bytes, sha256_hex } => { let der = std::mem::take(bytes); + let current_sha256_hex = sha256_hex + .take() + .unwrap_or_else(|| crate::audit::sha256_hex(&der)); let crl = crate::data_model::crl::RpkixCrl::decode_der(&der) .map_err(CertPathError::CrlDecode)?; crl.verify_signature_with_issuer_certificate_der(issuer_ca_der) @@ -2653,6 +2855,7 @@ fn ensure_issuer_crl_verified<'a>( *entry = CachedIssuerCrl::Ok(Arc::new(VerifiedIssuerCrl { crl, revoked_serials, + sha256_hex: current_sha256_hex, })); match entry { CachedIssuerCrl::Ok(v) => Ok(Arc::clone(v)), @@ -3126,8 +3329,9 @@ mod tests { }; use crate::policy::Policy; use crate::storage::{ - PackTime, RoaCacheProjection, ValidatedManifestMeta, VcirAuditSummary, - VcirCcrManifestProjection, VcirInstanceGate, VcirRelatedArtifact, VcirSummary, + PackTime, RoaCacheObjectMeta, RoaCacheProjection, RoaCacheProjectionContext, + ValidatedManifestMeta, VcirAuditSummary, VcirCcrManifestProjection, VcirInstanceGate, + VcirRelatedArtifact, VcirSummary, }; use crate::validation::publication_point::PublicationPointSnapshot; use std::collections::HashMap; @@ -3143,6 +3347,49 @@ mod tests { OffsetDateTime::parse(value, &Rfc3339).expect("parse fixed test time") } + const TEST_PARENT_CONTEXT: [u8; 32] = [0x70; 32]; + const TEST_POLICY_FINGERPRINT: [u8; 32] = [0x71; 32]; + const TEST_CRL_URI: &str = "rsync://example.test/repo/current.crl"; + const TEST_ROA_URI: &str = "rsync://example.test/repo/a.roa"; + + fn sha256_32(bytes: &[u8]) -> [u8; 32] { + let digest = sha2::Sha256::digest(bytes); + let mut out = [0u8; 32]; + out.copy_from_slice(&digest); + out + } + + fn sample_roa_cache_projection( + vcir: &ValidatedCaInstanceResult, + roa_hash: [u8; 32], + ) -> RoaCacheProjection { + RoaCacheProjection::from_vcir_with_context( + vcir, + Some(&RoaCacheProjectionContext { + parent_context_digest: TEST_PARENT_CONTEXT, + policy_fingerprint: TEST_POLICY_FINGERPRINT, + object_meta: vec![RoaCacheObjectMeta { + source_object_uri: TEST_ROA_URI.to_string(), + source_object_hash: roa_hash, + ee_serial: vec![0x01], + crl_uri: TEST_CRL_URI.to_string(), + }], + }), + ) + .expect("build projection") + .expect("projection exists") + } + + fn sample_crl_cache(crl_bytes: Vec) -> HashMap { + HashMap::from([( + TEST_CRL_URI.to_string(), + CachedIssuerCrl::Pending { + bytes: crl_bytes, + sha256_hex: None, + }, + )]) + } + fn sample_roa_cache_vcir( issuer_der: &[u8], crl_hash: [u8; 32], @@ -3239,7 +3486,8 @@ mod tests { fn roa_validation_cache_view_hits_when_context_and_hash_match() { let validation_time = fixed_time("2026-06-05T00:00:00Z"); let issuer_der = b"issuer-ca"; - let crl_hash = [0x22; 32]; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); let roa_hash = [0x11; 32]; let vcir = sample_roa_cache_vcir( issuer_der, @@ -3248,22 +3496,17 @@ mod tests { fixed_time("2026-06-07T00:00:00Z"), fixed_time("2026-06-08T00:00:00Z"), ); - let view = RoaValidationCacheView::from_vcir(&vcir, validation_time); - let files = vec![ - PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/a.roa", - vec![0x01], - roa_hash, - ), - PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/current.crl", - vec![0x02], - crl_hash, - ), - ]; + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(crl_bytes); - assert!(view.matches_current_context(issuer_der, &files)); - let hit = view.lookup(&files[0], validation_time); + assert!(view.matches_current_context( + issuer_der, + Some(TEST_PARENT_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + let hit = view.lookup(&file, &mut crl_cache, issuer_der, validation_time); let RoaCacheLookupResult::Hit(ok) = hit else { panic!("expected cache hit, got {hit:?}"); }; @@ -3271,13 +3514,16 @@ mod tests { assert_eq!(ok.vrps.len(), 1); assert_eq!(ok.vrps[0].asn, 64500); assert_eq!(ok.local_outputs.len(), 1); + assert!(ok.cache_object_meta.is_some()); } #[test] - fn roa_validation_cache_view_from_projection_matches_vcir_view() { + fn roa_validation_cache_lookup_memoizes_current_crl_hash() { let validation_time = fixed_time("2026-06-05T00:00:00Z"); let issuer_der = b"issuer-ca"; - let crl_hash = [0x22; 32]; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let crl_hash_hex = sha256_hex(&crl_bytes); let roa_hash = [0x11; 32]; let vcir = sample_roa_cache_vcir( issuer_der, @@ -3286,43 +3532,88 @@ mod tests { fixed_time("2026-06-07T00:00:00Z"), fixed_time("2026-06-08T00:00:00Z"), ); - let projection = RoaCacheProjection::from_vcir(&vcir) - .expect("build projection") - .expect("projection exists"); + let projection = sample_roa_cache_projection(&vcir, roa_hash); let view = RoaValidationCacheView::from_projection(&projection, validation_time); - let files = vec![ - PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/a.roa", - vec![0x01], - roa_hash, - ), - PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/current.crl", - vec![0x02], - crl_hash, - ), - ]; + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(crl_bytes); - assert!(view.matches_current_context(issuer_der, &files)); - let hit = view.lookup(&files[0], validation_time); + let first = view.lookup(&file, &mut crl_cache, issuer_der, validation_time); + assert!(matches!(first, RoaCacheLookupResult::Hit(_))); + match crl_cache.get(TEST_CRL_URI).expect("test CRL cache entry") { + CachedIssuerCrl::Pending { + sha256_hex: Some(cached), + .. + } => assert_eq!(cached, &crl_hash_hex), + other => panic!("expected pending CRL with memoized hash, got {other:?}"), + } + + let second = view.lookup(&file, &mut crl_cache, issuer_der, validation_time); + assert!(matches!(second, RoaCacheLookupResult::Hit(_))); + } + + #[test] + fn cached_verified_crl_reports_hash_and_validity_window() { + let crl_bytes = fixture_bytes( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl", + ); + let crl = crate::data_model::crl::RpkixCrl::decode_der(&crl_bytes).expect("decode CRL"); + let verified = Arc::new(VerifiedIssuerCrl { + crl, + revoked_serials: std::collections::HashSet::new(), + sha256_hex: sha256_hex(&crl_bytes), + }); + let mut cached = CachedIssuerCrl::Ok(Arc::clone(&verified)); + + assert_eq!(cached.current_sha256_hex(), verified.sha256_hex); + assert!(crl_valid_at_time( + &verified.crl, + fixed_time("2026-01-21T00:00:00Z") + )); + assert!(!crl_valid_at_time( + &verified.crl, + fixed_time("2026-01-22T00:00:00Z") + )); + } + + #[test] + fn roa_validation_cache_view_from_projection_preserves_metadata() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + assert_eq!(projection.parent_context_digest, Some(TEST_PARENT_CONTEXT)); + assert_eq!(projection.policy_fingerprint, Some(TEST_POLICY_FINGERPRINT)); + assert_eq!( + projection.entries[0].ee_serial.as_deref(), + Some(&[0x01][..]) + ); + assert_eq!(projection.entries[0].crl_uri.as_deref(), Some(TEST_CRL_URI)); + + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(crl_bytes); + let hit = view.lookup(&file, &mut crl_cache, issuer_der, validation_time); let RoaCacheLookupResult::Hit(ok) = hit else { panic!("expected projection cache hit, got {hit:?}"); }; - assert!(ok.reused_from_cache); - assert_eq!(ok.vrps.len(), 1); - assert_eq!(ok.vrps[0].asn, 64500); - assert_eq!(ok.local_outputs.len(), 1); - assert_eq!( - ok.local_outputs[0].source_object_uri, - "rsync://example.test/repo/a.roa" - ); + assert_eq!(ok.local_outputs[0].source_object_uri, TEST_ROA_URI); } #[test] - fn roa_validation_cache_view_from_projection_blocks_on_gates() { + fn roa_validation_cache_view_blocks_on_context_hash_and_expiry_gates() { let validation_time = fixed_time("2026-06-05T00:00:00Z"); let issuer_der = b"issuer-ca"; - let crl_hash = [0x22; 32]; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); let roa_hash = [0x11; 32]; let vcir = sample_roa_cache_vcir( issuer_der, @@ -3331,41 +3622,37 @@ mod tests { fixed_time("2026-06-07T00:00:00Z"), fixed_time("2026-06-08T00:00:00Z"), ); - let mut projection = RoaCacheProjection::from_vcir(&vcir) - .expect("build projection") - .expect("projection exists"); - let files = vec![ - PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/a.roa", - vec![0x01], - roa_hash, - ), - PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/current.crl", - vec![0x02], - crl_hash, - ), - ]; + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut projection = sample_roa_cache_projection(&vcir, roa_hash); projection.issuer_ca_sha256_hex = Some("00".repeat(32)); let issuer_changed = RoaValidationCacheView::from_projection(&projection, validation_time); - assert!(!issuer_changed.matches_current_context(issuer_der, &files)); + assert!(!issuer_changed.matches_current_context( + issuer_der, + Some(TEST_PARENT_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); - let mut projection = RoaCacheProjection::from_vcir(&vcir) - .expect("build projection") - .expect("projection exists"); - projection.crl_sha256_by_uri[0].sha256 = "11".repeat(32); - let crl_changed = RoaValidationCacheView::from_projection(&projection, validation_time); - assert!(!crl_changed.matches_current_context(issuer_der, &files)); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let parent_changed = RoaValidationCacheView::from_projection(&projection, validation_time); + assert!(!parent_changed.matches_current_context( + issuer_der, + Some([0x99; 32]), + Some(TEST_POLICY_FINGERPRINT) + )); + assert!(!parent_changed.matches_current_context( + issuer_der, + Some(TEST_PARENT_CONTEXT), + Some([0x98; 32]) + )); - let mut projection = RoaCacheProjection::from_vcir(&vcir) - .expect("build projection") - .expect("projection exists"); + let mut projection = sample_roa_cache_projection(&vcir, roa_hash); projection.entries[0].source_object_hash = [0xff; 32]; let roa_changed = RoaValidationCacheView::from_projection(&projection, validation_time); + let mut crl_cache = sample_crl_cache(crl_bytes); assert!(matches!( - roa_changed.lookup(&files[0], validation_time), - RoaCacheLookupResult::Blocked + roa_changed.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::HashBlocked )); let expired_vcir = sample_roa_cache_vcir( @@ -3375,20 +3662,23 @@ mod tests { fixed_time("2026-06-07T00:00:00Z"), fixed_time("2026-06-04T00:00:00Z"), ); - let expired_projection = RoaCacheProjection::from_vcir(&expired_vcir) - .expect("build projection") - .expect("projection exists"); + let expired_projection = sample_roa_cache_projection(&expired_vcir, roa_hash); let expired_view = RoaValidationCacheView::from_projection(&expired_projection, validation_time); - assert!(!expired_view.matches_current_context(issuer_der, &files)); + assert!(!expired_view.matches_current_context( + issuer_der, + Some(TEST_PARENT_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + let mut crl_cache = sample_crl_cache(b"current-crl".to_vec()); assert!(matches!( - expired_view.lookup(&files[0], validation_time), - RoaCacheLookupResult::Blocked + expired_view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::ExpiredBlocked )); } #[test] - fn roa_validation_cache_view_blocks_when_context_changes() { + fn active_roa_cache_view_records_context_block() { let validation_time = fixed_time("2026-06-05T00:00:00Z"); let issuer_der = b"issuer-ca"; let crl_hash = [0x22; 32]; @@ -3400,34 +3690,18 @@ mod tests { fixed_time("2026-06-07T00:00:00Z"), fixed_time("2026-06-08T00:00:00Z"), ); - let view = RoaValidationCacheView::from_vcir(&vcir, validation_time); - let files = vec![ - PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/a.roa", - vec![0x01], - roa_hash, - ), - PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/current.crl", - vec![0x02], - [0x33; 32], - ), - ]; - let mut stats = - RoaValidationCacheStats::for_input(RoaValidationCacheInput::enabled(Some(&view)), 1); - - assert!(!view.matches_current_context(issuer_der, &files)); - assert!( - active_roa_cache_view( - RoaValidationCacheInput::enabled(Some(&view)), - issuer_der, - &files, - &mut stats, - 1, - ) - .is_none() + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let input = RoaValidationCacheInput::enabled_with_context( + Some(&view), + [0x99; 32], + TEST_POLICY_FINGERPRINT, ); + let mut stats = RoaValidationCacheStats::for_input(input, 1); + + assert!(active_roa_cache_view(input, issuer_der, &mut stats, 1).is_none()); assert_eq!(stats.blocked_roas, 1); + assert_eq!(stats.context_blocked_roas, 1); assert_eq!(stats.fresh_roas, 1); } @@ -3435,7 +3709,8 @@ mod tests { fn roa_validation_cache_view_blocks_expired_output() { let validation_time = fixed_time("2026-06-05T00:00:00Z"); let issuer_der = b"issuer-ca"; - let crl_hash = [0x22; 32]; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); let roa_hash = [0x11; 32]; let vcir = sample_roa_cache_vcir( issuer_der, @@ -3444,16 +3719,14 @@ mod tests { fixed_time("2026-06-04T00:00:00Z"), fixed_time("2026-06-08T00:00:00Z"), ); - let view = RoaValidationCacheView::from_vcir(&vcir, validation_time); - let file = PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/a.roa", - vec![0x01], - roa_hash, - ); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(crl_bytes); assert!(matches!( - view.lookup(&file, validation_time), - RoaCacheLookupResult::Blocked + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::ExpiredBlocked )); } @@ -3492,37 +3765,71 @@ mod tests { } #[test] - fn roa_validation_cache_view_blocks_expired_instance_gate() { - let validation_time = fixed_time("2026-06-05T00:00:00Z"); - let issuer_der = b"issuer-ca"; - let crl_hash = [0x22; 32]; - let roa_hash = [0x11; 32]; - let vcir = sample_roa_cache_vcir( - issuer_der, - crl_hash, - roa_hash, - fixed_time("2026-06-07T00:00:00Z"), - fixed_time("2026-06-04T00:00:00Z"), - ); - let view = RoaValidationCacheView::from_vcir(&vcir, validation_time); - let file = PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/a.roa", - vec![0x01], - roa_hash, - ); + fn roa_validation_cache_stats_records_block_breakdown_to_timing() { + let mut stats = RoaValidationCacheStats::default(); + record_roa_cache_block(&mut stats, &RoaCacheLookupResult::HashBlocked); + record_roa_cache_block(&mut stats, &RoaCacheLookupResult::ExpiredBlocked); + record_roa_cache_block(&mut stats, &RoaCacheLookupResult::RevokedBlocked); + record_roa_cache_block(&mut stats, &RoaCacheLookupResult::MetadataBlocked); + stats.crl_recheck_hit_roas = 2; + stats.context_blocked_roas = 3; + stats.context_gate_nanos = 4; + stats.lookup_nanos = 5; - assert!(!view.matches_current_context(issuer_der, &[])); - assert!(matches!( - view.lookup(&file, validation_time), - RoaCacheLookupResult::Blocked - )); + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + stats.record_to_timing(Some(&timing)); + let dir = tempfile::tempdir().expect("timing dir"); + let path = dir.path().join("timing.json"); + timing.write_json(&path, 10).expect("write timing"); + let report: serde_json::Value = + serde_json::from_slice(&std::fs::read(path).expect("read timing")) + .expect("parse timing"); + + assert_eq!(stats.blocked_roas, 4); + assert_eq!(stats.fresh_roas, 4); + assert_eq!(report["counts"]["roa_validation_cache_blocked_roas"], 4); + assert_eq!( + report["counts"]["roa_validation_cache_hash_blocked_roas"], + 1 + ); + assert_eq!( + report["counts"]["roa_validation_cache_expired_blocked_roas"], + 1 + ); + assert_eq!( + report["counts"]["roa_validation_cache_revoked_blocked_roas"], + 1 + ); + assert_eq!( + report["counts"]["roa_validation_cache_metadata_blocked_roas"], + 1 + ); + assert_eq!( + report["counts"]["roa_validation_cache_crl_recheck_hit_roas"], + 2 + ); + assert_eq!( + report["counts"]["roa_validation_cache_context_blocked_roas"], + 3 + ); + assert_eq!( + report["counts"]["roa_validation_cache_context_gate_nanos"], + 4 + ); + assert_eq!(report["counts"]["roa_validation_cache_lookup_nanos"], 5); } #[test] fn roa_validation_cache_view_ignores_rejected_artifacts_and_non_roa_outputs() { let validation_time = fixed_time("2026-06-05T00:00:00Z"); let issuer_der = b"issuer-ca"; - let crl_hash = [0x22; 32]; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); let roa_hash = [0x11; 32]; let mut vcir = sample_roa_cache_vcir( issuer_der, @@ -3557,23 +3864,22 @@ mod tests { }, rule_hash: [0x66; 32], }); - let view = RoaValidationCacheView::from_vcir(&vcir, validation_time); - let files = vec![ - PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/a.roa", - vec![0x01], - roa_hash, - ), - PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/current.crl", - vec![0x02], - crl_hash, - ), - ]; + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let mut crl_cache = sample_crl_cache(crl_bytes); - assert!(view.matches_current_context(issuer_der, &files)); + assert!(view.matches_current_context( + issuer_der, + Some(TEST_PARENT_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); assert!(matches!( - view.lookup(&files[0], validation_time), + view.lookup( + &PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash), + &mut crl_cache, + issuer_der, + validation_time + ), RoaCacheLookupResult::Hit(_) )); assert!(matches!( @@ -3583,6 +3889,8 @@ mod tests { vec![0x03], [0x44; 32], ), + &mut crl_cache, + issuer_der, validation_time ), RoaCacheLookupResult::Miss @@ -3590,34 +3898,62 @@ mod tests { } #[test] - fn roa_validation_cache_context_rejects_missing_and_changed_issuer() { + fn roa_validation_cache_context_requires_issuer_parent_and_policy() { let mut view = RoaValidationCacheView { entries_by_uri: HashMap::new(), issuer_ca_sha256_hex: None, + parent_context_digest: Some(TEST_PARENT_CONTEXT), + policy_fingerprint: Some(TEST_POLICY_FINGERPRINT), crl_sha256_by_uri: HashMap::new(), blocked: false, }; - assert!(!view.matches_current_context(b"issuer-ca", &[])); + assert!(!view.matches_current_context( + b"issuer-ca", + Some(TEST_PARENT_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); view.issuer_ca_sha256_hex = Some("00".repeat(32)); - assert!(!view.matches_current_context(b"issuer-ca", &[])); + assert!(!view.matches_current_context( + b"issuer-ca", + Some(TEST_PARENT_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); view.issuer_ca_sha256_hex = Some(sha256_hex(b"issuer-ca")); - assert!(view.matches_current_context(b"issuer-ca", &[])); + assert!(view.matches_current_context( + b"issuer-ca", + Some(TEST_PARENT_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + assert!(!view.matches_current_context(b"issuer-ca", None, Some(TEST_POLICY_FINGERPRINT))); + + view.parent_context_digest = None; + assert!(!view.matches_current_context( + b"issuer-ca", + Some(TEST_PARENT_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + view.parent_context_digest = Some(TEST_PARENT_CONTEXT); + view.policy_fingerprint = None; + assert!(!view.matches_current_context( + b"issuer-ca", + Some(TEST_PARENT_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); } #[test] fn roa_validation_cache_lookup_classifies_hash_empty_payload_and_missing_uri() { let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash_hex = sha256_hex(&crl_bytes); let roa_hash = [0x11; 32]; - let file = PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/a.roa", - vec![0x01], - roa_hash, - ); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); let mut output = sample_roa_cache_vcir( - b"issuer-ca", - [0x22; 32], + issuer_der, + sha256_32(&crl_bytes), roa_hash, fixed_time("2026-06-07T00:00:00Z"), fixed_time("2026-06-08T00:00:00Z"), @@ -3626,37 +3962,101 @@ mod tests { .remove(0); let mut view = RoaValidationCacheView { entries_by_uri: HashMap::new(), - issuer_ca_sha256_hex: Some(sha256_hex(b"issuer-ca")), - crl_sha256_by_uri: HashMap::new(), + issuer_ca_sha256_hex: Some(sha256_hex(issuer_der)), + parent_context_digest: Some(TEST_PARENT_CONTEXT), + policy_fingerprint: Some(TEST_POLICY_FINGERPRINT), + crl_sha256_by_uri: HashMap::from([(TEST_CRL_URI.to_string(), crl_hash_hex.clone())]), blocked: false, }; + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); assert!(matches!( - view.lookup(&file, validation_time), + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), RoaCacheLookupResult::Miss )); view.entries_by_uri.insert( file.rsync_uri.clone(), CachedRoaValidationResult { - source_object_hash: [0xff; 32], + source_object_hash: roa_hash, + ee_serial: Some(vec![0x01]), + crl_uri: None, outputs: vec![output.clone()], }, ); + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); assert!(matches!( - view.lookup(&file, validation_time), - RoaCacheLookupResult::Blocked + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::MetadataBlocked )); view.entries_by_uri.insert( file.rsync_uri.clone(), CachedRoaValidationResult { source_object_hash: roa_hash, + ee_serial: None, + crl_uri: Some(TEST_CRL_URI.to_string()), + outputs: vec![output.clone()], + }, + ); + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::MetadataBlocked + )); + + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: roa_hash, + ee_serial: Some(vec![0x01]), + crl_uri: Some(TEST_CRL_URI.to_string()), + outputs: vec![output.clone()], + }, + ); + let mut crl_cache = HashMap::new(); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::MetadataBlocked + )); + + view.crl_sha256_by_uri + .insert(TEST_CRL_URI.to_string(), "00".repeat(32)); + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::MetadataBlocked + )); + view.crl_sha256_by_uri + .insert(TEST_CRL_URI.to_string(), crl_hash_hex); + + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: [0xff; 32], + ee_serial: Some(vec![0x01]), + crl_uri: Some(TEST_CRL_URI.to_string()), + outputs: vec![output.clone()], + }, + ); + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::HashBlocked + )); + + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: roa_hash, + ee_serial: Some(vec![0x01]), + crl_uri: Some(TEST_CRL_URI.to_string()), outputs: Vec::new(), }, ); + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); assert!(matches!( - view.lookup(&file, validation_time), + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), RoaCacheLookupResult::Miss )); @@ -3668,12 +4068,15 @@ mod tests { file.rsync_uri.clone(), CachedRoaValidationResult { source_object_hash: roa_hash, + ee_serial: Some(vec![0x01]), + crl_uri: Some(TEST_CRL_URI.to_string()), outputs: vec![output], }, ); + let mut crl_cache = sample_crl_cache(crl_bytes); assert!(matches!( - view.lookup(&file, validation_time), - RoaCacheLookupResult::Blocked + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::MetadataBlocked )); } @@ -3685,7 +4088,10 @@ mod tests { let issuer_ca_der = fixture_bytes( "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", ); - let crl_hash = [0x22; 32]; + let crl_bytes = fixture_bytes( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl", + ); + let crl_hash = sha256_32(&crl_bytes); let actual_roa_hash = [0x11; 32]; let cached_roa_hash = [0x33; 32]; let validation_time = fixed_time("2026-06-05T00:00:00Z"); @@ -3703,11 +4109,7 @@ mod tests { verified_at: PackTime::from_utc_offset_datetime(validation_time), manifest_bytes, files: vec![ - PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/current.crl", - vec![0x02], - crl_hash, - ), + PackFile::from_bytes_with_sha256(TEST_CRL_URI, crl_bytes, crl_hash), PackFile::from_bytes_with_sha256( "rsync://example.test/repo/a.roa", vec![0x01], @@ -3722,7 +4124,8 @@ mod tests { fixed_time("2026-06-07T00:00:00Z"), fixed_time("2026-06-08T00:00:00Z"), ); - let view = RoaValidationCacheView::from_vcir(&vcir, validation_time); + let projection = sample_roa_cache_projection(&vcir, cached_roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); let stage = match prepare_publication_point_for_parallel_roa_with_cache( 7, @@ -3734,7 +4137,11 @@ mod tests { None, validation_time, false, - RoaValidationCacheInput::enabled(Some(&view)), + RoaValidationCacheInput::enabled_with_context( + Some(&view), + TEST_PARENT_CONTEXT, + TEST_POLICY_FINGERPRINT, + ), ) { ParallelObjectsPrepare::Staged(stage) => stage, ParallelObjectsPrepare::Complete(_) => panic!("expected staged ROA fallback"), @@ -3871,7 +4278,10 @@ mod tests { let mut crl_cache: HashMap = HashMap::new(); crl_cache.insert( "rsync://example.test/a.crl".to_string(), - CachedIssuerCrl::Pending(vec![0x01]), + CachedIssuerCrl::Pending { + bytes: vec![0x01], + sha256_hex: None, + }, ); let err = choose_crl_uri_for_certificate(None, &crl_cache).unwrap_err(); assert!(matches!(err, ObjectValidateError::MissingCrlDpUris)); @@ -3894,9 +4304,18 @@ mod tests { let mut crl_cache: HashMap = HashMap::new(); crl_cache.insert( "rsync://example.test/other.crl".to_string(), - CachedIssuerCrl::Pending(vec![0x00]), + CachedIssuerCrl::Pending { + bytes: vec![0x00], + sha256_hex: None, + }, + ); + crl_cache.insert( + matching_uri.clone(), + CachedIssuerCrl::Pending { + bytes: vec![0x01], + sha256_hex: None, + }, ); - crl_cache.insert(matching_uri.clone(), CachedIssuerCrl::Pending(vec![0x01])); let uri = choose_crl_uri_for_certificate(Some(ee_crldp_uris), &crl_cache).unwrap(); assert_eq!(uri, matching_uri); @@ -3917,7 +4336,10 @@ mod tests { let mut crl_cache: HashMap = HashMap::new(); crl_cache.insert( "rsync://example.test/other.crl".to_string(), - CachedIssuerCrl::Pending(vec![0x01]), + CachedIssuerCrl::Pending { + bytes: vec![0x01], + sha256_hex: None, + }, ); let err = choose_crl_uri_for_certificate(ee_crldp_uris, &crl_cache).unwrap_err(); assert!(matches!(err, ObjectValidateError::CrlNotFound(_))); diff --git a/src/validation/tree.rs b/src/validation/tree.rs index 097f1c3..ef4a646 100644 --- a/src/validation/tree.rs +++ b/src/validation/tree.rs @@ -410,6 +410,7 @@ mod tests { stats: ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit::default(), cir_fresh_objects: Vec::new(), @@ -496,6 +497,7 @@ mod tests { audit: Vec::new(), roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit { objects: vec![fresh_reject.clone(), cached.clone()], @@ -528,6 +530,7 @@ mod tests { audit: Vec::new(), roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit { objects: vec![fresh], diff --git a/src/validation/tree_parallel.rs b/src/validation/tree_parallel.rs index 9db51f8..352b8ca 100644 --- a/src/validation/tree_parallel.rs +++ b/src/validation/tree_parallel.rs @@ -1168,7 +1168,13 @@ fn stage_ready_publication_point( None }; let roa_cache = if runner.enable_roa_validation_cache && has_roa { - RoaValidationCacheInput::enabled(roa_cache_view.as_ref()) + RoaValidationCacheInput::enabled_with_context( + roa_cache_view.as_ref(), + crate::validation::tree_runner::parent_context_digest_for_ca(&ready.node.handle), + crate::validation::tree_runner::publication_point_cache_policy_fingerprint( + runner.policy, + ), + ) } else { RoaValidationCacheInput::disabled() }; @@ -2316,6 +2322,7 @@ mod tests { stats: ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit::default(), cir_fresh_objects: Vec::new(), diff --git a/src/validation/tree_runner.rs b/src/validation/tree_runner.rs index d878de5..1f2d57f 100644 --- a/src/validation/tree_runner.rs +++ b/src/validation/tree_runner.rs @@ -26,9 +26,9 @@ use crate::replay::delta_archive::ReplayDeltaArchiveIndex; use crate::report::{RfcRef, Warning}; use crate::storage::{ PackFile, PackTime, PublicationPointCacheChild, PublicationPointCacheOutput, - PublicationPointCacheProjection, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, - VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, - VcirCcrManifestProjection, VcirChildEntry, VcirInstanceGate, VcirLocalOutput, + PublicationPointCacheProjection, RawByHashEntry, RoaCacheProjectionContext, RocksStore, + ValidatedCaInstanceResult, VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, + VcirAuditSummary, VcirCcrManifestProjection, VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirLocalOutputPayload, VcirOutputType, VcirRelatedArtifact, VcirReplaceTimingBreakdown, VcirSourceObjectType, VcirSummary, }; @@ -1041,6 +1041,7 @@ impl<'a> Rpkiv1PublicationPointRunner<'a> { // local_outputs_cache only exists to build/persist VCIR. Release it before the // publication point result is retained for the rest of the run. let _released_local_outputs = std::mem::take(&mut objects.local_outputs_cache); + let _released_roa_cache_object_meta = std::mem::take(&mut objects.roa_cache_object_meta); let mut ccr_projection_build_ms = 0; let mut ccr_append_ms = 0; @@ -1460,7 +1461,11 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> { None }; let roa_cache = if self.enable_roa_validation_cache && has_roa { - RoaValidationCacheInput::enabled(roa_cache_view.as_ref()) + RoaValidationCacheInput::enabled_with_context( + roa_cache_view.as_ref(), + parent_context_digest_for_ca(ca), + publication_point_cache_policy_fingerprint(self.policy), + ) } else { RoaValidationCacheInput::disabled() }; @@ -1887,7 +1892,7 @@ fn ta_context_digest_for_ca(ca: &CaInstanceHandle) -> [u8; 32] { ]) } -fn parent_context_digest_for_ca(ca: &CaInstanceHandle) -> [u8; 32] { +pub(crate) fn parent_context_digest_for_ca(ca: &CaInstanceHandle) -> [u8; 32] { hash_serialized_parts(&[ ( "version", @@ -1913,7 +1918,7 @@ fn parent_context_digest_for_ca(ca: &CaInstanceHandle) -> [u8; 32] { ]) } -fn publication_point_cache_policy_fingerprint(policy: &Policy) -> [u8; 32] { +pub(crate) fn publication_point_cache_policy_fingerprint(policy: &Policy) -> [u8; 32] { hash_serialized_parts(&[ ("version", b"publication-point-cache-policy-v1".to_vec()), ("policy", cbor_or_debug_bytes(policy)), @@ -3074,6 +3079,7 @@ fn empty_objects_output() -> crate::validation::objects::ObjectsOutput { stats: crate::validation::objects::ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), } } @@ -4183,8 +4189,13 @@ fn persist_vcir_for_fresh_result_with_timing( None }; let replace_timing = store - .replace_vcir_manifest_replay_meta_and_publication_point_cache_projection( + .replace_vcir_manifest_replay_meta_and_projections( &vcir, + Some(&RoaCacheProjectionContext { + parent_context_digest: parent_context_digest_for_ca(ca), + policy_fingerprint: publication_point_cache_policy_fingerprint(policy), + object_meta: objects.roa_cache_object_meta.clone(), + }), publication_point_cache_projection.as_ref(), ) .map_err(|e| format!("store VCIR and manifest replay meta failed: {e}"))?; @@ -4351,10 +4362,29 @@ fn take_or_build_vcir_local_outputs( pack: &PublicationPointSnapshot, objects: &mut crate::validation::objects::ObjectsOutput, ) -> Result, String> { - if !objects.local_outputs_cache.is_empty() { - return Ok(std::mem::take(&mut objects.local_outputs_cache)); + let mut cached_outputs = std::mem::take(&mut objects.local_outputs_cache); + if cached_outputs.is_empty() { + return build_vcir_local_outputs(ca, pack, objects); } - build_vcir_local_outputs(ca, pack, objects) + + let covered_roa_uris: HashSet = cached_outputs + .iter() + .filter(|output| output.source_object_type == VcirSourceObjectType::Roa) + .map(|output| output.source_object_uri.clone()) + .collect(); + let covered_aspa_uris: HashSet = cached_outputs + .iter() + .filter(|output| output.source_object_type == VcirSourceObjectType::Aspa) + .map(|output| output.source_object_uri.clone()) + .collect(); + cached_outputs.extend(build_vcir_local_outputs_excluding( + ca, + pack, + objects, + &covered_roa_uris, + &covered_aspa_uris, + )?); + Ok(cached_outputs) } fn build_vcir_ccr_manifest_projection_from_fresh( @@ -4454,10 +4484,16 @@ fn build_vcir_local_outputs( pack: &PublicationPointSnapshot, objects: &crate::validation::objects::ObjectsOutput, ) -> Result, String> { - if !objects.local_outputs_cache.is_empty() { - return Ok(objects.local_outputs_cache.clone()); - } + build_vcir_local_outputs_excluding(_ca, pack, objects, &HashSet::new(), &HashSet::new()) +} +fn build_vcir_local_outputs_excluding( + _ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + objects: &crate::validation::objects::ObjectsOutput, + covered_roa_uris: &HashSet, + covered_aspa_uris: &HashSet, +) -> Result, String> { let accepted_roa_uris: HashSet<&str> = objects .audit .iter() @@ -4476,7 +4512,9 @@ fn build_vcir_local_outputs( let mut out = Vec::new(); for file in &pack.files { let source_object_hash = sha256_hex_from_32(&file.sha256); - if accepted_roa_uris.contains(file.rsync_uri.as_str()) { + if accepted_roa_uris.contains(file.rsync_uri.as_str()) + && !covered_roa_uris.contains(file.rsync_uri.as_str()) + { let roa = RoaObject::decode_der( file.bytes() .map_err(|e| format!("load accepted ROA bytes for VCIR failed: {e}"))?, @@ -4512,7 +4550,9 @@ fn build_vcir_local_outputs( rule_hash: sha256_hex_to_32(&rule_hash), }); } - } else if accepted_aspa_uris.contains(file.rsync_uri.as_str()) { + } else if accepted_aspa_uris.contains(file.rsync_uri.as_str()) + && !covered_aspa_uris.contains(file.rsync_uri.as_str()) + { let aspa = AspaObject::decode_der( file.bytes() .map_err(|e| format!("load accepted ASPA bytes for VCIR failed: {e}"))?, diff --git a/src/validation/tree_runner/tests.rs b/src/validation/tree_runner/tests.rs index 27a5609..9e36b04 100644 --- a/src/validation/tree_runner/tests.rs +++ b/src/validation/tree_runner/tests.rs @@ -755,22 +755,21 @@ fn build_vcir_local_outputs_prefers_cached_outputs() { }, rule_hash: sha256_32(b"cached-rule"), }]; - let outputs = build_vcir_local_outputs( - &ca, - &pack, - &crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: cached.clone(), - warnings: Vec::new(), - stats: crate::validation::objects::ObjectsStats::default(), - audit: Vec::new(), - roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), - }, - ) - .expect("reuse cached outputs"); + let mut objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: cached.clone(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + let outputs = + take_or_build_vcir_local_outputs(&ca, &pack, &mut objects).expect("reuse cached outputs"); assert_eq!(outputs, cached); + assert!(objects.local_outputs_cache.is_empty()); } #[test] @@ -1277,6 +1276,7 @@ fn build_vcir_related_artifacts_classifies_snapshot_files_and_audit_statuses() { }, ], roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), }; let artifacts = build_vcir_related_artifacts( &ca, @@ -2778,6 +2778,7 @@ fn build_publication_point_audit_emits_no_audit_entry_for_duplicate_pack_uri() { stats: crate::validation::objects::ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), }; let audit = build_publication_point_audit_from_snapshot( @@ -2847,6 +2848,7 @@ fn build_publication_point_audit_marks_invalid_crl_as_error_and_overlays_roa_aud detail: None, }], roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), }; let audit = build_publication_point_audit_from_snapshot( @@ -3895,6 +3897,7 @@ fn build_publication_point_audit_from_vcir_uses_vcir_metadata_and_overlays_child detail: Some("overridden from object audit".to_string()), }], roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), }; let child_audits = vec![ObjectAuditEntry { rsync_uri: vcir.child_entries[0].child_cert_rsync_uri.clone(), @@ -3992,6 +3995,7 @@ fn build_publication_point_audit_from_vcir_failed_no_cache_keeps_current_reject_ detail: Some("manifest is not valid at validation_time".to_string()), }], roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), }; let audit = build_publication_point_audit_from_vcir( @@ -4129,6 +4133,7 @@ fn build_publication_point_audit_from_vcir_without_cached_inputs_returns_empty_l stats: crate::validation::objects::ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), }, &[], &[], diff --git a/tests/test_tree_failure_handling.rs b/tests/test_tree_failure_handling.rs index c7b353f..000c83d 100644 --- a/tests/test_tree_failure_handling.rs +++ b/tests/test_tree_failure_handling.rs @@ -118,6 +118,7 @@ fn tree_continues_when_a_publication_point_fails() { stats: ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: Default::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit::default(), cir_fresh_objects: Vec::new(), @@ -147,6 +148,7 @@ fn tree_continues_when_a_publication_point_fails() { stats: ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: Default::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit::default(), cir_fresh_objects: Vec::new(), diff --git a/tests/test_tree_traversal_m14.rs b/tests/test_tree_traversal_m14.rs index 0221406..59b93e1 100644 --- a/tests/test_tree_traversal_m14.rs +++ b/tests/test_tree_traversal_m14.rs @@ -128,6 +128,7 @@ fn tree_enqueues_children_for_fresh_and_current_instance_vcir_results() { stats: ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: Default::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit::default(), cir_fresh_objects: Vec::new(), @@ -153,6 +154,7 @@ fn tree_enqueues_children_for_fresh_and_current_instance_vcir_results() { stats: ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: Default::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit::default(), cir_fresh_objects: Vec::new(), @@ -178,6 +180,7 @@ fn tree_enqueues_children_for_fresh_and_current_instance_vcir_results() { stats: ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: Default::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit::default(), cir_fresh_objects: Vec::new(), @@ -203,6 +206,7 @@ fn tree_enqueues_children_for_fresh_and_current_instance_vcir_results() { stats: ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: Default::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit::default(), cir_fresh_objects: Vec::new(), @@ -265,6 +269,7 @@ fn tree_respects_max_depth_and_max_instances() { stats: ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: Default::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit::default(), cir_fresh_objects: Vec::new(), @@ -290,6 +295,7 @@ fn tree_respects_max_depth_and_max_instances() { stats: ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: Default::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit::default(), cir_fresh_objects: Vec::new(), @@ -358,6 +364,7 @@ fn tree_audit_includes_parent_and_discovered_from_for_non_root_nodes() { stats: ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: Default::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit::default(), cir_fresh_objects: Vec::new(), @@ -383,6 +390,7 @@ fn tree_audit_includes_parent_and_discovered_from_for_non_root_nodes() { stats: ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: Default::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit::default(), cir_fresh_objects: Vec::new(), @@ -454,6 +462,7 @@ fn tree_aggregates_router_keys_from_publication_point_results() { stats: ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: Default::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit::default(), cir_fresh_objects: Vec::new(), @@ -498,6 +507,7 @@ fn tree_prefers_lexicographically_first_discovery_when_duplicate_manifest_is_que stats: ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: Default::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit::default(), cir_fresh_objects: Vec::new(), @@ -523,6 +533,7 @@ fn tree_prefers_lexicographically_first_discovery_when_duplicate_manifest_is_que stats: ObjectsStats::default(), audit: Vec::new(), roa_cache_stats: Default::default(), + roa_cache_object_meta: Vec::new(), }, audit: PublicationPointAudit::default(), cir_fresh_objects: Vec::new(),