20260624 优化ROA cache lookup长尾
This commit is contained in:
parent
8c4a677ffa
commit
adb68fd469
@ -1093,6 +1093,8 @@ pub struct RoaCacheObjectProjection {
|
|||||||
pub ee_serial: Option<Vec<u8>>,
|
pub ee_serial: Option<Vec<u8>>,
|
||||||
#[serde(rename = "c", default, skip_serializing_if = "Option::is_none")]
|
#[serde(rename = "c", default, skip_serializing_if = "Option::is_none")]
|
||||||
pub crl_uri: Option<String>,
|
pub crl_uri: Option<String>,
|
||||||
|
#[serde(rename = "x")]
|
||||||
|
pub outputs_effective_until_unix: i64,
|
||||||
#[serde(rename = "o")]
|
#[serde(rename = "o")]
|
||||||
pub outputs: Vec<RoaCacheLocalOutputProjection>,
|
pub outputs: Vec<RoaCacheLocalOutputProjection>,
|
||||||
}
|
}
|
||||||
@ -1123,6 +1125,29 @@ impl RoaCacheObjectProjection {
|
|||||||
for output in &self.outputs {
|
for output in &self.outputs {
|
||||||
output.validate_internal()?;
|
output.validate_internal()?;
|
||||||
}
|
}
|
||||||
|
let expected_effective_until = self
|
||||||
|
.outputs
|
||||||
|
.iter()
|
||||||
|
.map(|output| {
|
||||||
|
output
|
||||||
|
.item_effective_until
|
||||||
|
.parse()
|
||||||
|
.map(|time| time.unix_timestamp())
|
||||||
|
.map_err(|detail| StorageError::InvalidData {
|
||||||
|
entity: "roa_cache_projection.entries[].outputs_effective_until_unix",
|
||||||
|
detail,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect::<StorageResult<Vec<_>>>()?
|
||||||
|
.into_iter()
|
||||||
|
.min()
|
||||||
|
.expect("outputs must not be empty");
|
||||||
|
if self.outputs_effective_until_unix != expected_effective_until {
|
||||||
|
return Err(StorageError::InvalidData {
|
||||||
|
entity: "roa_cache_projection.entries[].outputs_effective_until_unix",
|
||||||
|
detail: "must equal the earliest output item_effective_until".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1638,6 +1663,14 @@ impl RoaCacheProjection {
|
|||||||
else {
|
else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
|
let projected_output_effective_until = projected_output
|
||||||
|
.item_effective_until
|
||||||
|
.parse()
|
||||||
|
.map(|time| time.unix_timestamp())
|
||||||
|
.map_err(|detail| StorageError::InvalidData {
|
||||||
|
entity: "roa_cache_projection.entries[].outputs_effective_until_unix",
|
||||||
|
detail,
|
||||||
|
})?;
|
||||||
let meta = meta_by_uri
|
let meta = meta_by_uri
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|meta| meta.get(output.source_object_uri.as_str()).copied());
|
.and_then(|meta| meta.get(output.source_object_uri.as_str()).copied());
|
||||||
@ -1666,6 +1699,9 @@ impl RoaCacheProjection {
|
|||||||
),
|
),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
entry.outputs_effective_until_unix = entry
|
||||||
|
.outputs_effective_until_unix
|
||||||
|
.min(projected_output_effective_until);
|
||||||
entry.outputs.push(projected_output);
|
entry.outputs.push(projected_output);
|
||||||
} else {
|
} else {
|
||||||
entry_index_by_uri.insert(output.source_object_uri.clone(), entries.len());
|
entry_index_by_uri.insert(output.source_object_uri.clone(), entries.len());
|
||||||
@ -1674,6 +1710,7 @@ impl RoaCacheProjection {
|
|||||||
source_object_hash: output.source_object_hash,
|
source_object_hash: output.source_object_hash,
|
||||||
ee_serial: meta.map(|meta| meta.ee_serial.clone()),
|
ee_serial: meta.map(|meta| meta.ee_serial.clone()),
|
||||||
crl_uri: meta.map(|meta| meta.crl_uri.clone()),
|
crl_uri: meta.map(|meta| meta.crl_uri.clone()),
|
||||||
|
outputs_effective_until_unix: projected_output_effective_until,
|
||||||
outputs: vec![projected_output],
|
outputs: vec![projected_output],
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@ -313,6 +313,10 @@ fn roa_cache_projection_from_vcir_keeps_only_roa_vrp_outputs() {
|
|||||||
projection.entries[0].source_object_uri,
|
projection.entries[0].source_object_uri,
|
||||||
"rsync://example.test/repo/object.roa"
|
"rsync://example.test/repo/object.roa"
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
projection.entries[0].outputs_effective_until_unix,
|
||||||
|
12 * 3600
|
||||||
|
);
|
||||||
assert_eq!(projection.entries[0].outputs.len(), 1);
|
assert_eq!(projection.entries[0].outputs.len(), 1);
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
projection.entries[0].outputs[0].payload,
|
projection.entries[0].outputs[0].payload,
|
||||||
@ -337,6 +341,10 @@ fn roa_cache_projection_groups_multiple_outputs_by_roa_uri() {
|
|||||||
|
|
||||||
assert_eq!(projection.entries.len(), 1);
|
assert_eq!(projection.entries.len(), 1);
|
||||||
assert_eq!(projection.entries[0].outputs.len(), 2);
|
assert_eq!(projection.entries[0].outputs.len(), 2);
|
||||||
|
assert_eq!(
|
||||||
|
projection.entries[0].outputs_effective_until_unix,
|
||||||
|
12 * 3600
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@ -16,8 +16,7 @@ use crate::parallel::object_worker::{
|
|||||||
use crate::policy::{Policy, SignedObjectFailurePolicy};
|
use crate::policy::{Policy, SignedObjectFailurePolicy};
|
||||||
use crate::report::{RfcRef, Warning};
|
use crate::report::{RfcRef, Warning};
|
||||||
use crate::storage::{
|
use crate::storage::{
|
||||||
PackFile, PackTime, RoaCacheObjectMeta, RoaCacheProjection, ValidatedCaInstanceResult,
|
PackFile, PackTime, RoaCacheObjectMeta, RoaCacheProjection, VcirLocalOutput,
|
||||||
VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirLocalOutput,
|
|
||||||
VcirLocalOutputPayload, VcirOutputType, VcirSourceObjectType,
|
VcirLocalOutputPayload, VcirOutputType, VcirSourceObjectType,
|
||||||
};
|
};
|
||||||
use crate::validation::cert_path::{CertPathError, validate_signed_object_ee_cert_path_fast};
|
use crate::validation::cert_path::{CertPathError, validate_signed_object_ee_cert_path_fast};
|
||||||
@ -170,6 +169,11 @@ pub struct RoaValidationCacheStats {
|
|||||||
pub metadata_blocked_roas: usize,
|
pub metadata_blocked_roas: usize,
|
||||||
pub context_gate_nanos: u64,
|
pub context_gate_nanos: u64,
|
||||||
pub lookup_nanos: u64,
|
pub lookup_nanos: u64,
|
||||||
|
pub lookup_entry_gate_nanos: u64,
|
||||||
|
pub lookup_crl_gate_nanos: u64,
|
||||||
|
pub lookup_materialize_nanos: u64,
|
||||||
|
pub lookup_crl_gate_verified_crls: usize,
|
||||||
|
pub lookup_crl_gate_reused_crls: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug)]
|
#[derive(Clone, Copy, Debug)]
|
||||||
@ -232,6 +236,17 @@ impl RoaValidationCacheStats {
|
|||||||
.context_gate_nanos
|
.context_gate_nanos
|
||||||
.saturating_add(other.context_gate_nanos);
|
.saturating_add(other.context_gate_nanos);
|
||||||
self.lookup_nanos = self.lookup_nanos.saturating_add(other.lookup_nanos);
|
self.lookup_nanos = self.lookup_nanos.saturating_add(other.lookup_nanos);
|
||||||
|
self.lookup_entry_gate_nanos = self
|
||||||
|
.lookup_entry_gate_nanos
|
||||||
|
.saturating_add(other.lookup_entry_gate_nanos);
|
||||||
|
self.lookup_crl_gate_nanos = self
|
||||||
|
.lookup_crl_gate_nanos
|
||||||
|
.saturating_add(other.lookup_crl_gate_nanos);
|
||||||
|
self.lookup_materialize_nanos = self
|
||||||
|
.lookup_materialize_nanos
|
||||||
|
.saturating_add(other.lookup_materialize_nanos);
|
||||||
|
self.lookup_crl_gate_verified_crls += other.lookup_crl_gate_verified_crls;
|
||||||
|
self.lookup_crl_gate_reused_crls += other.lookup_crl_gate_reused_crls;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn for_input(input: RoaValidationCacheInput<'_>, roa_total: usize) -> Self {
|
fn for_input(input: RoaValidationCacheInput<'_>, roa_total: usize) -> Self {
|
||||||
@ -251,6 +266,21 @@ impl RoaValidationCacheStats {
|
|||||||
stats
|
stats
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn record_lookup(&mut self, total_nanos: u64, metrics: RoaCacheLookupMetrics) {
|
||||||
|
self.lookup_nanos = self.lookup_nanos.saturating_add(total_nanos);
|
||||||
|
self.lookup_entry_gate_nanos = self
|
||||||
|
.lookup_entry_gate_nanos
|
||||||
|
.saturating_add(metrics.entry_gate_nanos);
|
||||||
|
self.lookup_crl_gate_nanos = self
|
||||||
|
.lookup_crl_gate_nanos
|
||||||
|
.saturating_add(metrics.crl_gate_nanos);
|
||||||
|
self.lookup_materialize_nanos = self
|
||||||
|
.lookup_materialize_nanos
|
||||||
|
.saturating_add(metrics.materialize_nanos);
|
||||||
|
self.lookup_crl_gate_verified_crls += metrics.crl_gate_verified_crls;
|
||||||
|
self.lookup_crl_gate_reused_crls += metrics.crl_gate_reused_crls;
|
||||||
|
}
|
||||||
|
|
||||||
fn record_to_timing(&self, timing: Option<&TimingHandle>) {
|
fn record_to_timing(&self, timing: Option<&TimingHandle>) {
|
||||||
let Some(timing) = timing else {
|
let Some(timing) = timing else {
|
||||||
return;
|
return;
|
||||||
@ -318,11 +348,48 @@ impl RoaValidationCacheStats {
|
|||||||
"roa_validation_cache_lookup_nanos",
|
"roa_validation_cache_lookup_nanos",
|
||||||
self.lookup_nanos as usize,
|
self.lookup_nanos as usize,
|
||||||
);
|
);
|
||||||
|
record_non_zero(
|
||||||
|
timing,
|
||||||
|
"roa_validation_cache_lookup_entry_gate_nanos",
|
||||||
|
self.lookup_entry_gate_nanos as usize,
|
||||||
|
);
|
||||||
|
record_non_zero(
|
||||||
|
timing,
|
||||||
|
"roa_validation_cache_lookup_crl_gate_nanos",
|
||||||
|
self.lookup_crl_gate_nanos as usize,
|
||||||
|
);
|
||||||
|
record_non_zero(
|
||||||
|
timing,
|
||||||
|
"roa_validation_cache_lookup_materialize_nanos",
|
||||||
|
self.lookup_materialize_nanos as usize,
|
||||||
|
);
|
||||||
|
record_non_zero(
|
||||||
|
timing,
|
||||||
|
"roa_validation_cache_lookup_crl_gate_verified_crls",
|
||||||
|
self.lookup_crl_gate_verified_crls,
|
||||||
|
);
|
||||||
|
record_non_zero(
|
||||||
|
timing,
|
||||||
|
"roa_validation_cache_lookup_crl_gate_reused_crls",
|
||||||
|
self.lookup_crl_gate_reused_crls,
|
||||||
|
);
|
||||||
timing.record_phase_nanos(
|
timing.record_phase_nanos(
|
||||||
"roa_validation_cache_context_gate_total",
|
"roa_validation_cache_context_gate_total",
|
||||||
self.context_gate_nanos,
|
self.context_gate_nanos,
|
||||||
);
|
);
|
||||||
timing.record_phase_nanos("roa_validation_cache_lookup_total", self.lookup_nanos);
|
timing.record_phase_nanos("roa_validation_cache_lookup_total", self.lookup_nanos);
|
||||||
|
timing.record_phase_nanos(
|
||||||
|
"roa_validation_cache_lookup_entry_gate_total",
|
||||||
|
self.lookup_entry_gate_nanos,
|
||||||
|
);
|
||||||
|
timing.record_phase_nanos(
|
||||||
|
"roa_validation_cache_lookup_crl_gate_total",
|
||||||
|
self.lookup_crl_gate_nanos,
|
||||||
|
);
|
||||||
|
timing.record_phase_nanos(
|
||||||
|
"roa_validation_cache_lookup_materialize_total",
|
||||||
|
self.lookup_materialize_nanos,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -332,6 +399,98 @@ fn record_non_zero(timing: &TimingHandle, key: &'static str, value: usize) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn elapsed_nanos_u64(started: Instant) -> u64 {
|
||||||
|
started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||||
|
struct RoaCacheLookupMetrics {
|
||||||
|
entry_gate_nanos: u64,
|
||||||
|
crl_gate_nanos: u64,
|
||||||
|
materialize_nanos: u64,
|
||||||
|
crl_gate_verified_crls: usize,
|
||||||
|
crl_gate_reused_crls: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
enum RoaCacheCrlGate {
|
||||||
|
Unchanged,
|
||||||
|
ChangedValid(Arc<VerifiedIssuerCrl>),
|
||||||
|
Expired,
|
||||||
|
Invalid,
|
||||||
|
Missing,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
struct RoaCacheCrlGateSet {
|
||||||
|
gates_by_uri: HashMap<String, RoaCacheCrlGate>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RoaCacheCrlGateSet {
|
||||||
|
fn evaluate(
|
||||||
|
&mut self,
|
||||||
|
crl_uri: &str,
|
||||||
|
expected_crl_sha256_by_uri: &HashMap<String, String>,
|
||||||
|
crl_cache: &mut std::collections::HashMap<String, CachedIssuerCrl>,
|
||||||
|
issuer_ca_der: &[u8],
|
||||||
|
validation_time: time::OffsetDateTime,
|
||||||
|
metrics: &mut RoaCacheLookupMetrics,
|
||||||
|
) -> RoaCacheCrlGate {
|
||||||
|
if let Some(gate) = self.gates_by_uri.get(crl_uri) {
|
||||||
|
metrics.crl_gate_reused_crls += 1;
|
||||||
|
return gate.clone();
|
||||||
|
}
|
||||||
|
|
||||||
|
let gate = evaluate_roa_cache_crl_gate(
|
||||||
|
crl_uri,
|
||||||
|
expected_crl_sha256_by_uri,
|
||||||
|
crl_cache,
|
||||||
|
issuer_ca_der,
|
||||||
|
validation_time,
|
||||||
|
metrics,
|
||||||
|
);
|
||||||
|
self.gates_by_uri.insert(crl_uri.to_string(), gate.clone());
|
||||||
|
gate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn evaluate_roa_cache_crl_gate(
|
||||||
|
crl_uri: &str,
|
||||||
|
expected_crl_sha256_by_uri: &HashMap<String, String>,
|
||||||
|
crl_cache: &mut std::collections::HashMap<String, CachedIssuerCrl>,
|
||||||
|
issuer_ca_der: &[u8],
|
||||||
|
validation_time: time::OffsetDateTime,
|
||||||
|
metrics: &mut RoaCacheLookupMetrics,
|
||||||
|
) -> RoaCacheCrlGate {
|
||||||
|
let crl_unchanged = {
|
||||||
|
let Some(current_crl_hash) = crl_cache
|
||||||
|
.get_mut(crl_uri)
|
||||||
|
.map(CachedIssuerCrl::current_sha256_hex)
|
||||||
|
else {
|
||||||
|
return RoaCacheCrlGate::Missing;
|
||||||
|
};
|
||||||
|
expected_crl_sha256_by_uri
|
||||||
|
.get(crl_uri)
|
||||||
|
.map(|expected| expected == current_crl_hash)
|
||||||
|
.unwrap_or(false)
|
||||||
|
};
|
||||||
|
if crl_unchanged {
|
||||||
|
return RoaCacheCrlGate::Unchanged;
|
||||||
|
}
|
||||||
|
|
||||||
|
let verified_crl = match ensure_issuer_crl_verified(crl_uri, crl_cache, issuer_ca_der) {
|
||||||
|
Ok(verified_crl) => {
|
||||||
|
metrics.crl_gate_verified_crls += 1;
|
||||||
|
verified_crl
|
||||||
|
}
|
||||||
|
Err(_) => return RoaCacheCrlGate::Invalid,
|
||||||
|
};
|
||||||
|
if !crl_valid_at_time(&verified_crl.crl, validation_time) {
|
||||||
|
return RoaCacheCrlGate::Expired;
|
||||||
|
}
|
||||||
|
RoaCacheCrlGate::ChangedValid(verified_crl)
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct RoaValidationCacheView {
|
pub struct RoaValidationCacheView {
|
||||||
entries_by_uri: HashMap<String, CachedRoaValidationResult>,
|
entries_by_uri: HashMap<String, CachedRoaValidationResult>,
|
||||||
@ -347,6 +506,7 @@ pub struct CachedRoaValidationResult {
|
|||||||
source_object_hash: [u8; 32],
|
source_object_hash: [u8; 32],
|
||||||
ee_serial: Option<Vec<u8>>,
|
ee_serial: Option<Vec<u8>>,
|
||||||
crl_uri: Option<String>,
|
crl_uri: Option<String>,
|
||||||
|
outputs_effective_until_unix: i64,
|
||||||
outputs: Vec<VcirLocalOutput>,
|
outputs: Vec<VcirLocalOutput>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -403,6 +563,7 @@ impl RoaValidationCacheView {
|
|||||||
source_object_hash: entry.source_object_hash,
|
source_object_hash: entry.source_object_hash,
|
||||||
ee_serial: entry.ee_serial.clone(),
|
ee_serial: entry.ee_serial.clone(),
|
||||||
crl_uri: entry.crl_uri.clone(),
|
crl_uri: entry.crl_uri.clone(),
|
||||||
|
outputs_effective_until_unix: entry.outputs_effective_until_unix,
|
||||||
outputs,
|
outputs,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@ -461,59 +622,112 @@ impl RoaValidationCacheView {
|
|||||||
issuer_ca_der: &[u8],
|
issuer_ca_der: &[u8],
|
||||||
validation_time: time::OffsetDateTime,
|
validation_time: time::OffsetDateTime,
|
||||||
) -> RoaCacheLookupResult {
|
) -> RoaCacheLookupResult {
|
||||||
|
self.lookup_with_metrics(file, crl_cache, issuer_ca_der, validation_time, None)
|
||||||
|
.0
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lookup_with_metrics(
|
||||||
|
&self,
|
||||||
|
file: &PackFile,
|
||||||
|
crl_cache: &mut std::collections::HashMap<String, CachedIssuerCrl>,
|
||||||
|
issuer_ca_der: &[u8],
|
||||||
|
validation_time: time::OffsetDateTime,
|
||||||
|
crl_gate_set: Option<&mut RoaCacheCrlGateSet>,
|
||||||
|
) -> (RoaCacheLookupResult, RoaCacheLookupMetrics) {
|
||||||
|
let mut metrics = RoaCacheLookupMetrics::default();
|
||||||
|
let entry_gate_started = Instant::now();
|
||||||
|
macro_rules! return_entry_gate {
|
||||||
|
($result:expr) => {{
|
||||||
|
metrics.entry_gate_nanos = metrics
|
||||||
|
.entry_gate_nanos
|
||||||
|
.saturating_add(elapsed_nanos_u64(entry_gate_started));
|
||||||
|
return ($result, metrics);
|
||||||
|
}};
|
||||||
|
}
|
||||||
|
macro_rules! return_crl_gate {
|
||||||
|
($started:expr, $result:expr) => {{
|
||||||
|
metrics.crl_gate_nanos = metrics
|
||||||
|
.crl_gate_nanos
|
||||||
|
.saturating_add(elapsed_nanos_u64($started));
|
||||||
|
return ($result, metrics);
|
||||||
|
}};
|
||||||
|
}
|
||||||
|
macro_rules! return_materialize {
|
||||||
|
($started:expr, $result:expr) => {{
|
||||||
|
metrics.materialize_nanos = metrics
|
||||||
|
.materialize_nanos
|
||||||
|
.saturating_add(elapsed_nanos_u64($started));
|
||||||
|
return ($result, metrics);
|
||||||
|
}};
|
||||||
|
}
|
||||||
|
|
||||||
if self.blocked {
|
if self.blocked {
|
||||||
return RoaCacheLookupResult::ExpiredBlocked;
|
return_entry_gate!(RoaCacheLookupResult::ExpiredBlocked);
|
||||||
}
|
}
|
||||||
|
|
||||||
let Some(cached) = self.entries_by_uri.get(file.rsync_uri.as_str()) else {
|
let Some(cached) = self.entries_by_uri.get(file.rsync_uri.as_str()) else {
|
||||||
return RoaCacheLookupResult::Miss;
|
return_entry_gate!(RoaCacheLookupResult::Miss);
|
||||||
};
|
};
|
||||||
if cached.source_object_hash != file.sha256 {
|
if cached.source_object_hash != file.sha256 {
|
||||||
return RoaCacheLookupResult::HashBlocked;
|
return_entry_gate!(RoaCacheLookupResult::HashBlocked);
|
||||||
}
|
}
|
||||||
if cached.outputs.is_empty() {
|
if cached.outputs.is_empty() {
|
||||||
return RoaCacheLookupResult::Miss;
|
return_entry_gate!(RoaCacheLookupResult::Miss);
|
||||||
}
|
}
|
||||||
if cached.outputs.iter().any(|output| {
|
if cached.outputs_effective_until_unix <= validation_time.unix_timestamp() {
|
||||||
output
|
return_entry_gate!(RoaCacheLookupResult::ExpiredBlocked);
|
||||||
.item_effective_until
|
|
||||||
.parse()
|
|
||||||
.map_or(true, |t| t <= validation_time)
|
|
||||||
}) {
|
|
||||||
return RoaCacheLookupResult::ExpiredBlocked;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let Some(crl_uri) = cached.crl_uri.as_deref() else {
|
let Some(crl_uri) = cached.crl_uri.as_deref() else {
|
||||||
return RoaCacheLookupResult::MetadataBlocked;
|
return_entry_gate!(RoaCacheLookupResult::MetadataBlocked);
|
||||||
};
|
};
|
||||||
let Some(ee_serial) = cached.ee_serial.as_ref() else {
|
let Some(ee_serial) = cached.ee_serial.as_ref() else {
|
||||||
return RoaCacheLookupResult::MetadataBlocked;
|
return_entry_gate!(RoaCacheLookupResult::MetadataBlocked);
|
||||||
};
|
};
|
||||||
let crl_unchanged = {
|
metrics.entry_gate_nanos = metrics
|
||||||
let Some(current_crl_hash) = crl_cache
|
.entry_gate_nanos
|
||||||
.get_mut(crl_uri)
|
.saturating_add(elapsed_nanos_u64(entry_gate_started));
|
||||||
.map(CachedIssuerCrl::current_sha256_hex)
|
|
||||||
else {
|
|
||||||
return RoaCacheLookupResult::MetadataBlocked;
|
|
||||||
};
|
|
||||||
self.crl_sha256_by_uri
|
|
||||||
.get(crl_uri)
|
|
||||||
.map(|expected| expected == current_crl_hash)
|
|
||||||
.unwrap_or(false)
|
|
||||||
};
|
|
||||||
if !crl_unchanged {
|
|
||||||
let verified_crl = match ensure_issuer_crl_verified(crl_uri, crl_cache, issuer_ca_der) {
|
|
||||||
Ok(verified_crl) => verified_crl,
|
|
||||||
Err(_) => return RoaCacheLookupResult::MetadataBlocked,
|
|
||||||
};
|
|
||||||
if !crl_valid_at_time(&verified_crl.crl, validation_time) {
|
|
||||||
return RoaCacheLookupResult::ExpiredBlocked;
|
|
||||||
}
|
|
||||||
if verified_crl.revoked_serials.contains(ee_serial) {
|
|
||||||
return RoaCacheLookupResult::RevokedBlocked;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
let crl_gate_started = Instant::now();
|
||||||
|
let crl_gate = if let Some(crl_gate_set) = crl_gate_set {
|
||||||
|
crl_gate_set.evaluate(
|
||||||
|
crl_uri,
|
||||||
|
&self.crl_sha256_by_uri,
|
||||||
|
crl_cache,
|
||||||
|
issuer_ca_der,
|
||||||
|
validation_time,
|
||||||
|
&mut metrics,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
evaluate_roa_cache_crl_gate(
|
||||||
|
crl_uri,
|
||||||
|
&self.crl_sha256_by_uri,
|
||||||
|
crl_cache,
|
||||||
|
issuer_ca_der,
|
||||||
|
validation_time,
|
||||||
|
&mut metrics,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let crl_rechecked = match crl_gate {
|
||||||
|
RoaCacheCrlGate::Unchanged => false,
|
||||||
|
RoaCacheCrlGate::ChangedValid(verified_crl) => {
|
||||||
|
if verified_crl.revoked_serials.contains(ee_serial) {
|
||||||
|
return_crl_gate!(crl_gate_started, RoaCacheLookupResult::RevokedBlocked);
|
||||||
|
}
|
||||||
|
true
|
||||||
|
}
|
||||||
|
RoaCacheCrlGate::Expired => {
|
||||||
|
return_crl_gate!(crl_gate_started, RoaCacheLookupResult::ExpiredBlocked);
|
||||||
|
}
|
||||||
|
RoaCacheCrlGate::Invalid | RoaCacheCrlGate::Missing => {
|
||||||
|
return_crl_gate!(crl_gate_started, RoaCacheLookupResult::MetadataBlocked);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
metrics.crl_gate_nanos = metrics
|
||||||
|
.crl_gate_nanos
|
||||||
|
.saturating_add(elapsed_nanos_u64(crl_gate_started));
|
||||||
|
|
||||||
|
let materialize_started = Instant::now();
|
||||||
let mut vrps = Vec::with_capacity(cached.outputs.len());
|
let mut vrps = Vec::with_capacity(cached.outputs.len());
|
||||||
for output in &cached.outputs {
|
for output in &cached.outputs {
|
||||||
let VcirLocalOutputPayload::Vrp {
|
let VcirLocalOutputPayload::Vrp {
|
||||||
@ -524,7 +738,7 @@ impl RoaValidationCacheView {
|
|||||||
max_length,
|
max_length,
|
||||||
} = &output.payload
|
} = &output.payload
|
||||||
else {
|
else {
|
||||||
return RoaCacheLookupResult::MetadataBlocked;
|
return_materialize!(materialize_started, RoaCacheLookupResult::MetadataBlocked);
|
||||||
};
|
};
|
||||||
vrps.push(Vrp {
|
vrps.push(Vrp {
|
||||||
asn: *asn,
|
asn: *asn,
|
||||||
@ -548,10 +762,13 @@ impl RoaValidationCacheView {
|
|||||||
crl_uri: crl_uri.to_string(),
|
crl_uri: crl_uri.to_string(),
|
||||||
}),
|
}),
|
||||||
};
|
};
|
||||||
if crl_unchanged {
|
metrics.materialize_nanos = metrics
|
||||||
RoaCacheLookupResult::Hit(ok)
|
.materialize_nanos
|
||||||
|
.saturating_add(elapsed_nanos_u64(materialize_started));
|
||||||
|
if crl_rechecked {
|
||||||
|
(RoaCacheLookupResult::CrlRecheckHit(ok), metrics)
|
||||||
} else {
|
} else {
|
||||||
RoaCacheLookupResult::CrlRecheckHit(ok)
|
(RoaCacheLookupResult::Hit(ok), metrics)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -946,19 +1163,24 @@ pub fn process_publication_point_for_issuer_with_cache_options<P: PublicationPoi
|
|||||||
&mut roa_cache_stats,
|
&mut roa_cache_stats,
|
||||||
stats.roa_total,
|
stats.roa_total,
|
||||||
);
|
);
|
||||||
|
let mut crl_gate_set = if active_cache_view.is_some() {
|
||||||
|
Some(RoaCacheCrlGateSet::default())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
for (idx, file) in locked_files.iter().enumerate() {
|
for (idx, file) in locked_files.iter().enumerate() {
|
||||||
if file.rsync_uri.ends_with(".roa") {
|
if file.rsync_uri.ends_with(".roa") {
|
||||||
let result = if let Some(cache_view) = active_cache_view {
|
let result = if let Some(cache_view) = active_cache_view {
|
||||||
let lookup_started = Instant::now();
|
let lookup_started = Instant::now();
|
||||||
let lookup_result =
|
let (lookup_result, lookup_metrics) = cache_view.lookup_with_metrics(
|
||||||
cache_view.lookup(file, &mut crl_cache, issuer_ca_der, validation_time);
|
file,
|
||||||
roa_cache_stats.lookup_nanos = roa_cache_stats.lookup_nanos.saturating_add(
|
&mut crl_cache,
|
||||||
lookup_started
|
issuer_ca_der,
|
||||||
.elapsed()
|
validation_time,
|
||||||
.as_nanos()
|
crl_gate_set.as_mut(),
|
||||||
.min(u128::from(u64::MAX)) as u64,
|
|
||||||
);
|
);
|
||||||
|
roa_cache_stats.record_lookup(elapsed_nanos_u64(lookup_started), lookup_metrics);
|
||||||
match lookup_result {
|
match lookup_result {
|
||||||
RoaCacheLookupResult::Hit(ok) => {
|
RoaCacheLookupResult::Hit(ok) => {
|
||||||
roa_cache_stats.hit_roas += 1;
|
roa_cache_stats.hit_roas += 1;
|
||||||
@ -1956,6 +2178,11 @@ pub(crate) fn prepare_publication_point_for_parallel_roa_with_cache<P: Publicati
|
|||||||
&mut roa_cache_stats,
|
&mut roa_cache_stats,
|
||||||
stats.roa_total,
|
stats.roa_total,
|
||||||
);
|
);
|
||||||
|
let mut crl_gate_set = if active_cache_view.is_some() {
|
||||||
|
Some(RoaCacheCrlGateSet::default())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
let mut roa_task_indices = Vec::new();
|
let mut roa_task_indices = Vec::new();
|
||||||
let mut cached_roa_results = Vec::new();
|
let mut cached_roa_results = Vec::new();
|
||||||
for (index, file) in locked_files.iter().enumerate() {
|
for (index, file) in locked_files.iter().enumerate() {
|
||||||
@ -1964,14 +2191,14 @@ pub(crate) fn prepare_publication_point_for_parallel_roa_with_cache<P: Publicati
|
|||||||
}
|
}
|
||||||
if let Some(cache_view) = active_cache_view {
|
if let Some(cache_view) = active_cache_view {
|
||||||
let lookup_started = Instant::now();
|
let lookup_started = Instant::now();
|
||||||
let lookup_result =
|
let (lookup_result, lookup_metrics) = cache_view.lookup_with_metrics(
|
||||||
cache_view.lookup(file, &mut crl_cache, issuer_ca_der, validation_time);
|
file,
|
||||||
roa_cache_stats.lookup_nanos = roa_cache_stats.lookup_nanos.saturating_add(
|
&mut crl_cache,
|
||||||
lookup_started
|
issuer_ca_der,
|
||||||
.elapsed()
|
validation_time,
|
||||||
.as_nanos()
|
crl_gate_set.as_mut(),
|
||||||
.min(u128::from(u64::MAX)) as u64,
|
|
||||||
);
|
);
|
||||||
|
roa_cache_stats.record_lookup(elapsed_nanos_u64(lookup_started), lookup_metrics);
|
||||||
match lookup_result {
|
match lookup_result {
|
||||||
RoaCacheLookupResult::Hit(ok) => {
|
RoaCacheLookupResult::Hit(ok) => {
|
||||||
roa_cache_stats.hit_roas += 1;
|
roa_cache_stats.hit_roas += 1;
|
||||||
@ -3330,8 +3557,9 @@ mod tests {
|
|||||||
use crate::policy::Policy;
|
use crate::policy::Policy;
|
||||||
use crate::storage::{
|
use crate::storage::{
|
||||||
PackTime, RoaCacheObjectMeta, RoaCacheProjection, RoaCacheProjectionContext,
|
PackTime, RoaCacheObjectMeta, RoaCacheProjection, RoaCacheProjectionContext,
|
||||||
ValidatedManifestMeta, VcirAuditSummary, VcirCcrManifestProjection, VcirInstanceGate,
|
ValidatedCaInstanceResult, ValidatedManifestMeta, VcirArtifactKind, VcirArtifactRole,
|
||||||
VcirRelatedArtifact, VcirSummary,
|
VcirArtifactValidationStatus, VcirAuditSummary, VcirCcrManifestProjection,
|
||||||
|
VcirInstanceGate, VcirRelatedArtifact, VcirSummary,
|
||||||
};
|
};
|
||||||
use crate::validation::publication_point::PublicationPointSnapshot;
|
use crate::validation::publication_point::PublicationPointSnapshot;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
@ -3551,6 +3779,50 @@ mod tests {
|
|||||||
assert!(matches!(second, RoaCacheLookupResult::Hit(_)));
|
assert!(matches!(second, RoaCacheLookupResult::Hit(_)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn roa_validation_cache_lookup_reuses_publication_point_crl_gate() {
|
||||||
|
let validation_time = fixed_time("2026-06-05T00:00:00Z");
|
||||||
|
let issuer_der = b"issuer-ca";
|
||||||
|
let crl_bytes = b"current-crl".to_vec();
|
||||||
|
let crl_hash = sha256_32(&crl_bytes);
|
||||||
|
let roa_hash = [0x11; 32];
|
||||||
|
let vcir = sample_roa_cache_vcir(
|
||||||
|
issuer_der,
|
||||||
|
crl_hash,
|
||||||
|
roa_hash,
|
||||||
|
fixed_time("2026-06-07T00:00:00Z"),
|
||||||
|
fixed_time("2026-06-08T00:00:00Z"),
|
||||||
|
);
|
||||||
|
let projection = sample_roa_cache_projection(&vcir, roa_hash);
|
||||||
|
let view = RoaValidationCacheView::from_projection(&projection, validation_time);
|
||||||
|
let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash);
|
||||||
|
let mut crl_cache = sample_crl_cache(crl_bytes);
|
||||||
|
let mut gate_set = RoaCacheCrlGateSet::default();
|
||||||
|
|
||||||
|
let (first, first_metrics) = view.lookup_with_metrics(
|
||||||
|
&file,
|
||||||
|
&mut crl_cache,
|
||||||
|
issuer_der,
|
||||||
|
validation_time,
|
||||||
|
Some(&mut gate_set),
|
||||||
|
);
|
||||||
|
assert!(matches!(first, RoaCacheLookupResult::Hit(_)));
|
||||||
|
assert_eq!(first_metrics.crl_gate_reused_crls, 0);
|
||||||
|
assert_eq!(gate_set.gates_by_uri.len(), 1);
|
||||||
|
|
||||||
|
let (second, second_metrics) = view.lookup_with_metrics(
|
||||||
|
&file,
|
||||||
|
&mut crl_cache,
|
||||||
|
issuer_der,
|
||||||
|
validation_time,
|
||||||
|
Some(&mut gate_set),
|
||||||
|
);
|
||||||
|
assert!(matches!(second, RoaCacheLookupResult::Hit(_)));
|
||||||
|
assert_eq!(second_metrics.crl_gate_reused_crls, 1);
|
||||||
|
assert_eq!(second_metrics.crl_gate_verified_crls, 0);
|
||||||
|
assert_eq!(gate_set.gates_by_uri.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn cached_verified_crl_reports_hash_and_validity_window() {
|
fn cached_verified_crl_reports_hash_and_validity_window() {
|
||||||
let crl_bytes = fixture_bytes(
|
let crl_bytes = fixture_bytes(
|
||||||
@ -3592,6 +3864,10 @@ mod tests {
|
|||||||
let projection = sample_roa_cache_projection(&vcir, roa_hash);
|
let projection = sample_roa_cache_projection(&vcir, roa_hash);
|
||||||
assert_eq!(projection.parent_context_digest, Some(TEST_PARENT_CONTEXT));
|
assert_eq!(projection.parent_context_digest, Some(TEST_PARENT_CONTEXT));
|
||||||
assert_eq!(projection.policy_fingerprint, Some(TEST_POLICY_FINGERPRINT));
|
assert_eq!(projection.policy_fingerprint, Some(TEST_POLICY_FINGERPRINT));
|
||||||
|
assert_eq!(
|
||||||
|
projection.entries[0].outputs_effective_until_unix,
|
||||||
|
fixed_time("2026-06-07T00:00:00Z").unix_timestamp()
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
projection.entries[0].ee_serial.as_deref(),
|
projection.entries[0].ee_serial.as_deref(),
|
||||||
Some(&[0x01][..])
|
Some(&[0x01][..])
|
||||||
@ -3775,6 +4051,11 @@ mod tests {
|
|||||||
stats.context_blocked_roas = 3;
|
stats.context_blocked_roas = 3;
|
||||||
stats.context_gate_nanos = 4;
|
stats.context_gate_nanos = 4;
|
||||||
stats.lookup_nanos = 5;
|
stats.lookup_nanos = 5;
|
||||||
|
stats.lookup_entry_gate_nanos = 6;
|
||||||
|
stats.lookup_crl_gate_nanos = 7;
|
||||||
|
stats.lookup_materialize_nanos = 8;
|
||||||
|
stats.lookup_crl_gate_verified_crls = 9;
|
||||||
|
stats.lookup_crl_gate_reused_crls = 10;
|
||||||
|
|
||||||
let timing = TimingHandle::new(TimingMeta {
|
let timing = TimingHandle::new(TimingMeta {
|
||||||
recorded_at_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(),
|
recorded_at_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(),
|
||||||
@ -3822,6 +4103,26 @@ mod tests {
|
|||||||
4
|
4
|
||||||
);
|
);
|
||||||
assert_eq!(report["counts"]["roa_validation_cache_lookup_nanos"], 5);
|
assert_eq!(report["counts"]["roa_validation_cache_lookup_nanos"], 5);
|
||||||
|
assert_eq!(
|
||||||
|
report["counts"]["roa_validation_cache_lookup_entry_gate_nanos"],
|
||||||
|
6
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
report["counts"]["roa_validation_cache_lookup_crl_gate_nanos"],
|
||||||
|
7
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
report["counts"]["roa_validation_cache_lookup_materialize_nanos"],
|
||||||
|
8
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
report["counts"]["roa_validation_cache_lookup_crl_gate_verified_crls"],
|
||||||
|
9
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
report["counts"]["roa_validation_cache_lookup_crl_gate_reused_crls"],
|
||||||
|
10
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -3960,6 +4261,11 @@ mod tests {
|
|||||||
)
|
)
|
||||||
.local_outputs
|
.local_outputs
|
||||||
.remove(0);
|
.remove(0);
|
||||||
|
let outputs_effective_until_unix = output
|
||||||
|
.item_effective_until
|
||||||
|
.parse()
|
||||||
|
.expect("parse output effective until")
|
||||||
|
.unix_timestamp();
|
||||||
let mut view = RoaValidationCacheView {
|
let mut view = RoaValidationCacheView {
|
||||||
entries_by_uri: HashMap::new(),
|
entries_by_uri: HashMap::new(),
|
||||||
issuer_ca_sha256_hex: Some(sha256_hex(issuer_der)),
|
issuer_ca_sha256_hex: Some(sha256_hex(issuer_der)),
|
||||||
@ -3981,6 +4287,7 @@ mod tests {
|
|||||||
source_object_hash: roa_hash,
|
source_object_hash: roa_hash,
|
||||||
ee_serial: Some(vec![0x01]),
|
ee_serial: Some(vec![0x01]),
|
||||||
crl_uri: None,
|
crl_uri: None,
|
||||||
|
outputs_effective_until_unix,
|
||||||
outputs: vec![output.clone()],
|
outputs: vec![output.clone()],
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@ -3996,6 +4303,7 @@ mod tests {
|
|||||||
source_object_hash: roa_hash,
|
source_object_hash: roa_hash,
|
||||||
ee_serial: None,
|
ee_serial: None,
|
||||||
crl_uri: Some(TEST_CRL_URI.to_string()),
|
crl_uri: Some(TEST_CRL_URI.to_string()),
|
||||||
|
outputs_effective_until_unix,
|
||||||
outputs: vec![output.clone()],
|
outputs: vec![output.clone()],
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@ -4011,6 +4319,7 @@ mod tests {
|
|||||||
source_object_hash: roa_hash,
|
source_object_hash: roa_hash,
|
||||||
ee_serial: Some(vec![0x01]),
|
ee_serial: Some(vec![0x01]),
|
||||||
crl_uri: Some(TEST_CRL_URI.to_string()),
|
crl_uri: Some(TEST_CRL_URI.to_string()),
|
||||||
|
outputs_effective_until_unix,
|
||||||
outputs: vec![output.clone()],
|
outputs: vec![output.clone()],
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@ -4036,6 +4345,7 @@ mod tests {
|
|||||||
source_object_hash: [0xff; 32],
|
source_object_hash: [0xff; 32],
|
||||||
ee_serial: Some(vec![0x01]),
|
ee_serial: Some(vec![0x01]),
|
||||||
crl_uri: Some(TEST_CRL_URI.to_string()),
|
crl_uri: Some(TEST_CRL_URI.to_string()),
|
||||||
|
outputs_effective_until_unix,
|
||||||
outputs: vec![output.clone()],
|
outputs: vec![output.clone()],
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@ -4051,6 +4361,7 @@ mod tests {
|
|||||||
source_object_hash: roa_hash,
|
source_object_hash: roa_hash,
|
||||||
ee_serial: Some(vec![0x01]),
|
ee_serial: Some(vec![0x01]),
|
||||||
crl_uri: Some(TEST_CRL_URI.to_string()),
|
crl_uri: Some(TEST_CRL_URI.to_string()),
|
||||||
|
outputs_effective_until_unix,
|
||||||
outputs: Vec::new(),
|
outputs: Vec::new(),
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@ -4070,6 +4381,7 @@ mod tests {
|
|||||||
source_object_hash: roa_hash,
|
source_object_hash: roa_hash,
|
||||||
ee_serial: Some(vec![0x01]),
|
ee_serial: Some(vec![0x01]),
|
||||||
crl_uri: Some(TEST_CRL_URI.to_string()),
|
crl_uri: Some(TEST_CRL_URI.to_string()),
|
||||||
|
outputs_effective_until_unix,
|
||||||
outputs: vec![output],
|
outputs: vec![output],
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user