20260607 完成transport预热、roa cache和finalize长尾优化
This commit is contained in:
@@ -15,9 +15,9 @@ use crate::parallel::object_worker::{
|
||||
use crate::policy::{Policy, SignedObjectFailurePolicy};
|
||||
use crate::report::{RfcRef, Warning};
|
||||
use crate::storage::{
|
||||
PackFile, PackTime, ValidatedCaInstanceResult, VcirArtifactKind, VcirArtifactRole,
|
||||
VcirArtifactValidationStatus, VcirLocalOutput, VcirLocalOutputPayload, VcirOutputType,
|
||||
VcirSourceObjectType,
|
||||
PackFile, PackTime, RoaCacheProjection, ValidatedCaInstanceResult, VcirArtifactKind,
|
||||
VcirArtifactRole, VcirArtifactValidationStatus, VcirLocalOutput, VcirLocalOutputPayload,
|
||||
VcirOutputType, VcirSourceObjectType,
|
||||
};
|
||||
use crate::validation::cert_path::{CertPathError, validate_signed_object_ee_cert_path_fast};
|
||||
use crate::validation::manifest::PublicationPointData;
|
||||
@@ -264,6 +264,65 @@ pub struct CachedRoaValidationResult {
|
||||
}
|
||||
|
||||
impl RoaValidationCacheView {
|
||||
pub fn from_projection(
|
||||
projection: &RoaCacheProjection,
|
||||
validation_time: time::OffsetDateTime,
|
||||
) -> Self {
|
||||
let mut entries_by_uri: HashMap<String, CachedRoaValidationResult> =
|
||||
HashMap::with_capacity(projection.entries.len());
|
||||
let issuer_ca_sha256_hex = projection.issuer_ca_sha256_hex.clone();
|
||||
let crl_sha256_by_uri = projection
|
||||
.crl_sha256_by_uri
|
||||
.iter()
|
||||
.map(|crl| (crl.uri.clone(), crl.sha256.clone()))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let blocked = projection
|
||||
.instance_effective_until
|
||||
.parse()
|
||||
.map(|effective_until| effective_until <= validation_time)
|
||||
.unwrap_or(true);
|
||||
|
||||
if blocked {
|
||||
return Self {
|
||||
entries_by_uri,
|
||||
issuer_ca_sha256_hex,
|
||||
crl_sha256_by_uri,
|
||||
blocked,
|
||||
};
|
||||
}
|
||||
|
||||
for entry in &projection.entries {
|
||||
let outputs = entry
|
||||
.outputs
|
||||
.iter()
|
||||
.map(|output| VcirLocalOutput {
|
||||
output_type: VcirOutputType::Vrp,
|
||||
item_effective_until: output.item_effective_until.clone(),
|
||||
source_object_uri: entry.source_object_uri.clone(),
|
||||
source_object_type: VcirSourceObjectType::Roa,
|
||||
source_object_hash: entry.source_object_hash,
|
||||
source_ee_cert_hash: output.source_ee_cert_hash,
|
||||
payload: output.payload.clone(),
|
||||
rule_hash: output.rule_hash,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
entries_by_uri.insert(
|
||||
entry.source_object_uri.clone(),
|
||||
CachedRoaValidationResult {
|
||||
source_object_hash: entry.source_object_hash,
|
||||
outputs,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
Self {
|
||||
entries_by_uri,
|
||||
issuer_ca_sha256_hex,
|
||||
crl_sha256_by_uri,
|
||||
blocked,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_vcir(
|
||||
vcir: &ValidatedCaInstanceResult,
|
||||
validation_time: time::OffsetDateTime,
|
||||
@@ -3067,8 +3126,8 @@ mod tests {
|
||||
};
|
||||
use crate::policy::Policy;
|
||||
use crate::storage::{
|
||||
PackTime, ValidatedManifestMeta, VcirAuditSummary, VcirCcrManifestProjection,
|
||||
VcirInstanceGate, VcirRelatedArtifact, VcirSummary,
|
||||
PackTime, RoaCacheProjection, ValidatedManifestMeta, VcirAuditSummary,
|
||||
VcirCcrManifestProjection, VcirInstanceGate, VcirRelatedArtifact, VcirSummary,
|
||||
};
|
||||
use crate::validation::publication_point::PublicationPointSnapshot;
|
||||
use std::collections::HashMap;
|
||||
@@ -3214,6 +3273,120 @@ mod tests {
|
||||
assert_eq!(ok.local_outputs.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roa_validation_cache_view_from_projection_matches_vcir_view() {
|
||||
let validation_time = fixed_time("2026-06-05T00:00:00Z");
|
||||
let issuer_der = b"issuer-ca";
|
||||
let crl_hash = [0x22; 32];
|
||||
let roa_hash = [0x11; 32];
|
||||
let vcir = sample_roa_cache_vcir(
|
||||
issuer_der,
|
||||
crl_hash,
|
||||
roa_hash,
|
||||
fixed_time("2026-06-07T00:00:00Z"),
|
||||
fixed_time("2026-06-08T00:00:00Z"),
|
||||
);
|
||||
let projection = RoaCacheProjection::from_vcir(&vcir)
|
||||
.expect("build projection")
|
||||
.expect("projection exists");
|
||||
let view = RoaValidationCacheView::from_projection(&projection, validation_time);
|
||||
let files = vec![
|
||||
PackFile::from_bytes_with_sha256(
|
||||
"rsync://example.test/repo/a.roa",
|
||||
vec![0x01],
|
||||
roa_hash,
|
||||
),
|
||||
PackFile::from_bytes_with_sha256(
|
||||
"rsync://example.test/repo/current.crl",
|
||||
vec![0x02],
|
||||
crl_hash,
|
||||
),
|
||||
];
|
||||
|
||||
assert!(view.matches_current_context(issuer_der, &files));
|
||||
let hit = view.lookup(&files[0], validation_time);
|
||||
let RoaCacheLookupResult::Hit(ok) = hit else {
|
||||
panic!("expected projection cache hit, got {hit:?}");
|
||||
};
|
||||
assert!(ok.reused_from_cache);
|
||||
assert_eq!(ok.vrps.len(), 1);
|
||||
assert_eq!(ok.vrps[0].asn, 64500);
|
||||
assert_eq!(ok.local_outputs.len(), 1);
|
||||
assert_eq!(
|
||||
ok.local_outputs[0].source_object_uri,
|
||||
"rsync://example.test/repo/a.roa"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roa_validation_cache_view_from_projection_blocks_on_gates() {
|
||||
let validation_time = fixed_time("2026-06-05T00:00:00Z");
|
||||
let issuer_der = b"issuer-ca";
|
||||
let crl_hash = [0x22; 32];
|
||||
let roa_hash = [0x11; 32];
|
||||
let vcir = sample_roa_cache_vcir(
|
||||
issuer_der,
|
||||
crl_hash,
|
||||
roa_hash,
|
||||
fixed_time("2026-06-07T00:00:00Z"),
|
||||
fixed_time("2026-06-08T00:00:00Z"),
|
||||
);
|
||||
let mut projection = RoaCacheProjection::from_vcir(&vcir)
|
||||
.expect("build projection")
|
||||
.expect("projection exists");
|
||||
let files = vec![
|
||||
PackFile::from_bytes_with_sha256(
|
||||
"rsync://example.test/repo/a.roa",
|
||||
vec![0x01],
|
||||
roa_hash,
|
||||
),
|
||||
PackFile::from_bytes_with_sha256(
|
||||
"rsync://example.test/repo/current.crl",
|
||||
vec![0x02],
|
||||
crl_hash,
|
||||
),
|
||||
];
|
||||
|
||||
projection.issuer_ca_sha256_hex = Some("00".repeat(32));
|
||||
let issuer_changed = RoaValidationCacheView::from_projection(&projection, validation_time);
|
||||
assert!(!issuer_changed.matches_current_context(issuer_der, &files));
|
||||
|
||||
let mut projection = RoaCacheProjection::from_vcir(&vcir)
|
||||
.expect("build projection")
|
||||
.expect("projection exists");
|
||||
projection.crl_sha256_by_uri[0].sha256 = "11".repeat(32);
|
||||
let crl_changed = RoaValidationCacheView::from_projection(&projection, validation_time);
|
||||
assert!(!crl_changed.matches_current_context(issuer_der, &files));
|
||||
|
||||
let mut projection = RoaCacheProjection::from_vcir(&vcir)
|
||||
.expect("build projection")
|
||||
.expect("projection exists");
|
||||
projection.entries[0].source_object_hash = [0xff; 32];
|
||||
let roa_changed = RoaValidationCacheView::from_projection(&projection, validation_time);
|
||||
assert!(matches!(
|
||||
roa_changed.lookup(&files[0], validation_time),
|
||||
RoaCacheLookupResult::Blocked
|
||||
));
|
||||
|
||||
let expired_vcir = sample_roa_cache_vcir(
|
||||
issuer_der,
|
||||
crl_hash,
|
||||
roa_hash,
|
||||
fixed_time("2026-06-07T00:00:00Z"),
|
||||
fixed_time("2026-06-04T00:00:00Z"),
|
||||
);
|
||||
let expired_projection = RoaCacheProjection::from_vcir(&expired_vcir)
|
||||
.expect("build projection")
|
||||
.expect("projection exists");
|
||||
let expired_view =
|
||||
RoaValidationCacheView::from_projection(&expired_projection, validation_time);
|
||||
assert!(!expired_view.matches_current_context(issuer_der, &files));
|
||||
assert!(matches!(
|
||||
expired_view.lookup(&files[0], validation_time),
|
||||
RoaCacheLookupResult::Blocked
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roa_validation_cache_view_blocks_when_context_changes() {
|
||||
let validation_time = fixed_time("2026-06-05T00:00:00Z");
|
||||
|
||||
@@ -13,6 +13,7 @@ use crate::parallel::repo_worker::{
|
||||
};
|
||||
use crate::parallel::run_coordinator::GlobalRunCoordinator;
|
||||
use crate::parallel::types::{TalInputSpec, TalSource};
|
||||
use crate::policy::SyncPreference;
|
||||
use crate::replay::archive::ReplayArchiveIndex;
|
||||
use crate::replay::delta_archive::ReplayDeltaArchiveIndex;
|
||||
use crate::replay::delta_fetch_http::PayloadDeltaReplayHttpFetcher;
|
||||
@@ -171,6 +172,7 @@ fn build_phase1_repo_sync_runtime<H, R>(
|
||||
timing: Option<TimingHandle>,
|
||||
download_log: Option<DownloadLogHandle>,
|
||||
tal_inputs: Vec<crate::parallel::types::TalInputSpec>,
|
||||
record_transport_prefetch_requests: bool,
|
||||
) -> Result<(Arc<dyn RepoSyncRuntime>, CurrentRepoIndexHandle), RunTreeFromTalError>
|
||||
where
|
||||
H: Fetcher + Clone + 'static,
|
||||
@@ -184,7 +186,7 @@ where
|
||||
current_repo_index.clone(),
|
||||
Arc::new(http_fetcher.clone()),
|
||||
Arc::clone(&rsync_fetcher_arc),
|
||||
timing,
|
||||
timing.clone(),
|
||||
download_log,
|
||||
);
|
||||
let pool = RepoTransportWorkerPool::new(RepoWorkerPoolConfig::from(¶llel_config), executor)
|
||||
@@ -196,16 +198,142 @@ where
|
||||
let failure_resolver: Arc<dyn Fn(&str) -> Option<String> + Send + Sync> =
|
||||
Arc::new(move |base: &str| failure_rsync_fetcher_arc.failure_dedup_key(base));
|
||||
let _ = policy; // policy reserved for later runtime-level decisions
|
||||
let runtime = Arc::new(Phase1RepoSyncRuntime::new_with_failure_scope(
|
||||
coordinator,
|
||||
pool,
|
||||
resolver,
|
||||
failure_resolver,
|
||||
policy.sync_preference,
|
||||
));
|
||||
let runtime = Arc::new(
|
||||
Phase1RepoSyncRuntime::new_with_failure_scope_and_prefetch_recording(
|
||||
coordinator,
|
||||
pool,
|
||||
resolver,
|
||||
failure_resolver,
|
||||
policy.sync_preference,
|
||||
record_transport_prefetch_requests,
|
||||
),
|
||||
);
|
||||
Ok((runtime, current_repo_index))
|
||||
}
|
||||
|
||||
fn record_transport_prefetch_count(timing: Option<&TimingHandle>, key: &'static str, value: u64) {
|
||||
if value == 0 {
|
||||
return;
|
||||
}
|
||||
if let Some(timing) = timing {
|
||||
timing.record_count(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_transport_request_prefetch(
|
||||
store: &crate::storage::RocksStore,
|
||||
runtime: &Arc<dyn RepoSyncRuntime>,
|
||||
sync_preference: SyncPreference,
|
||||
validation_time: time::OffsetDateTime,
|
||||
config: &TreeRunConfig,
|
||||
timing: Option<&TimingHandle>,
|
||||
) -> Result<(), RunTreeFromTalError> {
|
||||
if !config.enable_transport_request_prefetch {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let snapshot = match store.get_transport_prefetch_snapshot() {
|
||||
Ok(Some(snapshot)) => snapshot,
|
||||
Ok(None) => {
|
||||
crate::progress_log::emit(
|
||||
"phase1_repo_prefetch_missing_snapshot",
|
||||
serde_json::json!({ "status": "missing" }),
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
Err(err) => {
|
||||
record_transport_prefetch_count(timing, "transport_prefetch_load_errors", 1);
|
||||
crate::progress_log::emit(
|
||||
"phase1_repo_prefetch_load_error",
|
||||
serde_json::json!({ "error": err.to_string() }),
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
if !snapshot.is_compatible_with(sync_preference) {
|
||||
record_transport_prefetch_count(
|
||||
timing,
|
||||
"transport_prefetch_skipped_incompatible_snapshots",
|
||||
1,
|
||||
);
|
||||
crate::progress_log::emit(
|
||||
"phase1_repo_prefetch_skipped",
|
||||
serde_json::json!({
|
||||
"reason": "incompatible_snapshot",
|
||||
"schema_version": snapshot.schema_version,
|
||||
"request_count": snapshot.requests.len(),
|
||||
}),
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let stats = runtime
|
||||
.prefetch_transport_requests(&snapshot, validation_time)
|
||||
.map_err(TreeRunError::Runner)?;
|
||||
record_transport_prefetch_count(
|
||||
timing.clone(),
|
||||
"transport_prefetch_loaded_requests",
|
||||
stats.loaded_requests,
|
||||
);
|
||||
record_transport_prefetch_count(
|
||||
timing.clone(),
|
||||
"transport_prefetch_enqueued_tasks",
|
||||
stats.enqueued_tasks,
|
||||
);
|
||||
record_transport_prefetch_count(
|
||||
timing,
|
||||
"transport_prefetch_waiting_requests",
|
||||
stats.waiting_requests,
|
||||
);
|
||||
record_transport_prefetch_count(
|
||||
timing,
|
||||
"transport_prefetch_reused_results",
|
||||
stats.reused_results,
|
||||
);
|
||||
record_transport_prefetch_count(
|
||||
timing,
|
||||
"transport_prefetch_skipped_incompatible_requests",
|
||||
stats.skipped_incompatible,
|
||||
);
|
||||
crate::progress_log::emit(
|
||||
"phase1_repo_prefetch_applied",
|
||||
serde_json::json!({
|
||||
"loaded_requests": stats.loaded_requests,
|
||||
"enqueued_tasks": stats.enqueued_tasks,
|
||||
"waiting_requests": stats.waiting_requests,
|
||||
"reused_results": stats.reused_results,
|
||||
"skipped_incompatible": stats.skipped_incompatible,
|
||||
}),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn persist_transport_request_prefetch_snapshot(
|
||||
store: &crate::storage::RocksStore,
|
||||
runtime: &Arc<dyn RepoSyncRuntime>,
|
||||
config: &TreeRunConfig,
|
||||
timing: Option<&TimingHandle>,
|
||||
) -> Result<(), RunTreeFromTalError> {
|
||||
if !config.enable_transport_request_prefetch {
|
||||
return Ok(());
|
||||
}
|
||||
let snapshot = runtime.transport_prefetch_snapshot();
|
||||
let recorded = snapshot.requests.len() as u64;
|
||||
store
|
||||
.put_transport_prefetch_snapshot(&snapshot)
|
||||
.map_err(|err| TreeRunError::Runner(err.to_string()))?;
|
||||
record_transport_prefetch_count(timing, "transport_prefetch_recorded_requests", recorded);
|
||||
crate::progress_log::emit(
|
||||
"phase1_repo_prefetch_persisted",
|
||||
serde_json::json!({
|
||||
"recorded_requests": recorded,
|
||||
"schema_version": snapshot.schema_version,
|
||||
}),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn root_discovery_from_tal_input(
|
||||
tal_input: &TalInputSpec,
|
||||
http_fetcher: &dyn Fetcher,
|
||||
@@ -588,6 +716,15 @@ where
|
||||
timing.clone(),
|
||||
Some(download_log.clone()),
|
||||
tal_inputs,
|
||||
config.enable_transport_request_prefetch,
|
||||
)?;
|
||||
apply_transport_request_prefetch(
|
||||
store.as_ref(),
|
||||
&runtime,
|
||||
policy.sync_preference,
|
||||
validation_time,
|
||||
config,
|
||||
timing.as_ref(),
|
||||
)?;
|
||||
let current_repo_index_for_output = current_repo_index.clone();
|
||||
let runner = make_live_runner(
|
||||
@@ -596,10 +733,10 @@ where
|
||||
http_fetcher,
|
||||
rsync_fetcher,
|
||||
validation_time,
|
||||
timing,
|
||||
timing.clone(),
|
||||
Some(download_log.clone()),
|
||||
Some(current_repo_index),
|
||||
Some(runtime),
|
||||
Some(Arc::clone(&runtime)),
|
||||
phase2_config,
|
||||
(phase2_enabled && config.build_ccr_accumulator)
|
||||
.then(|| CcrAccumulator::new(vec![discovery.trust_anchor.clone()])),
|
||||
@@ -622,6 +759,7 @@ where
|
||||
} else {
|
||||
run_tree_serial_audit(root, &runner, config)?
|
||||
};
|
||||
persist_transport_request_prefetch_snapshot(store.as_ref(), &runtime, config, timing.as_ref())?;
|
||||
let downloads = download_log.snapshot_events();
|
||||
let download_stats = DownloadLogHandle::stats_from_events(&downloads);
|
||||
Ok(RunTreeFromTalAuditOutput {
|
||||
@@ -696,6 +834,15 @@ where
|
||||
timing.clone(),
|
||||
Some(download_log.clone()),
|
||||
successful_tal_inputs.clone(),
|
||||
config.enable_transport_request_prefetch,
|
||||
)?;
|
||||
apply_transport_request_prefetch(
|
||||
store.as_ref(),
|
||||
&runtime,
|
||||
policy.sync_preference,
|
||||
validation_time,
|
||||
config,
|
||||
timing.as_ref(),
|
||||
)?;
|
||||
let current_repo_index_for_output = current_repo_index.clone();
|
||||
let runner = make_live_runner(
|
||||
@@ -704,10 +851,10 @@ where
|
||||
http_fetcher,
|
||||
rsync_fetcher,
|
||||
validation_time,
|
||||
timing,
|
||||
timing.clone(),
|
||||
Some(download_log.clone()),
|
||||
Some(current_repo_index),
|
||||
Some(runtime),
|
||||
Some(Arc::clone(&runtime)),
|
||||
phase2_config,
|
||||
(phase2_enabled && config.build_ccr_accumulator).then(|| {
|
||||
CcrAccumulator::new(
|
||||
@@ -730,6 +877,7 @@ where
|
||||
} else {
|
||||
run_tree_serial_audit_multi_root(root_handles, &runner, config)?
|
||||
};
|
||||
persist_transport_request_prefetch_snapshot(store.as_ref(), &runtime, config, timing.as_ref())?;
|
||||
let downloads = download_log.snapshot_events();
|
||||
let download_stats = DownloadLogHandle::stats_from_events(&downloads);
|
||||
Ok(RunTreeFromTalAuditOutput {
|
||||
@@ -2329,6 +2477,7 @@ mod replay_api_tests {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
},
|
||||
)
|
||||
.unwrap_err();
|
||||
@@ -2365,6 +2514,7 @@ mod replay_api_tests {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
},
|
||||
)
|
||||
.expect("run replay root-only audit");
|
||||
@@ -2411,6 +2561,7 @@ mod replay_api_tests {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
},
|
||||
)
|
||||
.expect("run replay root-only audit");
|
||||
@@ -2458,6 +2609,7 @@ mod replay_api_tests {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
},
|
||||
&timing,
|
||||
)
|
||||
@@ -2515,6 +2667,7 @@ mod replay_api_tests {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
},
|
||||
)
|
||||
.unwrap_err();
|
||||
@@ -2548,6 +2701,7 @@ mod replay_api_tests {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
},
|
||||
)
|
||||
.unwrap_err();
|
||||
@@ -2601,6 +2755,7 @@ mod replay_api_tests {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
},
|
||||
)
|
||||
.expect("run delta replay root-only audit");
|
||||
@@ -2663,6 +2818,7 @@ mod replay_api_tests {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
},
|
||||
&timing,
|
||||
)
|
||||
|
||||
@@ -22,6 +22,8 @@ pub struct TreeRunConfig {
|
||||
pub build_ccr_accumulator: bool,
|
||||
/// Reuse accepted ROA validation outputs from previous VCIR when explicitly enabled.
|
||||
pub enable_roa_validation_cache: bool,
|
||||
/// Prefetch transport requests from the previous run before normal tree traversal.
|
||||
pub enable_transport_request_prefetch: bool,
|
||||
}
|
||||
|
||||
impl Default for TreeRunConfig {
|
||||
@@ -33,6 +35,7 @@ impl Default for TreeRunConfig {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -113,6 +113,8 @@ pub(crate) struct FreshPublicationPointFinalizeOutput {
|
||||
pub(crate) snapshot_pack_ms: u64,
|
||||
pub(crate) persist_vcir_ms: u64,
|
||||
pub(crate) persist_vcir_timing: PersistVcirTimingBreakdown,
|
||||
pub(crate) ccr_projection_build_ms: u64,
|
||||
pub(crate) ccr_append_ms: u64,
|
||||
pub(crate) audit_build_ms: u64,
|
||||
}
|
||||
|
||||
@@ -163,20 +165,25 @@ impl<'a> Rpkiv1PublicationPointRunner<'a> {
|
||||
return None;
|
||||
}
|
||||
let load_started = std::time::Instant::now();
|
||||
let loaded_vcir = self.store.get_vcir(manifest_rsync_uri);
|
||||
let loaded_projection = self.store.get_roa_cache_projection(manifest_rsync_uri);
|
||||
if let Some(timing) = self.timing.as_ref() {
|
||||
timing.record_phase_nanos(
|
||||
"roa_validation_cache_vcir_load_total",
|
||||
"roa_validation_cache_projection_load_total",
|
||||
load_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64,
|
||||
);
|
||||
}
|
||||
match loaded_vcir {
|
||||
Ok(Some(vcir)) => {
|
||||
match loaded_projection {
|
||||
Ok(Some(projection)) => {
|
||||
if let Some(timing) = self.timing.as_ref() {
|
||||
timing
|
||||
.record_count("roa_validation_cache_projection_hit_publication_points", 1);
|
||||
}
|
||||
let view_started = std::time::Instant::now();
|
||||
let view = RoaValidationCacheView::from_vcir(&vcir, self.validation_time);
|
||||
let view =
|
||||
RoaValidationCacheView::from_projection(&projection, self.validation_time);
|
||||
if let Some(timing) = self.timing.as_ref() {
|
||||
timing.record_phase_nanos(
|
||||
"roa_validation_cache_view_build_total",
|
||||
"roa_validation_cache_projection_build_total",
|
||||
view_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64,
|
||||
);
|
||||
}
|
||||
@@ -184,16 +191,19 @@ impl<'a> Rpkiv1PublicationPointRunner<'a> {
|
||||
}
|
||||
Ok(None) => {
|
||||
if let Some(timing) = self.timing.as_ref() {
|
||||
timing.record_count("roa_validation_cache_vcir_missing_publication_points", 1);
|
||||
timing.record_count(
|
||||
"roa_validation_cache_projection_missing_publication_points",
|
||||
1,
|
||||
);
|
||||
}
|
||||
None
|
||||
}
|
||||
Err(err) => {
|
||||
if let Some(timing) = self.timing.as_ref() {
|
||||
timing.record_count("roa_validation_cache_vcir_load_errors", 1);
|
||||
timing.record_count("roa_validation_cache_projection_load_errors", 1);
|
||||
}
|
||||
crate::progress_log::emit(
|
||||
"roa_validation_cache_vcir_load_error",
|
||||
"roa_validation_cache_projection_load_error",
|
||||
serde_json::json!({
|
||||
"manifest_rsync_uri": manifest_rsync_uri,
|
||||
"error": err.to_string(),
|
||||
@@ -381,12 +391,18 @@ impl<'a> Rpkiv1PublicationPointRunner<'a> {
|
||||
// publication point result is retained for the rest of the run.
|
||||
let _released_local_outputs = std::mem::take(&mut objects.local_outputs_cache);
|
||||
|
||||
let mut ccr_projection_build_ms = 0;
|
||||
let mut ccr_append_ms = 0;
|
||||
if self.ccr_accumulator.is_some() {
|
||||
let ccr_projection_build_started = std::time::Instant::now();
|
||||
let child_entries =
|
||||
build_vcir_child_entries(&discovered_children, self.validation_time)?;
|
||||
let ccr_manifest_projection =
|
||||
build_vcir_ccr_manifest_projection_from_fresh(ca, &pack, &child_entries)?;
|
||||
ccr_projection_build_ms = ccr_projection_build_started.elapsed().as_millis() as u64;
|
||||
let ccr_append_started = std::time::Instant::now();
|
||||
self.append_ccr_manifest_projection(&ccr_manifest_projection)?;
|
||||
ccr_append_ms = ccr_append_started.elapsed().as_millis() as u64;
|
||||
}
|
||||
|
||||
let audit_build_started = std::time::Instant::now();
|
||||
@@ -416,6 +432,8 @@ impl<'a> Rpkiv1PublicationPointRunner<'a> {
|
||||
snapshot_pack_ms,
|
||||
persist_vcir_ms,
|
||||
persist_vcir_timing,
|
||||
ccr_projection_build_ms,
|
||||
ccr_append_ms,
|
||||
audit_build_ms,
|
||||
})
|
||||
}
|
||||
@@ -805,6 +823,8 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
snapshot_pack_ms,
|
||||
persist_vcir_ms,
|
||||
persist_vcir_timing,
|
||||
ccr_projection_build_ms,
|
||||
ccr_append_ms,
|
||||
audit_build_ms,
|
||||
} = finalized;
|
||||
let total_duration_ms = publication_point_started.elapsed().as_millis() as u64;
|
||||
@@ -841,6 +861,8 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
"persist_related_artifacts_ms": persist_vcir_timing.build_vcir.related_artifacts_ms,
|
||||
"persist_vcir_struct_ms": persist_vcir_timing.build_vcir.struct_build_ms,
|
||||
"persist_replace_breakdown": &persist_vcir_timing.replace_vcir,
|
||||
"ccr_projection_build_ms": ccr_projection_build_ms,
|
||||
"ccr_append_ms": ccr_append_ms,
|
||||
"audit_build_ms": audit_build_ms,
|
||||
"warning_count": result.warnings.len(),
|
||||
"vrp_count": result.objects.vrps.len(),
|
||||
@@ -884,6 +906,8 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
"persist_related_artifacts_ms": persist_vcir_timing.build_vcir.related_artifacts_ms,
|
||||
"persist_vcir_struct_ms": persist_vcir_timing.build_vcir.struct_build_ms,
|
||||
"persist_replace_breakdown": &persist_vcir_timing.replace_vcir,
|
||||
"ccr_projection_build_ms": ccr_projection_build_ms,
|
||||
"ccr_append_ms": ccr_append_ms,
|
||||
"audit_build_ms": audit_build_ms,
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -3034,6 +3034,85 @@ fn parse_snapshot_time_value_reports_invalid_timestamp() {
|
||||
assert!(err.contains("invalid RFC3339 time 'not-a-time'"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runner_roa_validation_cache_uses_projection_not_full_vcir_fallback() {
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
let child_cert_hash = sha256_hex(b"child-cert");
|
||||
let mut vcir = sample_vcir_for_projection(now, &child_cert_hash);
|
||||
vcir.local_outputs
|
||||
.retain(|output| output.source_object_type != VcirSourceObjectType::Roa);
|
||||
vcir.summary.local_vrp_count = 0;
|
||||
vcir.summary.local_aspa_count = 1;
|
||||
vcir.summary.local_router_key_count = 1;
|
||||
|
||||
let store_dir = tempfile::tempdir().expect("store dir");
|
||||
let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
|
||||
store.put_vcir(&vcir).expect("put vcir without projection");
|
||||
assert!(
|
||||
store
|
||||
.get_vcir(&vcir.manifest_rsync_uri)
|
||||
.expect("get vcir")
|
||||
.is_some()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_roa_cache_projection(&vcir.manifest_rsync_uri)
|
||||
.expect("get projection")
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta {
|
||||
recorded_at_utc_rfc3339: "2026-06-07T00:00:00Z".to_string(),
|
||||
validation_time_utc_rfc3339: "2026-06-07T00:00:00Z".to_string(),
|
||||
tal_url: None,
|
||||
db_path: None,
|
||||
});
|
||||
let policy = Policy::default();
|
||||
let runner = Rpkiv1PublicationPointRunner {
|
||||
store: &store,
|
||||
policy: &policy,
|
||||
http_fetcher: &NeverHttpFetcher,
|
||||
rsync_fetcher: &FailingRsyncFetcher,
|
||||
validation_time: now,
|
||||
timing: Some(timing.clone()),
|
||||
download_log: None,
|
||||
replay_archive_index: None,
|
||||
replay_delta_index: None,
|
||||
rrdp_dedup: false,
|
||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||
rsync_dedup: false,
|
||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||
current_repo_index: None,
|
||||
repo_sync_runtime: None,
|
||||
parallel_phase2_config: None,
|
||||
parallel_roa_worker_pool: None,
|
||||
ccr_accumulator: None,
|
||||
persist_vcir: true,
|
||||
enable_roa_validation_cache: true,
|
||||
};
|
||||
|
||||
assert!(
|
||||
runner
|
||||
.roa_validation_cache_view_for_fresh_point(&vcir.manifest_rsync_uri)
|
||||
.is_none()
|
||||
);
|
||||
let dir = tempfile::tempdir().expect("timing dir");
|
||||
let path = dir.path().join("timing.json");
|
||||
timing.write_json(&path, 10).expect("write timing");
|
||||
let report: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(path).expect("read timing")).expect("parse timing");
|
||||
assert_eq!(
|
||||
report["counts"]["roa_validation_cache_projection_missing_publication_points"],
|
||||
1
|
||||
);
|
||||
assert!(
|
||||
report["phases"]["roa_validation_cache_projection_load_total"]["count"]
|
||||
.as_u64()
|
||||
.unwrap_or_default()
|
||||
>= 1
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_objects_output_from_vcir_tracks_expired_and_invalid_cached_outputs() {
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
|
||||
Reference in New Issue
Block a user