NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260413_2 并行化架构优化第一阶段，apnic 串行98s->并行74s，传输层任务并行，同repo内发布点串行

Browse Source
This commit is contained in:
yuyr
2026-04-15 09:53:11 +08:00
parent af1c2c7f88
commit 585c41b83b
21 changed files with 5098 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/validation/run.rs
View File

@@ -73,6 +73,7 @@ pub fn run_publication_point_once(
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: false,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
let result = runner

257
src/validation/run_tree_from_tal.rs
View File

@@ -4,6 +4,13 @@ use crate::analysis::timing::TimingHandle;
use crate::audit::PublicationPointAudit;
use crate::audit_downloads::DownloadLogHandle;
use crate::data_model::ta::TrustAnchor;
use crate::parallel::config::ParallelPhase1Config;
use crate::parallel::repo_runtime::{Phase1RepoSyncRuntime, RepoSyncRuntime};
use crate::parallel::repo_worker::{
LiveRepoTransportExecutor, RepoTransportWorkerPool, RepoWorkerPoolConfig,
};
use crate::parallel::run_coordinator::GlobalRunCoordinator;
use crate::parallel::types::{TalInputSpec, TalSource};
use crate::replay::archive::ReplayArchiveIndex;
use crate::replay::delta_archive::ReplayDeltaArchiveIndex;
use crate::replay::delta_fetch_http::PayloadDeltaReplayHttpFetcher;
@@ -78,6 +85,70 @@ pub struct RunTreeFromTalAuditOutput {
pub download_stats: crate::audit::AuditDownloadStats,
}
fn make_live_runner<'a>(
store: &'a crate::storage::RocksStore,
policy: &'a crate::policy::Policy,
http_fetcher: &'a dyn Fetcher,
rsync_fetcher: &'a dyn crate::fetch::rsync::RsyncFetcher,
validation_time: time::OffsetDateTime,
timing: Option<TimingHandle>,
download_log: Option<DownloadLogHandle>,
repo_sync_runtime: Option<Arc<dyn RepoSyncRuntime>>,
) -> Rpkiv1PublicationPointRunner<'a> {
Rpkiv1PublicationPointRunner {
store,
policy,
http_fetcher,
rsync_fetcher,
validation_time,
timing,
download_log,
replay_archive_index: None,
replay_delta_index: None,
rrdp_dedup: true,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime,
}
}
fn build_phase1_repo_sync_runtime<H, R>(
store: Arc<crate::storage::RocksStore>,
policy: &crate::policy::Policy,
http_fetcher: &H,
rsync_fetcher: &R,
parallel_config: ParallelPhase1Config,
timing: Option<TimingHandle>,
download_log: Option<DownloadLogHandle>,
tal_inputs: Vec<crate::parallel::types::TalInputSpec>,
) -> Result<Arc<dyn RepoSyncRuntime>, RunTreeFromTalError>
where
H: Fetcher + Clone + 'static,
R: crate::fetch::rsync::RsyncFetcher + Clone + 'static,
{
let coordinator = GlobalRunCoordinator::new(parallel_config.clone(), tal_inputs);
let rsync_fetcher_arc = Arc::new(rsync_fetcher.clone());
let executor = LiveRepoTransportExecutor::new(
Arc::clone(&store),
Arc::new(http_fetcher.clone()),
Arc::clone(&rsync_fetcher_arc),
timing,
download_log,
);
let pool = RepoTransportWorkerPool::new(RepoWorkerPoolConfig::from(&parallel_config), executor)
.map_err(RunTreeFromTalError::Replay)?;
let resolver: Arc<dyn Fn(&str) -> String + Send + Sync> =
Arc::new(move |base: &str| rsync_fetcher_arc.dedup_key(base));
let _ = policy; // policy reserved for later runtime-level decisions
Ok(Arc::new(Phase1RepoSyncRuntime::new(
coordinator,
pool,
resolver,
policy.sync_preference,
)))
}
#[derive(Debug, thiserror::Error)]
pub enum RunTreeFromTalError {
#[error("{0}")]
@@ -123,21 +194,16 @@ pub fn run_tree_from_tal_url_serial(
) -> Result<RunTreeFromTalOutput, RunTreeFromTalError> {
let discovery = discover_root_ca_instance_from_tal_url(http_fetcher, tal_url)?;
let runner = Rpkiv1PublicationPointRunner {
let runner = make_live_runner(
store,
policy,
http_fetcher,
rsync_fetcher,
validation_time,
timing: None,
download_log: None,
replay_archive_index: None,
replay_delta_index: None,
rrdp_dedup: true,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
};
None,
None,
None,
);
let root = root_handle_from_trust_anchor(
&discovery.trust_anchor,
@@ -162,21 +228,16 @@ pub fn run_tree_from_tal_url_serial_audit(
let discovery = discover_root_ca_instance_from_tal_url(http_fetcher, tal_url)?;
let download_log = DownloadLogHandle::new();
let runner = Rpkiv1PublicationPointRunner {
let runner = make_live_runner(
store,
policy,
http_fetcher,
rsync_fetcher,
validation_time,
timing: None,
download_log: Some(download_log.clone()),
replay_archive_index: None,
replay_delta_index: None,
rrdp_dedup: true,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
};
None,
Some(download_log.clone()),
None,
);
let root = root_handle_from_trust_anchor(
&discovery.trust_anchor,
@@ -215,21 +276,16 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing(
drop(_tal);
let download_log = DownloadLogHandle::new();
let runner = Rpkiv1PublicationPointRunner {
let runner = make_live_runner(
store,
policy,
http_fetcher,
rsync_fetcher,
validation_time,
timing: Some(timing.clone()),
download_log: Some(download_log.clone()),
replay_archive_index: None,
replay_delta_index: None,
rrdp_dedup: true,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
};
Some(timing.clone()),
Some(download_log.clone()),
None,
);
let root = root_handle_from_trust_anchor(
&discovery.trust_anchor,
@@ -254,6 +310,136 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing(
})
}
pub fn run_tree_from_tal_url_parallel_phase1_audit<H, R>(
store: Arc<crate::storage::RocksStore>,
policy: &crate::policy::Policy,
tal_url: &str,
http_fetcher: &H,
rsync_fetcher: &R,
validation_time: time::OffsetDateTime,
config: &TreeRunConfig,
parallel_config: ParallelPhase1Config,
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError>
where
H: Fetcher + Clone + 'static,
R: crate::fetch::rsync::RsyncFetcher + Clone + 'static,
{
let discovery = discover_root_ca_instance_from_tal_url(http_fetcher, tal_url)?;
let download_log = DownloadLogHandle::new();
let runtime = build_phase1_repo_sync_runtime(
Arc::clone(&store),
policy,
http_fetcher,
rsync_fetcher,
parallel_config,
None,
Some(download_log.clone()),
vec![TalInputSpec::from_url(tal_url.to_string())],
)?;
let runner = make_live_runner(
store.as_ref(),
policy,
http_fetcher,
rsync_fetcher,
validation_time,
None,
Some(download_log.clone()),
Some(runtime),
);
let root = root_handle_from_trust_anchor(
&discovery.trust_anchor,
derive_tal_id(&discovery),
None,
&discovery.ca_instance,
);
let TreeRunAuditOutput {
tree,
publication_points,
} = run_tree_serial_audit(root, &runner, config)?;
let downloads = download_log.snapshot_events();
let download_stats = DownloadLogHandle::stats_from_events(&downloads);
Ok(RunTreeFromTalAuditOutput {
discovery,
tree,
publication_points,
downloads,
download_stats,
})
}
pub fn run_tree_from_tal_and_ta_der_parallel_phase1_audit<H, R>(
store: Arc<crate::storage::RocksStore>,
policy: &crate::policy::Policy,
tal_bytes: &[u8],
ta_der: &[u8],
resolved_ta_uri: Option<&url::Url>,
http_fetcher: &H,
rsync_fetcher: &R,
validation_time: time::OffsetDateTime,
config: &TreeRunConfig,
parallel_config: ParallelPhase1Config,
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError>
where
H: Fetcher + Clone + 'static,
R: crate::fetch::rsync::RsyncFetcher + Clone + 'static,
{
let discovery = discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri)?;
let download_log = DownloadLogHandle::new();
let derived_tal_id = derive_tal_id(&discovery);
let tal_inputs = vec![TalInputSpec {
tal_id: derived_tal_id.clone(),
rir_id: derived_tal_id,
source: TalSource::DerBytes {
tal_url: discovery
.tal_url
.clone()
.unwrap_or_else(|| "embedded-tal".to_string()),
ta_der: ta_der.to_vec(),
},
}];
let runtime = build_phase1_repo_sync_runtime(
Arc::clone(&store),
policy,
http_fetcher,
rsync_fetcher,
parallel_config,
None,
Some(download_log.clone()),
tal_inputs,
)?;
let runner = make_live_runner(
store.as_ref(),
policy,
http_fetcher,
rsync_fetcher,
validation_time,
None,
Some(download_log.clone()),
Some(runtime),
);
let root = root_handle_from_trust_anchor(
&discovery.trust_anchor,
derive_tal_id(&discovery),
None,
&discovery.ca_instance,
);
let TreeRunAuditOutput {
tree,
publication_points,
} = run_tree_serial_audit(root, &runner, config)?;
let downloads = download_log.snapshot_events();
let download_stats = DownloadLogHandle::stats_from_events(&downloads);
Ok(RunTreeFromTalAuditOutput {
discovery,
tree,
publication_points,
downloads,
download_stats,
})
}
pub fn run_tree_from_tal_and_ta_der_serial(
store: &crate::storage::RocksStore,
policy: &crate::policy::Policy,
@@ -282,6 +468,7 @@ pub fn run_tree_from_tal_and_ta_der_serial(
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
let root = root_handle_from_trust_anchor(
@@ -328,6 +515,7 @@ pub fn run_tree_from_tal_bytes_serial_audit(
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
let root = root_handle_from_trust_anchor(
@@ -388,6 +576,7 @@ pub fn run_tree_from_tal_bytes_serial_audit_with_timing(
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
let root = root_handle_from_trust_anchor(
@@ -443,6 +632,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit(
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
let root = root_handle_from_trust_anchor(
@@ -499,6 +689,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing(
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
let root = root_handle_from_trust_anchor(
@@ -562,6 +753,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial(
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
let root = root_handle_from_trust_anchor(
@@ -614,6 +806,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit(
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
let root = root_handle_from_trust_anchor(
@@ -680,6 +873,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing(
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
let root = root_handle_from_trust_anchor(
@@ -729,6 +923,7 @@ fn build_payload_replay_runner<'a>(
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
}
}
@@ -756,6 +951,7 @@ fn build_payload_delta_replay_runner<'a>(
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
}
}
@@ -783,6 +979,7 @@ fn build_payload_delta_replay_current_store_runner<'a>(
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
}
}

119
src/validation/tree.rs
View File

@@ -95,6 +95,13 @@ pub trait PublicationPointRunner {
&self,
ca: &CaInstanceHandle,
) -> Result<PublicationPointRunResult, String>;
fn prefetch_discovered_children(
&self,
_children: &[DiscoveredChildCaInstance],
) -> Result<(), String> {
Ok(())
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
@@ -198,6 +205,12 @@ pub fn run_tree_serial_audit(
.cmp(&b.discovered_from.child_ca_certificate_rsync_uri)
})
});
if let Err(e) = runner.prefetch_discovered_children(&children) {
warnings.push(
Warning::new(format!("prefetch discovered children failed: {e}"))
.with_context(&ca.manifest_rsync_uri),
);
}
for child in children {
queue.push_back(QueuedCaInstance {
id: next_id,
@@ -221,3 +234,109 @@ pub fn run_tree_serial_audit(
publication_points,
})
}
#[cfg(test)]
mod tests {
use std::sync::{Arc, Mutex};
use crate::audit::{DiscoveredFrom, PublicationPointAudit};
use crate::validation::objects::{ObjectsOutput, ObjectsStats};
use super::{
CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult,
PublicationPointRunner, TreeRunConfig, run_tree_serial_audit,
};
fn sample_handle(manifest: &str) -> CaInstanceHandle {
CaInstanceHandle {
depth: 0,
tal_id: "arin".to_string(),
parent_manifest_rsync_uri: None,
ca_certificate_der: vec![1, 2, 3],
ca_certificate_rsync_uri: None,
effective_ip_resources: None,
effective_as_resources: None,
rsync_base_uri: "rsync://example.test/repo/".to_string(),
manifest_rsync_uri: manifest.to_string(),
publication_point_rsync_uri: "rsync://example.test/repo/".to_string(),
rrdp_notification_uri: None,
}
}
struct PrefetchRecordingRunner {
seen_prefetch_children: Arc<Mutex<Vec<String>>>,
}
impl PublicationPointRunner for PrefetchRecordingRunner {
fn prefetch_discovered_children(
&self,
children: &[DiscoveredChildCaInstance],
) -> Result<(), String> {
let mut seen = self
.seen_prefetch_children
.lock()
.expect("prefetch lock poisoned");
seen.extend(
children
.iter()
.map(|child| child.handle.manifest_rsync_uri.clone()),
);
Ok(())
}
fn run_publication_point(
&self,
ca: &CaInstanceHandle,
) -> Result<PublicationPointRunResult, String> {
let children = if ca.manifest_rsync_uri.ends_with("root.mft") {
vec![DiscoveredChildCaInstance {
handle: sample_handle("rsync://example.test/repo/child.mft"),
discovered_from: DiscoveredFrom {
parent_manifest_rsync_uri: ca.manifest_rsync_uri.clone(),
child_ca_certificate_rsync_uri: "rsync://example.test/repo/child.cer"
.to_string(),
child_ca_certificate_sha256_hex: "00".repeat(32),
},
}]
} else {
Vec::new()
};
Ok(PublicationPointRunResult {
source: crate::validation::manifest::PublicationPointSource::Fresh,
snapshot: None,
warnings: Vec::new(),
objects: ObjectsOutput {
vrps: Vec::new(),
aspas: Vec::new(),
router_keys: Vec::new(),
local_outputs_cache: Vec::new(),
warnings: Vec::new(),
stats: ObjectsStats::default(),
audit: Vec::new(),
},
audit: PublicationPointAudit::default(),
discovered_children: children,
})
}
}
#[test]
fn run_tree_serial_audit_invokes_prefetch_hook_for_discovered_children() {
let seen = Arc::new(Mutex::new(Vec::new()));
let runner = PrefetchRecordingRunner {
seen_prefetch_children: Arc::clone(&seen),
};
let out = run_tree_serial_audit(
sample_handle("rsync://example.test/repo/root.mft"),
&runner,
&TreeRunConfig::default(),
)
.expect("tree run");
assert_eq!(out.tree.instances_processed, 2);
assert_eq!(
seen.lock().expect("lock").as_slice(),
["rsync://example.test/repo/child.mft"]
);
}
}

33
src/validation/tree_runner.rs
View File

@@ -15,6 +15,7 @@ use crate::data_model::router_cert::{
};
use crate::fetch::rsync::RsyncFetcher;
use crate::policy::Policy;
use crate::parallel::repo_runtime::{RepoSyncRuntime, RepoSyncRuntimeOutcome};
use crate::replay::archive::ReplayArchiveIndex;
use crate::replay::delta_archive::ReplayDeltaArchiveIndex;
use crate::report::{RfcRef, Warning};
@@ -77,9 +78,20 @@ pub struct Rpkiv1PublicationPointRunner<'a> {
/// same `rsync_base_uri` (observed in APNIC full sync timing reports).
pub rsync_dedup: bool,
pub rsync_repo_cache: Mutex<HashMap<String, bool>>, // rsync_base_uri -> rsync_ok
pub repo_sync_runtime: Option<Arc<dyn RepoSyncRuntime>>,
}
impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
fn prefetch_discovered_children(
&self,
children: &[DiscoveredChildCaInstance],
) -> Result<(), String> {
if let Some(runtime) = self.repo_sync_runtime.as_ref() {
runtime.prefetch_discovered_children(children)?;
}
Ok(())
}
fn run_publication_point(
&self,
ca: &CaInstanceHandle,
@@ -174,7 +186,18 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
Option<String>,
Option<String>,
Option<String>,
) = if skip_sync_due_to_dedup {
) = if let Some(runtime) = self.repo_sync_runtime.as_ref() {
let RepoSyncRuntimeOutcome {
repo_sync_ok,
repo_sync_err,
repo_sync_source,
repo_sync_phase,
repo_sync_duration_ms: _,
warnings: repo_warnings,
} = runtime.sync_publication_point_repo(ca)?;
warnings.extend(repo_warnings);
(repo_sync_ok, repo_sync_err, repo_sync_source, repo_sync_phase)
} else if skip_sync_due_to_dedup {
let source = if effective_notification_uri.is_some() {
Some("rrdp_dedup_skip".to_string())
} else {
@@ -4023,6 +4046,7 @@ authorityKeyIdentifier = keyid:always
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: false,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
// For this fixture-driven smoke, we provide the correct issuer CA certificate (the CA for
@@ -4191,6 +4215,7 @@ authorityKeyIdentifier = keyid:always
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
let first = runner.run_publication_point(&handle).expect("first run ok");
@@ -4298,6 +4323,7 @@ authorityKeyIdentifier = keyid:always
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
let first = runner.run_publication_point(&handle).expect("first run ok");
@@ -4408,6 +4434,7 @@ authorityKeyIdentifier = keyid:always
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
let first = runner.run_publication_point(&handle).expect("first run ok");
@@ -4490,6 +4517,7 @@ authorityKeyIdentifier = keyid:always
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: false,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
let first = ok_runner
.run_publication_point(&handle)
@@ -4515,6 +4543,7 @@ authorityKeyIdentifier = keyid:always
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: false,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
let second = bad_runner
.run_publication_point(&handle)
@@ -5699,6 +5728,7 @@ authorityKeyIdentifier = keyid:always
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: false,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
let first = runner_rrdp
.run_publication_point(&handle)
@@ -5727,6 +5757,7 @@ authorityKeyIdentifier = keyid:always
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
repo_sync_runtime: None,
};
let third = runner_rsync
.run_publication_point(&handle)