NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260421 优化内存布局，降低内存峰值需求压低到4.5GB左右，在4c8g 机器上，稳定跑snapshot + delta

Browse Source
This commit is contained in:
yuyr
2026-04-21 15:21:25 +08:00
parent 542bd7be80
commit 944ea6ca00
7 changed files with 373 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/cli.rs
View File

@@ -1143,6 +1143,7 @@ pub fn run(argv: &[String]) -> Result<(), String> {
let total_started = std::time::Instant::now();
let validation_started = std::time::Instant::now();
let collect_current_repo_objects = args.cir_enabled;
let out = if delta_replay_mode {
let tal_path = args
.tal_path
@@ -1279,6 +1280,7 @@ pub fn run(argv: &[String]) -> Result<(), String> {
args.parallel_phase2_config
.clone()
.expect("phase2 config present"),
collect_current_repo_objects,
)
.map_err(|e| e.to_string())?
} else {
@@ -1293,6 +1295,7 @@ pub fn run(argv: &[String]) -> Result<(), String> {
args.parallel_phase1_config
.clone()
.expect("phase1 config present"),
collect_current_repo_objects,
)
.map_err(|e| e.to_string())?
}
@@ -1319,6 +1322,7 @@ pub fn run(argv: &[String]) -> Result<(), String> {
args.parallel_phase2_config
.clone()
.expect("phase2 config present"),
collect_current_repo_objects,
)
.map_err(|e| e.to_string())?
} else {
@@ -1333,6 +1337,7 @@ pub fn run(argv: &[String]) -> Result<(), String> {
args.parallel_phase1_config
.clone()
.expect("phase1 config present"),
collect_current_repo_objects,
)
.map_err(|e| e.to_string())?
}
@@ -1384,6 +1389,7 @@ pub fn run(argv: &[String]) -> Result<(), String> {
args.parallel_phase2_config
.clone()
.expect("phase2 config present"),
collect_current_repo_objects,
)
.map_err(|e| e.to_string())?
} else {
@@ -1400,6 +1406,7 @@ pub fn run(argv: &[String]) -> Result<(), String> {
args.parallel_phase1_config
.clone()
.expect("phase1 config present"),
collect_current_repo_objects,
)
.map_err(|e| e.to_string())?
}
@@ -1497,6 +1504,7 @@ pub fn run(argv: &[String]) -> Result<(), String> {
args.parallel_phase2_config
.clone()
.expect("phase2 config present"),
collect_current_repo_objects,
)
.map_err(|e| e.to_string())?
} else {
@@ -1511,6 +1519,7 @@ pub fn run(argv: &[String]) -> Result<(), String> {
args.parallel_phase1_config
.clone()
.expect("phase1 config present"),
collect_current_repo_objects,
)
.map_err(|e| e.to_string())?
}
@@ -1537,6 +1546,7 @@ pub fn run(argv: &[String]) -> Result<(), String> {
args.parallel_phase2_config
.clone()
.expect("phase2 config present"),
collect_current_repo_objects,
)
.map_err(|e| e.to_string())?
} else {
@@ -1551,6 +1561,7 @@ pub fn run(argv: &[String]) -> Result<(), String> {
args.parallel_phase1_config
.clone()
.expect("phase1 config present"),
collect_current_repo_objects,
)
.map_err(|e| e.to_string())?
}
@@ -1602,6 +1613,7 @@ pub fn run(argv: &[String]) -> Result<(), String> {
args.parallel_phase2_config
.clone()
.expect("phase2 config present"),
collect_current_repo_objects,
)
.map_err(|e| e.to_string())?
} else {
@@ -1618,6 +1630,7 @@ pub fn run(argv: &[String]) -> Result<(), String> {
args.parallel_phase1_config
.clone()
.expect("phase1 config present"),
collect_current_repo_objects,
)
.map_err(|e| e.to_string())?
}

51
src/current_repo_index.rs
View File

@@ -23,7 +23,6 @@ pub struct CurrentRepoObject {
#[derive(Default, Debug)]
pub struct CurrentRepoIndex {
by_uri: HashMap<String, CurrentRepoEntry>,
by_scope: HashMap<String, HashSet<String>>,
}
pub type CurrentRepoIndexHandle = Arc<Mutex<CurrentRepoIndex>>;
@@ -43,10 +42,12 @@ impl CurrentRepoIndex {
pub fn list_scope_uris(&self, repository_source: &str) -> Vec<String> {
let mut out = self
.by_scope
.get(repository_source)
.map(|set| set.iter().cloned().collect::<Vec<_>>())
.unwrap_or_default();
.by_uri
.iter()
.filter_map(|(rsync_uri, entry)| {
(entry.repository_source == repository_source).then(|| rsync_uri.clone())
})
.collect::<Vec<_>>();
out.sort();
out
}
@@ -56,7 +57,11 @@ impl CurrentRepoIndex {
}
pub fn scope_count(&self) -> usize {
self.by_scope.len()
self.by_uri
.values()
.map(|entry| entry.repository_source.as_str())
.collect::<HashSet<_>>()
.len()
}
pub fn snapshot_objects(&self) -> Vec<CurrentRepoObject> {
@@ -76,7 +81,6 @@ impl CurrentRepoIndex {
pub fn clear(&mut self) {
self.by_uri.clear();
self.by_scope.clear();
}
pub fn apply_repository_view_entries(
@@ -92,11 +96,6 @@ impl CurrentRepoIndex {
fn apply_repository_view_entry(&mut self, entry: &RepositoryViewEntry) -> Result<(), String> {
entry.validate_internal().map_err(|e| e.to_string())?;
let old_scope = self
.by_uri
.get(&entry.rsync_uri)
.map(|existing| existing.repository_source.clone());
match entry.state {
RepositoryViewState::Present | RepositoryViewState::Replaced => {
let repository_source = entry.repository_source.clone().ok_or_else(|| {
@@ -112,17 +111,6 @@ impl CurrentRepoIndex {
)
})?;
let current_hash = decode_sha256_hex_32(&current_hash_hex)?;
if let Some(old_scope) = old_scope.as_ref() {
if old_scope != &repository_source {
self.remove_uri_from_scope(old_scope, &entry.rsync_uri);
}
}
self.by_scope
.entry(repository_source.clone())
.or_default()
.insert(entry.rsync_uri.clone());
self.by_uri.insert(
entry.rsync_uri.clone(),
CurrentRepoEntry {
@@ -135,27 +123,12 @@ impl CurrentRepoIndex {
);
}
RepositoryViewState::Withdrawn => {
if let Some(scope) = entry.repository_source.as_ref().or(old_scope.as_ref()) {
self.remove_uri_from_scope(scope, &entry.rsync_uri);
}
self.by_uri.remove(&entry.rsync_uri);
}
}
Ok(())
}
fn remove_uri_from_scope(&mut self, scope: &str, rsync_uri: &str) {
let empty = if let Some(entries) = self.by_scope.get_mut(scope) {
entries.remove(rsync_uri);
entries.is_empty()
} else {
false
};
if empty {
self.by_scope.remove(scope);
}
}
}
fn decode_sha256_hex_32(value: &str) -> Result<[u8; 32], String> {
@@ -281,7 +254,7 @@ mod tests {
.expect_err("invalid hash should fail");
assert!(err.contains("invalid"), "{err}");
let err = index
index
.apply_repository_view_entries(&[RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/b.roa".to_string(),
current_hash: Some("22".repeat(32)),

55
src/validation/run_tree_from_tal.rs
View File

@@ -103,7 +103,11 @@ pub struct TalRootDiscovery {
fn snapshot_current_repo_objects(
current_repo_index: Option<&CurrentRepoIndexHandle>,
collect: bool,
) -> Vec<CurrentRepoObject> {
if !collect {
return Vec::new();
}
current_repo_index
.and_then(|handle| handle.lock().ok().map(|idx| idx.snapshot_objects()))
.unwrap_or_default()
@@ -426,6 +430,7 @@ fn run_single_root_parallel_audit_inner<H, R>(
config: &TreeRunConfig,
parallel_config: ParallelPhase1Config,
phase2_config: Option<ParallelPhase2Config>,
collect_current_repo_objects: bool,
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError>
where
H: Fetcher + Clone + 'static,
@@ -481,7 +486,10 @@ where
publication_points,
downloads,
download_stats,
current_repo_objects: snapshot_current_repo_objects(Some(&current_repo_index_for_output)),
current_repo_objects: snapshot_current_repo_objects(
Some(&current_repo_index_for_output),
collect_current_repo_objects,
),
ccr_accumulator: phase2_enabled
.then(|| runner.ccr_accumulator_snapshot())
.flatten(),
@@ -498,6 +506,7 @@ fn run_multi_root_parallel_audit_inner<H, R>(
config: &TreeRunConfig,
parallel_config: ParallelPhase1Config,
phase2_config: Option<ParallelPhase2Config>,
collect_current_repo_objects: bool,
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError>
where
H: Fetcher + Clone + 'static,
@@ -572,7 +581,10 @@ where
publication_points,
downloads,
download_stats,
current_repo_objects: snapshot_current_repo_objects(Some(&current_repo_index_for_output)),
current_repo_objects: snapshot_current_repo_objects(
Some(&current_repo_index_for_output),
collect_current_repo_objects,
),
ccr_accumulator: phase2_enabled
.then(|| runner.ccr_accumulator_snapshot())
.flatten(),
@@ -588,6 +600,7 @@ pub fn run_tree_from_tal_url_parallel_phase1_audit<H, R>(
validation_time: time::OffsetDateTime,
config: &TreeRunConfig,
parallel_config: ParallelPhase1Config,
collect_current_repo_objects: bool,
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError>
where
H: Fetcher + Clone + 'static,
@@ -605,6 +618,7 @@ where
config,
parallel_config,
None,
collect_current_repo_objects,
)
}
@@ -619,6 +633,7 @@ pub fn run_tree_from_tal_and_ta_der_parallel_phase1_audit<H, R>(
validation_time: time::OffsetDateTime,
config: &TreeRunConfig,
parallel_config: ParallelPhase1Config,
collect_current_repo_objects: bool,
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError>
where
H: Fetcher + Clone + 'static,
@@ -650,6 +665,7 @@ where
config,
parallel_config,
None,
collect_current_repo_objects,
)
}
@@ -662,6 +678,7 @@ pub fn run_tree_from_multiple_tals_parallel_phase1_audit<H, R>(
validation_time: time::OffsetDateTime,
config: &TreeRunConfig,
parallel_config: ParallelPhase1Config,
collect_current_repo_objects: bool,
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError>
where
H: Fetcher + Clone + 'static,
@@ -677,6 +694,7 @@ where
config,
parallel_config,
None,
collect_current_repo_objects,
)
}
@@ -690,6 +708,7 @@ pub fn run_tree_from_tal_url_parallel_phase2_audit<H, R>(
config: &TreeRunConfig,
parallel_config: ParallelPhase1Config,
phase2_config: ParallelPhase2Config,
collect_current_repo_objects: bool,
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError>
where
H: Fetcher + Clone + 'static,
@@ -707,6 +726,7 @@ where
config,
parallel_config,
Some(phase2_config),
collect_current_repo_objects,
)
}
@@ -722,6 +742,7 @@ pub fn run_tree_from_tal_and_ta_der_parallel_phase2_audit<H, R>(
config: &TreeRunConfig,
parallel_config: ParallelPhase1Config,
phase2_config: ParallelPhase2Config,
collect_current_repo_objects: bool,
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError>
where
H: Fetcher + Clone + 'static,
@@ -753,6 +774,7 @@ where
config,
parallel_config,
Some(phase2_config),
collect_current_repo_objects,
)
}
@@ -766,6 +788,7 @@ pub fn run_tree_from_multiple_tals_parallel_phase2_audit<H, R>(
config: &TreeRunConfig,
parallel_config: ParallelPhase1Config,
phase2_config: ParallelPhase2Config,
collect_current_repo_objects: bool,
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError>
where
H: Fetcher + Clone + 'static,
@@ -781,6 +804,7 @@ where
config,
parallel_config,
Some(phase2_config),
collect_current_repo_objects,
)
}
@@ -1681,6 +1705,8 @@ pub fn run_tree_from_tal_and_ta_der_payload_delta_replay_step_serial_audit(
#[cfg(test)]
mod multi_tal_tests {
use super::*;
use crate::current_repo_index::CurrentRepoIndex;
use crate::storage::{RepositoryViewEntry, RepositoryViewState};
struct RejectingHttpFetcher;
@@ -1707,6 +1733,31 @@ mod multi_tal_tests {
}
}
#[test]
fn snapshot_current_repo_objects_is_on_demand() {
let handle = CurrentRepoIndex::shared();
handle
.lock()
.expect("lock index")
.apply_repository_view_entries(&[RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
current_hash: Some("11".repeat(32)),
repository_source: Some("rsync://example.test/repo/".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
}])
.expect("apply present entry");
assert!(
snapshot_current_repo_objects(Some(&handle), false).is_empty(),
"collection should be skipped when disabled"
);
let collected = snapshot_current_repo_objects(Some(&handle), true);
assert_eq!(collected.len(), 1);
assert_eq!(collected[0].rsync_uri, "rsync://example.test/repo/a.roa");
}
#[test]
fn discover_multiple_roots_from_tal_inputs_builds_multiple_root_handles() {
let apnic_tal =

89
src/validation/tree_parallel.rs
View File

@@ -49,6 +49,21 @@ struct FinishedPublicationPoint {
result: Result<PublicationPointRunResult, String>,
}
fn compact_phase2_finished_result(
mut result: PublicationPointRunResult,
) -> PublicationPointRunResult {
// Phase2 only needs warnings, objects, audit, and traversal metadata after finalize.
// Dropping the snapshot here avoids retaining manifest/files/raw-byte caches until run end.
result.snapshot = None;
result
}
fn compact_phase2_finished_result_result(
result: Result<PublicationPointRunResult, String>,
) -> Result<PublicationPointRunResult, String> {
result.map(compact_phase2_finished_result)
}
pub fn run_tree_parallel_phase2_audit_multi_root(
roots: Vec<CaInstanceHandle>,
runner: &Rpkiv1PublicationPointRunner<'_>,
@@ -214,7 +229,7 @@ fn stage_ready_publication_point(
}
finished.push(FinishedPublicationPoint {
node: ready.node,
result: fallback,
result: compact_phase2_finished_result_result(fallback),
});
return;
}
@@ -367,7 +382,10 @@ fn finalize_ready_objects(
repo_outcome.repo_sync_err.as_deref(),
)
.map(|out| out.result);
finished.push(FinishedPublicationPoint { node, result });
finished.push(FinishedPublicationPoint {
node,
result: compact_phase2_finished_result_result(result),
});
}
fn flush_pending_roa_dispatch(
@@ -461,7 +479,7 @@ fn drain_object_results(
);
finished.push(FinishedPublicationPoint {
node: state.node,
result,
result: compact_phase2_finished_result_result(result),
});
}
Err(err) => finished.push(FinishedPublicationPoint {
@@ -596,3 +614,68 @@ pub fn run_tree_parallel_phase2_audit(
) -> Result<TreeRunAuditOutput, TreeRunError> {
run_tree_parallel_phase2_audit_multi_root(vec![root], runner, config)
}
#[cfg(test)]
mod tests {
use super::{compact_phase2_finished_result, compact_phase2_finished_result_result};
use crate::audit::PublicationPointAudit;
use crate::storage::PackTime;
use crate::validation::manifest::PublicationPointSource;
use crate::validation::objects::{ObjectsOutput, ObjectsStats};
use crate::validation::publication_point::PublicationPointSnapshot;
use crate::validation::tree::PublicationPointRunResult;
fn sample_snapshot() -> PublicationPointSnapshot {
PublicationPointSnapshot {
format_version: PublicationPointSnapshot::FORMAT_VERSION_V1,
manifest_rsync_uri: "rsync://example.test/repo/example.mft".to_string(),
publication_point_rsync_uri: "rsync://example.test/repo/".to_string(),
manifest_number_be: vec![1],
this_update: PackTime {
rfc3339_utc: "2026-04-21T00:00:00Z".to_string(),
},
next_update: PackTime {
rfc3339_utc: "2026-04-22T00:00:00Z".to_string(),
},
verified_at: PackTime {
rfc3339_utc: "2026-04-21T00:00:01Z".to_string(),
},
manifest_bytes: vec![1, 2, 3],
files: Vec::new(),
}
}
fn sample_result() -> PublicationPointRunResult {
PublicationPointRunResult {
source: PublicationPointSource::Fresh,
snapshot: Some(sample_snapshot()),
warnings: Vec::new(),
objects: ObjectsOutput {
vrps: Vec::new(),
aspas: Vec::new(),
router_keys: Vec::new(),
local_outputs_cache: Vec::new(),
warnings: Vec::new(),
stats: ObjectsStats::default(),
audit: Vec::new(),
},
audit: PublicationPointAudit::default(),
discovered_children: Vec::new(),
}
}
#[test]
fn compact_phase2_finished_result_drops_snapshot() {
let result = compact_phase2_finished_result(sample_result());
assert!(result.snapshot.is_none());
assert_eq!(result.source, PublicationPointSource::Fresh);
assert!(result.discovered_children.is_empty());
}
#[test]
fn compact_phase2_finished_result_result_preserves_err() {
let err = compact_phase2_finished_result_result(Err("boom".to_string()))
.expect_err("error should be preserved");
assert_eq!(err, "boom");
}
}

115
src/validation/tree_runner.rs
View File

@@ -252,7 +252,7 @@ impl<'a> Rpkiv1PublicationPointRunner<'a> {
ca: &CaInstanceHandle,
fresh_point: &FreshValidatedPublicationPoint,
warnings: Vec<Warning>,
objects: crate::validation::objects::ObjectsOutput,
mut objects: crate::validation::objects::ObjectsOutput,
child_audits: Vec<ObjectAuditEntry>,
discovered_children: Vec<DiscoveredChildCaInstance>,
repo_sync_source: Option<&str>,
@@ -278,6 +278,10 @@ impl<'a> Rpkiv1PublicationPointRunner<'a> {
.map_err(|e| format!("persist VCIR failed: {e}"))?;
let persist_vcir_ms = persist_vcir_started.elapsed().as_millis() as u64;
// local_outputs_cache only exists to build/persist VCIR. Release it before the
// publication point result is retained for the rest of the run.
let _released_local_outputs = std::mem::take(&mut objects.local_outputs_cache);
if self.ccr_accumulator.is_some() {
let child_entries =
build_vcir_child_entries(&discovered_children, self.validation_time)?;
@@ -4101,6 +4105,115 @@ authorityKeyIdentifier = keyid:always
.all(|output| output.output_type == VcirOutputType::Vrp));
}
#[test]
fn finalize_fresh_publication_point_releases_local_outputs_cache_after_persist() {
let (pack, issuer_ca_der, validation_time) =
cernet_publication_point_snapshot_for_vcir_tests();
let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca");
let mut objects = crate::validation::objects::process_publication_point_snapshot_for_issuer(
&pack,
&Policy::default(),
issuer_ca_der.as_slice(),
Some(
"rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer",
),
issuer_ca.tbs.extensions.ip_resources.as_ref(),
issuer_ca.tbs.extensions.as_resources.as_ref(),
validation_time,
None,
);
assert!(
!objects.local_outputs_cache.is_empty(),
"expected local outputs from signed objects"
);
let store_dir = tempfile::tempdir().expect("store dir");
let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
let policy = Policy::default();
let runner = Rpkiv1PublicationPointRunner {
store: &store,
policy: &policy,
http_fetcher: &NeverHttpFetcher,
rsync_fetcher: &FailingRsyncFetcher,
validation_time,
timing: None,
download_log: None,
replay_archive_index: None,
replay_delta_index: None,
rrdp_dedup: false,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: false,
rsync_repo_cache: Mutex::new(HashMap::new()),
current_repo_index: None,
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let ca = CaInstanceHandle {
depth: 0,
tal_id: "test-tal".to_string(),
parent_manifest_rsync_uri: None,
ca_certificate_der: issuer_ca_der.clone(),
ca_certificate_rsync_uri: Some(
"rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(),
),
effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(),
effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(),
rsync_base_uri: pack.publication_point_rsync_uri.clone(),
manifest_rsync_uri: pack.manifest_rsync_uri.clone(),
publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(),
rrdp_notification_uri: None,
};
let fresh_point = FreshValidatedPublicationPoint {
manifest_rsync_uri: pack.manifest_rsync_uri.clone(),
publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(),
manifest_number_be: pack.manifest_number_be.clone(),
this_update: pack.this_update.clone(),
next_update: pack.next_update.clone(),
verified_at: pack.verified_at.clone(),
manifest_bytes: pack.manifest_bytes.clone(),
files: pack.files.clone(),
};
objects.local_outputs_cache.shrink_to_fit();
let original_cache_capacity = objects.local_outputs_cache.capacity();
let finalized = runner
.finalize_fresh_publication_point_from_reducer(
&ca,
&fresh_point,
Vec::new(),
objects,
Vec::new(),
Vec::new(),
None,
None,
0,
None,
)
.expect("finalize fresh publication point");
assert!(
finalized.result.objects.local_outputs_cache.is_empty(),
"local outputs cache should be released after VCIR persistence"
);
assert_eq!(
finalized.result.objects.local_outputs_cache.capacity(),
0,
"released cache should not keep its backing allocation"
);
assert!(original_cache_capacity > 0);
let persisted = store
.get_vcir(&pack.manifest_rsync_uri)
.expect("load persisted vcir")
.expect("persisted vcir");
assert!(
!persisted.local_outputs.is_empty(),
"VCIR should still persist local outputs before cache release"
);
}
#[test]
fn persist_vcir_for_fresh_result_stores_vcir_and_audit_indexes_for_real_snapshot() {
let (pack, issuer_ca_der, validation_time) =