NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260416_2 并行优化phase1后进行snapshot fast path优化，通过四方面关键优化技术消除了验证主链路上snapshot构建是的db访问性能热点问题，性能热点转移到ROA验证处理本身，目前APNIC+ARIN全量同步从500秒压缩到212秒，离rpkclient 112秒差距变小

Browse Source
This commit is contained in:
yuyr
2026-04-17 14:58:47 +08:00
parent 38421b1ae7
commit 224ae10052
36 changed files with 2935 additions and 937 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
tests/bench_stage2_decode_profile_selected_der_v2.rs
View File

@@ -297,6 +297,13 @@ fn decode_validate(obj_type: ObjType, bytes: &[u8]) {
}
fn landing_packfile_cbor_put(store: &RocksStore, obj_type: ObjType, sample: &str, bytes: &[u8]) {
#[derive(serde::Serialize)]
struct BenchPackFile<'a> {
rsync_uri: &'a str,
bytes: &'a [u8],
sha256: [u8; 32],
}
let rsync_uri = format!(
"rsync://bench.invalid/{}/{}.{}",
obj_type.as_str(),
@@ -304,7 +311,13 @@ fn landing_packfile_cbor_put(store: &RocksStore, obj_type: ObjType, sample: &str
obj_type.ext()
);
let pf = PackFile::from_bytes_compute_sha256(rsync_uri, bytes.to_vec());
let encoded = serde_cbor::to_vec(std::hint::black_box(&pf)).expect("cbor encode packfile");
let bench_pf = BenchPackFile {
rsync_uri: &pf.rsync_uri,
bytes: pf.bytes().expect("load packfile bytes"),
sha256: pf.sha256,
};
let encoded =
serde_cbor::to_vec(std::hint::black_box(&bench_pf)).expect("cbor encode packfile");
let key = format!("bench:packfile:{}:{}", obj_type.as_str(), sample);
let sha256_hex = hex::encode(sha2::Sha256::digest(&encoded));
let mut entry = RawByHashEntry::from_bytes(sha256_hex, encoded);

20
tests/test_apnic_rrdp_delta_live_20260226.rs
View File

@@ -1,6 +1,6 @@
use std::cell::RefCell;
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use rpki::fetch::http::{BlockingHttpFetcher, HttpFetcherConfig};
@@ -52,7 +52,7 @@ impl RsyncFetcher for AlwaysFailRsyncFetcher {
struct CountingDenyUriFetcher {
inner: BlockingHttpFetcher,
deny_uri: String,
counts: std::rc::Rc<RefCell<HashMap<String, u64>>>,
counts: Arc<Mutex<HashMap<String, u64>>>,
}
impl CountingDenyUriFetcher {
@@ -60,18 +60,28 @@ impl CountingDenyUriFetcher {
Self {
inner,
deny_uri,
counts: std::rc::Rc::new(RefCell::new(HashMap::new())),
counts: Arc::new(Mutex::new(HashMap::new())),
}
}
fn count(&self, uri: &str) -> u64 {
*self.counts.borrow().get(uri).unwrap_or(&0)
*self
.counts
.lock()
.expect("lock counts")
.get(uri)
.unwrap_or(&0)
}
}
impl Fetcher for CountingDenyUriFetcher {
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
*self.counts.borrow_mut().entry(uri.to_string()).or_insert(0) += 1;
*self
.counts
.lock()
.expect("lock counts")
.entry(uri.to_string())
.or_insert(0) += 1;
if uri == self.deny_uri {
return Err(format!("snapshot fetch denied: {uri}"));
}

4
tests/test_apnic_stats_live_stage2.rs
View File

@@ -88,7 +88,7 @@ impl LiveStats {
if f.rsync_uri.ends_with(".crl") {
self.crl_total += 1;
if RpkixCrl::decode_der(&f.bytes).is_ok() {
if f.bytes().ok().and_then(|b| RpkixCrl::decode_der(b).ok()).is_some() {
self.crl_decode_ok += 1;
}
}
@@ -176,6 +176,8 @@ fn apnic_tree_full_stats_serial() {
rrdp_repo_cache: std::sync::Mutex::new(std::collections::HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: std::sync::Mutex::new(std::collections::HashMap::new()),
current_repo_index: None,
repo_sync_runtime: None,
};
let stats = RefCell::new(LiveStats::default());

19
tests/test_cli_run_offline_m18.rs
View File

@@ -88,8 +88,6 @@ fn cli_run_offline_mode_writes_cir_and_static_pool() {
let repo_dir = tempfile::tempdir().expect("repo tempdir");
let out_dir = tempfile::tempdir().expect("out tempdir");
let cir_path = out_dir.path().join("result.cir");
let static_root = out_dir.path().join("static");
let policy_path = out_dir.path().join("policy.toml");
std::fs::write(&policy_path, "sync_preference = \"rsync_only\"\n").expect("write policy");
@@ -117,8 +115,6 @@ fn cli_run_offline_mode_writes_cir_and_static_pool() {
"--cir-enable".to_string(),
"--cir-out".to_string(),
cir_path.to_string_lossy().to_string(),
"--cir-static-root".to_string(),
static_root.to_string_lossy().to_string(),
"--cir-tal-uri".to_string(),
"https://example.test/root.tal".to_string(),
];
@@ -134,21 +130,6 @@ fn cli_run_offline_mode_writes_cir_and_static_pool() {
.iter()
.any(|item| item.rsync_uri.contains("apnic-rpki-root-iana-origin.cer"))
);
let mut file_count = 0usize;
let mut stack = vec![static_root.clone()];
while let Some(path) = stack.pop() {
for entry in std::fs::read_dir(path).expect("read_dir") {
let entry = entry.expect("entry");
let path = entry.path();
if path.is_dir() {
stack.push(path);
} else {
file_count += 1;
}
}
}
assert!(file_count >= 1);
}
#[test]

21
tests/test_multi_rir_case_info.rs
View File

@@ -19,11 +19,10 @@ fn wrapper_script() -> std::path::PathBuf {
#[test]
fn multi_rir_case_info_resolves_all_five_rirs_and_timings() {
let bundle_root = multi_rir_bundle_root();
assert!(
bundle_root.is_dir(),
"bundle root missing: {}",
bundle_root.display()
);
if !bundle_root.is_dir() {
eprintln!("skipping multi-rir case info test; bundle root missing: {}", bundle_root.display());
return;
}
let expected = [
("afrinic", "afrinic", "afrinic.tal", "afrinic-ta.cer"),
@@ -210,11 +209,13 @@ stderr={}",
#[test]
fn multi_rir_wrapper_describe_mode_works_for_ripe() {
let bundle_root = multi_rir_bundle_root();
assert!(
bundle_root.is_dir(),
"bundle root missing: {}",
bundle_root.display()
);
if !bundle_root.is_dir() {
eprintln!(
"skipping multi-rir wrapper describe test; bundle root missing: {}",
bundle_root.display()
);
return;
}
let out = Command::new(wrapper_script())
.env("BUNDLE_ROOT", &bundle_root)

17
tests/test_multi_tal_parallel_m2.rs
View File

@@ -28,7 +28,9 @@ fn run_case(
"--parallel-phase1".to_string(),
"--disable-rrdp".to_string(),
"--rsync-local-dir".to_string(),
fixture("tests/fixtures/repository").to_string_lossy().to_string(),
fixture("tests/fixtures/repository")
.to_string_lossy()
.to_string(),
"--validation-time".to_string(),
"2026-04-07T00:00:00Z".to_string(),
"--max-depth".to_string(),
@@ -67,14 +69,19 @@ fn multi_tal_parallel_output_matches_union_of_single_tal_outputs() {
let arin_tal = fixture("tests/fixtures/tal/arin.tal");
let arin_ta = fixture("tests/fixtures/ta/arin-ta.cer");
let (apnic_report, apnic_vrps, apnic_vaps, _) =
run_case("apnic", &[(&apnic_tal, &apnic_ta)]);
let (apnic_report, apnic_vrps, apnic_vaps, _) = run_case("apnic", &[(&apnic_tal, &apnic_ta)]);
let (arin_report, arin_vrps, arin_vaps, _) = run_case("arin", &[(&arin_tal, &arin_ta)]);
let (multi_report, multi_vrps, multi_vaps, multi_ccr) =
run_case("multi", &[(&apnic_tal, &apnic_ta), (&arin_tal, &arin_ta)]);
let expected_vrps = apnic_vrps.union(&arin_vrps).cloned().collect::<BTreeSet<_>>();
let expected_vaps = apnic_vaps.union(&arin_vaps).cloned().collect::<BTreeSet<_>>();
let expected_vrps = apnic_vrps
.union(&arin_vrps)
.cloned()
.collect::<BTreeSet<_>>();
let expected_vaps = apnic_vaps
.union(&arin_vaps)
.cloned()
.collect::<BTreeSet<_>>();
assert_eq!(multi_vrps, expected_vrps);
assert_eq!(multi_vaps, expected_vaps);

2
tests/test_objects_errors_more.rs
View File

@@ -113,7 +113,7 @@ fn build_cernet_pack_and_validation_time() -> (
.iter()
.find(|f| f.rsync_uri.ends_with(".crl"))
.expect("crl present in snapshot");
let crl = RpkixCrl::decode_der(&crl_file.bytes).expect("decode crl");
let crl = RpkixCrl::decode_der(crl_file.bytes().expect("load crl bytes")).expect("decode crl");
let mut t = manifest.manifest.this_update;
if issuer_ca.tbs.validity_not_before > t {

10
tests/test_objects_policy_m8.rs
View File

@@ -113,7 +113,7 @@ fn build_cernet_pack_and_validation_time() -> (
.iter()
.find(|f| f.rsync_uri.ends_with(".crl"))
.expect("crl present in snapshot");
let crl = RpkixCrl::decode_der(&crl_file.bytes).expect("decode crl");
let crl = RpkixCrl::decode_der(crl_file.bytes().expect("load crl bytes")).expect("decode crl");
// Choose a validation_time that is within:
// - manifest thisUpdate..nextUpdate (RFC 9286 §6.3)
@@ -151,7 +151,9 @@ fn drop_object_policy_drops_only_failing_object() {
.expect("another ROA present in snapshot");
let victim_uri = pack.files[tamper_idx].rsync_uri.clone();
let mut tampered = pack.files[tamper_idx].bytes.clone();
let mut tampered = pack.files[tamper_idx]
.bytes_cloned()
.expect("clone roa bytes");
let last = tampered.len() - 1;
tampered[last] ^= 0xFF;
pack.files[tamper_idx] = PackFile::from_bytes_compute_sha256(victim_uri.clone(), tampered);
@@ -194,7 +196,9 @@ fn drop_publication_point_policy_drops_the_publication_point() {
.expect("a ROA present in snapshot");
let victim_uri = pack.files[tamper_idx].rsync_uri.clone();
let mut tampered = pack.files[tamper_idx].bytes.clone();
let mut tampered = pack.files[tamper_idx]
.bytes_cloned()
.expect("clone roa bytes");
let last = tampered.len() - 1;
tampered[last] ^= 0xFF;
pack.files[tamper_idx] = PackFile::from_bytes_compute_sha256(victim_uri.clone(), tampered);

8
tests/test_parallel_phase1_transport_offline_r5.rs
View File

@@ -61,11 +61,9 @@ fn offline_serial_and_parallel_phase1_match_compare_views() {
let (serial_vrps, serial_vaps) =
rpki::bundle::decode_ccr_compare_views(&serial_ccr, "apnic").expect("serial compare view");
let (parallel_vrps, parallel_vaps) = rpki::bundle::decode_ccr_compare_views(
&parallel_ccr,
"apnic",
)
.expect("parallel compare view");
let (parallel_vrps, parallel_vaps) =
rpki::bundle::decode_ccr_compare_views(&parallel_ccr, "apnic")
.expect("parallel compare view");
assert_eq!(serial_vrps, parallel_vrps, "VRP compare views must match");
assert_eq!(serial_vaps, parallel_vaps, "VAP compare views must match");

12
tests/test_signed_object_ber_indefinite.rs
View File

@@ -17,11 +17,13 @@ fn signed_object_decode_accepts_real_arin_ber_indefinite_roa_from_multi_rir_bund
let xml_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(
"../../rpki/target/live/20260316-112341-multi-final3/arin/base-payload-archive/v1/captures/arin-base-arin/rrdp/repos/8bd1405242614aed9f13321d266fe3193db0ca842e0cbffda2b3df05481c331b/4a394319-7460-4141-a416-1addb69284ff/snapshot-160090-880434ae2a6f0e5fff224391c65a22ab037e09ac1d3ebad0ceda18558b49b13e.xml",
);
assert!(
xml_path.is_file(),
"xml path missing: {}",
xml_path.display()
);
if !xml_path.is_file() {
eprintln!(
"skipping BER-indefinite signed object fixture test; xml path missing: {}",
xml_path.display()
);
return;
}
let xml = std::fs::read_to_string(&xml_path).expect("read snapshot xml");
let uri = "rsync://rpki.arin.net/repository/arin-rpki-ta/5e4a23ea-e80a-403e-b08c-2171da2157d3/2a246947-2d62-4a6c-ba05-87187f0099b2/9ed5ce80-224e-46ab-94f1-1afce8ccf13f/0b13beb5-6bbb-3994-a254-02c5b10175c5.roa";
let der = extract_publish_bytes(&xml, uri);