NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260418_2 phase2 并行优化 mix quick 耗时105/48秒

Browse Source
This commit is contained in:
yuyr
2026-04-19 00:08:29 +08:00
parent 417c82bef6
commit f6a601e16c
21 changed files with 4127 additions and 594 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/test_apnic_stats_live_stage2.rs
View File

@@ -182,6 +182,8 @@ fn apnic_tree_full_stats_serial() {
rsync_repo_cache: std::sync::Mutex::new(std::collections::HashMap::new()),
current_repo_index: None,
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
};
let stats = RefCell::new(LiveStats::default());

151
tests/test_objects_process_publication_point_snapshot.rs
View File

@@ -1,10 +1,14 @@
use rpki::fetch::rsync::LocalDirRsyncFetcher;
use rpki::parallel::config::ParallelPhase2Config;
use rpki::policy::{Policy, SignedObjectFailurePolicy, SyncPreference};
use rpki::storage::{PackFile, PackTime, RocksStore, VcirOutputType};
use rpki::sync::repo::sync_publication_point;
use rpki::sync::rrdp::Fetcher;
use rpki::validation::manifest::process_manifest_publication_point;
use rpki::validation::objects::process_publication_point_snapshot_for_issuer;
use rpki::validation::objects::{
process_publication_point_snapshot_for_issuer,
process_publication_point_snapshot_for_issuer_parallel_roa,
};
use rpki::validation::publication_point::PublicationPointSnapshot;
struct NoopHttpFetcher;
@@ -148,6 +152,151 @@ fn process_snapshot_for_issuer_extracts_vrps_from_real_cernet_fixture() {
assert!(out.aspas.is_empty());
}
#[test]
fn parallel_roa_processing_matches_serial_for_real_cernet_fixture() {
let (dir, rsync_base_uri, manifest_file) = cernet_fixture();
let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}");
let validation_time = validation_time_from_manifest_fixture(&dir, &manifest_file);
let pack = build_publication_point_snapshot_from_local_rsync_fixture(
&dir,
&rsync_base_uri,
&manifest_rsync_uri,
validation_time,
);
let issuer_ca_der = issuer_ca_fixture();
let issuer_ca = rpki::data_model::rc::ResourceCertificate::decode_der(&issuer_ca_der)
.expect("decode issuer ca");
let policy = Policy::default();
let serial = process_publication_point_snapshot_for_issuer(
&pack,
&policy,
&issuer_ca_der,
Some(issuer_ca_rsync_uri()),
issuer_ca.tbs.extensions.ip_resources.as_ref(),
issuer_ca.tbs.extensions.as_resources.as_ref(),
validation_time,
None,
);
let parallel = process_publication_point_snapshot_for_issuer_parallel_roa(
&pack,
&policy,
&issuer_ca_der,
Some(issuer_ca_rsync_uri()),
issuer_ca.tbs.extensions.ip_resources.as_ref(),
issuer_ca.tbs.extensions.as_resources.as_ref(),
validation_time,
None,
&ParallelPhase2Config {
object_workers: 2,
worker_queue_capacity: 4,
},
);
assert_eq!(parallel.vrps, serial.vrps);
assert_eq!(parallel.aspas, serial.aspas);
assert_eq!(parallel.local_outputs_cache, serial.local_outputs_cache);
assert_eq!(parallel.audit, serial.audit);
assert_eq!(parallel.warnings, serial.warnings);
assert_eq!(parallel.stats, serial.stats);
}
#[test]
fn parallel_roa_processing_reports_issuer_decode_failure_like_serial() {
let (dir, rsync_base_uri, manifest_file) = cernet_fixture();
let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}");
let validation_time = validation_time_from_manifest_fixture(&dir, &manifest_file);
let pack = build_publication_point_snapshot_from_local_rsync_fixture(
&dir,
&rsync_base_uri,
&manifest_rsync_uri,
validation_time,
);
let policy = Policy::default();
let serial = process_publication_point_snapshot_for_issuer(
&pack,
&policy,
&[0, 1, 2],
Some(issuer_ca_rsync_uri()),
None,
None,
validation_time,
None,
);
let parallel = process_publication_point_snapshot_for_issuer_parallel_roa(
&pack,
&policy,
&[0, 1, 2],
Some(issuer_ca_rsync_uri()),
None,
None,
validation_time,
None,
&ParallelPhase2Config {
object_workers: 2,
worker_queue_capacity: 4,
},
);
assert_eq!(parallel.vrps, serial.vrps);
assert_eq!(parallel.audit, serial.audit);
assert_eq!(parallel.warnings, serial.warnings);
assert_eq!(parallel.stats, serial.stats);
assert!(parallel.stats.publication_point_dropped);
}
#[test]
fn parallel_roa_processing_reports_missing_crl_like_serial() {
let (dir, rsync_base_uri, manifest_file) = cernet_fixture();
let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}");
let validation_time = validation_time_from_manifest_fixture(&dir, &manifest_file);
let mut pack = build_publication_point_snapshot_from_local_rsync_fixture(
&dir,
&rsync_base_uri,
&manifest_rsync_uri,
validation_time,
);
pack.files.retain(|file| !file.rsync_uri.ends_with(".crl"));
let issuer_ca_der = issuer_ca_fixture();
let issuer_ca = rpki::data_model::rc::ResourceCertificate::decode_der(&issuer_ca_der)
.expect("decode issuer ca");
let policy = Policy::default();
let serial = process_publication_point_snapshot_for_issuer(
&pack,
&policy,
&issuer_ca_der,
Some(issuer_ca_rsync_uri()),
issuer_ca.tbs.extensions.ip_resources.as_ref(),
issuer_ca.tbs.extensions.as_resources.as_ref(),
validation_time,
None,
);
let parallel = process_publication_point_snapshot_for_issuer_parallel_roa(
&pack,
&policy,
&issuer_ca_der,
Some(issuer_ca_rsync_uri()),
issuer_ca.tbs.extensions.ip_resources.as_ref(),
issuer_ca.tbs.extensions.as_resources.as_ref(),
validation_time,
None,
&ParallelPhase2Config {
object_workers: 2,
worker_queue_capacity: 4,
},
);
assert_eq!(parallel.vrps, serial.vrps);
assert_eq!(parallel.audit, serial.audit);
assert_eq!(parallel.warnings, serial.warnings);
assert_eq!(parallel.stats, serial.stats);
assert!(parallel.stats.publication_point_dropped);
}
#[test]
fn signed_object_failure_policy_drop_object_drops_only_bad_object() {
let (dir, rsync_base_uri, manifest_file) = cernet_fixture();

232
tests/test_objects_processing_coverage_m18.rs
View File

@@ -1,6 +1,11 @@
use rpki::parallel::config::ParallelPhase2Config;
use rpki::policy::{Policy, SignedObjectFailurePolicy};
use rpki::storage::{PackFile, PackTime};
use rpki::validation::objects::process_publication_point_snapshot_for_issuer;
use rpki::validation::objects::{
process_publication_point_for_issuer_parallel_roa_with_pool,
process_publication_point_snapshot_for_issuer,
process_publication_point_snapshot_for_issuer_parallel_roa, ParallelRoaWorkerPool,
};
use rpki::validation::publication_point::PublicationPointSnapshot;
fn fixture_bytes(path: &str) -> Vec<u8> {
@@ -517,3 +522,228 @@ fn process_snapshot_for_issuer_selects_crl_by_ee_crldp_uri_aspa() {
assert_eq!(out.audit.len(), 1);
assert_eq!(out.warnings.len(), 1);
}
#[test]
fn parallel_roa_processing_drop_object_records_roa_and_aspa_errors_like_serial() {
let manifest_bytes = fixture_bytes(
"tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft",
);
let pack = dummy_snapshot(
manifest_bytes,
vec![
PackFile::from_bytes_compute_sha256(
"rsync://example.test/repo/pp/issuer.crl",
vec![0x01],
),
PackFile::from_bytes_compute_sha256("rsync://example.test/repo/pp/a.roa", vec![0x00]),
PackFile::from_bytes_compute_sha256("rsync://example.test/repo/pp/b.roa", vec![0x00]),
PackFile::from_bytes_compute_sha256("rsync://example.test/repo/pp/c.asa", vec![0x00]),
],
);
let policy = Policy {
signed_object_failure_policy: SignedObjectFailurePolicy::DropObject,
..Policy::default()
};
let issuer_ca_der = fixture_bytes("tests/fixtures/ta/apnic-ta.cer");
let serial = process_publication_point_snapshot_for_issuer(
&pack,
&policy,
&issuer_ca_der,
None,
None,
None,
time::OffsetDateTime::now_utc(),
None,
);
let phase2_config = ParallelPhase2Config {
object_workers: 2,
worker_queue_capacity: 1,
};
let pool = ParallelRoaWorkerPool::new(&phase2_config).expect("parallel roa pool");
let parallel = process_publication_point_for_issuer_parallel_roa_with_pool(
&pack,
&policy,
&issuer_ca_der,
None,
None,
None,
time::OffsetDateTime::now_utc(),
None,
&pool,
);
let parallel_again = process_publication_point_for_issuer_parallel_roa_with_pool(
&pack,
&policy,
&issuer_ca_der,
None,
None,
None,
time::OffsetDateTime::now_utc(),
None,
&pool,
);
assert_eq!(parallel.stats, serial.stats);
assert_eq!(parallel_again.stats, serial.stats);
assert_eq!(parallel.audit, serial.audit);
assert_eq!(parallel.warnings.len(), serial.warnings.len());
assert_eq!(parallel.stats.roa_total, 2);
assert_eq!(parallel.stats.aspa_total, 1);
assert_eq!(parallel.audit.len(), 3);
}
#[test]
fn parallel_roa_processing_falls_back_for_drop_publication_point_policy() {
let manifest_bytes = fixture_bytes(
"tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft",
);
let pack = dummy_snapshot(
manifest_bytes,
vec![
PackFile::from_bytes_compute_sha256(
"rsync://example.test/repo/pp/issuer.crl",
vec![0x01],
),
PackFile::from_bytes_compute_sha256("rsync://example.test/repo/pp/a.roa", vec![0x00]),
PackFile::from_bytes_compute_sha256("rsync://example.test/repo/pp/b.roa", vec![0x00]),
],
);
let policy = Policy {
signed_object_failure_policy: SignedObjectFailurePolicy::DropPublicationPoint,
..Policy::default()
};
let issuer_ca_der = fixture_bytes("tests/fixtures/ta/apnic-ta.cer");
let serial = process_publication_point_snapshot_for_issuer(
&pack,
&policy,
&issuer_ca_der,
None,
None,
None,
time::OffsetDateTime::now_utc(),
None,
);
let parallel = process_publication_point_snapshot_for_issuer_parallel_roa(
&pack,
&policy,
&issuer_ca_der,
None,
None,
None,
time::OffsetDateTime::now_utc(),
None,
&ParallelPhase2Config {
object_workers: 2,
worker_queue_capacity: 1,
},
);
assert_eq!(parallel.stats, serial.stats);
assert_eq!(parallel.audit, serial.audit);
assert_eq!(parallel.warnings, serial.warnings);
assert!(parallel.stats.publication_point_dropped);
}
#[test]
fn parallel_roa_processing_falls_back_for_single_worker_config() {
let manifest_bytes = fixture_bytes(
"tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft",
);
let pack = dummy_snapshot(
manifest_bytes,
vec![
PackFile::from_bytes_compute_sha256(
"rsync://example.test/repo/pp/issuer.crl",
vec![0x01],
),
PackFile::from_bytes_compute_sha256("rsync://example.test/repo/pp/a.roa", vec![0x00]),
],
);
let policy = Policy {
signed_object_failure_policy: SignedObjectFailurePolicy::DropObject,
..Policy::default()
};
let issuer_ca_der = fixture_bytes("tests/fixtures/ta/apnic-ta.cer");
let serial = process_publication_point_snapshot_for_issuer(
&pack,
&policy,
&issuer_ca_der,
None,
None,
None,
time::OffsetDateTime::now_utc(),
None,
);
let parallel = process_publication_point_snapshot_for_issuer_parallel_roa(
&pack,
&policy,
&issuer_ca_der,
None,
None,
None,
time::OffsetDateTime::now_utc(),
None,
&ParallelPhase2Config {
object_workers: 1,
worker_queue_capacity: 1,
},
);
assert_eq!(parallel.stats, serial.stats);
assert_eq!(parallel.audit, serial.audit);
assert_eq!(parallel.warnings, serial.warnings);
}
#[test]
fn parallel_roa_processing_falls_back_when_pool_creation_fails() {
let manifest_bytes = fixture_bytes(
"tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft",
);
let pack = dummy_snapshot(
manifest_bytes,
vec![
PackFile::from_bytes_compute_sha256(
"rsync://example.test/repo/pp/issuer.crl",
vec![0x01],
),
PackFile::from_bytes_compute_sha256("rsync://example.test/repo/pp/a.roa", vec![0x00]),
],
);
let policy = Policy {
signed_object_failure_policy: SignedObjectFailurePolicy::DropObject,
..Policy::default()
};
let issuer_ca_der = fixture_bytes("tests/fixtures/ta/apnic-ta.cer");
let serial = process_publication_point_snapshot_for_issuer(
&pack,
&policy,
&issuer_ca_der,
None,
None,
None,
time::OffsetDateTime::now_utc(),
None,
);
let parallel = process_publication_point_snapshot_for_issuer_parallel_roa(
&pack,
&policy,
&issuer_ca_der,
None,
None,
None,
time::OffsetDateTime::now_utc(),
None,
&ParallelPhase2Config {
object_workers: 2,
worker_queue_capacity: 0,
},
);
assert_eq!(parallel.stats, serial.stats);
assert_eq!(parallel.audit, serial.audit);
assert_eq!(parallel.warnings, serial.warnings);
}

42
tests/test_parallel_phase1_transport_offline_r5.rs
View File

@@ -4,7 +4,7 @@ fn fixture_path(rel: &str) -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(rel)
}
fn run_offline_case(parallel_phase1: bool) -> (serde_json::Value, Vec<u8>) {
fn run_offline_case(parallel_phase1: bool, parallel_phase2: bool) -> (serde_json::Value, Vec<u8>) {
let db_dir = tempfile::tempdir().expect("db tempdir");
let out_dir = tempfile::tempdir().expect("out tempdir");
let report_path = out_dir.path().join("report.json");
@@ -41,6 +41,13 @@ fn run_offline_case(parallel_phase1: bool) -> (serde_json::Value, Vec<u8>) {
if parallel_phase1 {
argv.push("--parallel-phase1".to_string());
}
if parallel_phase2 {
argv.push("--parallel-phase2".to_string());
argv.push("--parallel-phase2-object-workers".to_string());
argv.push("4".to_string());
argv.push("--parallel-phase2-worker-queue-capacity".to_string());
argv.push("64".to_string());
}
rpki::cli::run(&argv).expect("cli run");
@@ -52,8 +59,8 @@ fn run_offline_case(parallel_phase1: bool) -> (serde_json::Value, Vec<u8>) {
#[test]
fn offline_serial_and_parallel_phase1_match_compare_views() {
let (serial_report, serial_ccr_bytes) = run_offline_case(false);
let (parallel_report, parallel_ccr_bytes) = run_offline_case(true);
let (serial_report, serial_ccr_bytes) = run_offline_case(false, false);
let (parallel_report, parallel_ccr_bytes) = run_offline_case(true, false);
let serial_ccr = rpki::ccr::decode_content_info(&serial_ccr_bytes).expect("decode serial ccr");
let parallel_ccr =
@@ -80,3 +87,32 @@ fn offline_serial_and_parallel_phase1_match_compare_views() {
"publication point counts must match"
);
}
#[test]
fn offline_serial_and_parallel_phase2_match_compare_views() {
let (serial_report, serial_ccr_bytes) = run_offline_case(false, false);
let (phase2_report, phase2_ccr_bytes) = run_offline_case(true, true);
let serial_ccr = rpki::ccr::decode_content_info(&serial_ccr_bytes).expect("decode serial ccr");
let phase2_ccr = rpki::ccr::decode_content_info(&phase2_ccr_bytes).expect("decode phase2 ccr");
let (serial_vrps, serial_vaps) =
rpki::bundle::decode_ccr_compare_views(&serial_ccr, "apnic").expect("serial compare view");
let (phase2_vrps, phase2_vaps) =
rpki::bundle::decode_ccr_compare_views(&phase2_ccr, "apnic").expect("phase2 compare view");
assert_eq!(serial_vrps, phase2_vrps, "VRP compare views must match");
assert_eq!(serial_vaps, phase2_vaps, "VAP compare views must match");
let serial_points = serial_report["publication_points"]
.as_array()
.expect("serial publication_points");
let phase2_points = phase2_report["publication_points"]
.as_array()
.expect("phase2 publication_points");
assert_eq!(
serial_points.len(),
phase2_points.len(),
"publication point counts must match"
);
}