NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加 payload replay for snapshot，20260313 迭代

Browse Source
This commit is contained in:
yuyr
2026-03-15 22:49:06 +08:00
parent cf764c35bb
commit 73d8ebb5c1
17 changed files with 3060 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/validation/run.rs
View File

@@ -67,6 +67,7 @@ pub fn run_publication_point_once(
validation_time,
timing: None,
download_log: None,
replay_archive_index: None,
rrdp_dedup: false,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: false,

336
src/validation/run_tree_from_tal.rs
View File

@@ -4,6 +4,9 @@ use crate::analysis::timing::TimingHandle;
use crate::audit::PublicationPointAudit;
use crate::audit_downloads::DownloadLogHandle;
use crate::data_model::ta::TrustAnchor;
use crate::replay::archive::ReplayArchiveIndex;
use crate::replay::fetch_http::PayloadReplayHttpFetcher;
use crate::replay::fetch_rsync::PayloadReplayRsyncFetcher;
use crate::sync::rrdp::Fetcher;
use crate::validation::from_tal::{
DiscoveredRootCaInstance, FromTalError, discover_root_ca_instance_from_tal_and_ta_der,
@@ -15,7 +18,7 @@ use crate::validation::tree::{
};
use crate::validation::tree_runner::Rpkiv1PublicationPointRunner;
use std::collections::HashMap;
use std::sync::Mutex;
use std::sync::{Arc, Mutex};
fn tal_id_from_url_like(s: &str) -> Option<String> {
let url = Url::parse(s).ok()?;
@@ -75,6 +78,9 @@ pub enum RunTreeFromTalError {
#[error("{0}")]
FromTal(#[from] FromTalError),
#[error("payload replay setup failed: {0}")]
Replay(String),
#[error("{0}")]
Tree(#[from] TreeRunError),
}
@@ -120,6 +126,7 @@ pub fn run_tree_from_tal_url_serial(
validation_time,
timing: None,
download_log: None,
replay_archive_index: None,
rrdp_dedup: true,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
@@ -157,6 +164,7 @@ pub fn run_tree_from_tal_url_serial_audit(
validation_time,
timing: None,
download_log: Some(download_log.clone()),
replay_archive_index: None,
rrdp_dedup: true,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
@@ -208,6 +216,7 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing(
validation_time,
timing: Some(timing.clone()),
download_log: Some(download_log.clone()),
replay_archive_index: None,
rrdp_dedup: true,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
@@ -259,6 +268,7 @@ pub fn run_tree_from_tal_and_ta_der_serial(
validation_time,
timing: None,
download_log: None,
replay_archive_index: None,
rrdp_dedup: true,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
@@ -299,6 +309,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit(
validation_time,
timing: None,
download_log: Some(download_log.clone()),
replay_archive_index: None,
rrdp_dedup: true,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
@@ -353,6 +364,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing(
validation_time,
timing: Some(timing.clone()),
download_log: Some(download_log.clone()),
replay_archive_index: None,
rrdp_dedup: true,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
@@ -381,3 +393,325 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing(
download_stats,
})
}
pub fn run_tree_from_tal_and_ta_der_payload_replay_serial(
store: &crate::storage::RocksStore,
policy: &crate::policy::Policy,
tal_bytes: &[u8],
ta_der: &[u8],
resolved_ta_uri: Option<&Url>,
payload_archive_root: &std::path::Path,
payload_locks_path: &std::path::Path,
validation_time: time::OffsetDateTime,
config: &TreeRunConfig,
) -> Result<RunTreeFromTalOutput, RunTreeFromTalError> {
let discovery =
discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri)?;
let replay_index = Arc::new(
ReplayArchiveIndex::load(payload_archive_root, payload_locks_path)
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?,
);
let http_fetcher = PayloadReplayHttpFetcher::new(replay_index.clone())
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?;
let rsync_fetcher = PayloadReplayRsyncFetcher::new(replay_index.clone());
let runner = Rpkiv1PublicationPointRunner {
store,
policy,
http_fetcher: &http_fetcher,
rsync_fetcher: &rsync_fetcher,
validation_time,
timing: None,
download_log: None,
replay_archive_index: Some(replay_index),
rrdp_dedup: true,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
};
let root = root_handle_from_trust_anchor(
&discovery.trust_anchor,
derive_tal_id(&discovery),
None,
&discovery.ca_instance,
);
let tree = run_tree_serial(root, &runner, config)?;
Ok(RunTreeFromTalOutput { discovery, tree })
}
pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit(
store: &crate::storage::RocksStore,
policy: &crate::policy::Policy,
tal_bytes: &[u8],
ta_der: &[u8],
resolved_ta_uri: Option<&Url>,
payload_archive_root: &std::path::Path,
payload_locks_path: &std::path::Path,
validation_time: time::OffsetDateTime,
config: &TreeRunConfig,
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError> {
let discovery =
discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri)?;
let replay_index = Arc::new(
ReplayArchiveIndex::load(payload_archive_root, payload_locks_path)
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?,
);
let http_fetcher = PayloadReplayHttpFetcher::new(replay_index.clone())
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?;
let rsync_fetcher = PayloadReplayRsyncFetcher::new(replay_index.clone());
let download_log = DownloadLogHandle::new();
let runner = Rpkiv1PublicationPointRunner {
store,
policy,
http_fetcher: &http_fetcher,
rsync_fetcher: &rsync_fetcher,
validation_time,
timing: None,
download_log: Some(download_log.clone()),
replay_archive_index: Some(replay_index),
rrdp_dedup: true,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
};
let root = root_handle_from_trust_anchor(
&discovery.trust_anchor,
derive_tal_id(&discovery),
None,
&discovery.ca_instance,
);
let TreeRunAuditOutput {
tree,
publication_points,
} = run_tree_serial_audit(root, &runner, config)?;
let downloads = download_log.snapshot_events();
let download_stats = DownloadLogHandle::stats_from_events(&downloads);
Ok(RunTreeFromTalAuditOutput {
discovery,
tree,
publication_points,
downloads,
download_stats,
})
}
pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing(
store: &crate::storage::RocksStore,
policy: &crate::policy::Policy,
tal_bytes: &[u8],
ta_der: &[u8],
resolved_ta_uri: Option<&Url>,
payload_archive_root: &std::path::Path,
payload_locks_path: &std::path::Path,
validation_time: time::OffsetDateTime,
config: &TreeRunConfig,
timing: &TimingHandle,
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError> {
let _tal = timing.span_phase("tal_bootstrap");
let discovery =
discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri)?;
drop(_tal);
let replay_index = Arc::new(
ReplayArchiveIndex::load(payload_archive_root, payload_locks_path)
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?,
);
let http_fetcher = PayloadReplayHttpFetcher::new(replay_index.clone())
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?;
let rsync_fetcher = PayloadReplayRsyncFetcher::new(replay_index.clone());
let download_log = DownloadLogHandle::new();
let runner = Rpkiv1PublicationPointRunner {
store,
policy,
http_fetcher: &http_fetcher,
rsync_fetcher: &rsync_fetcher,
validation_time,
timing: Some(timing.clone()),
download_log: Some(download_log.clone()),
replay_archive_index: Some(replay_index),
rrdp_dedup: true,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
rsync_repo_cache: Mutex::new(HashMap::new()),
};
let root = root_handle_from_trust_anchor(
&discovery.trust_anchor,
derive_tal_id(&discovery),
None,
&discovery.ca_instance,
);
let _tree = timing.span_phase("tree_run_total");
let TreeRunAuditOutput {
tree,
publication_points,
} = run_tree_serial_audit(root, &runner, config)?;
let downloads = download_log.snapshot_events();
let download_stats = DownloadLogHandle::stats_from_events(&downloads);
Ok(RunTreeFromTalAuditOutput {
discovery,
tree,
publication_points,
downloads,
download_stats,
})
}
#[cfg(test)]
mod replay_api_tests {
use super::*;
use crate::analysis::timing::{TimingHandle, TimingMeta};
use time::format_description::well_known::Rfc3339;
fn apnic_replay_inputs() -> (
Vec<u8>,
Vec<u8>,
std::path::PathBuf,
std::path::PathBuf,
time::OffsetDateTime,
) {
let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal")
.expect("read apnic tal fixture");
let ta_der =
std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture");
let archive_root = std::path::PathBuf::from("target/live/payload_replay/payload-archive");
let locks_path = std::path::PathBuf::from("target/live/payload_replay/locks.json");
let validation_time = time::OffsetDateTime::parse("2026-03-13T02:30:00Z", &Rfc3339)
.expect("parse validation time");
(tal_bytes, ta_der, archive_root, locks_path, validation_time)
}
#[test]
fn payload_replay_api_reports_setup_error_for_missing_archive() {
let temp = tempfile::tempdir().expect("tempdir");
let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db");
let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal")
.expect("read apnic tal fixture");
let ta_der =
std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture");
let err = run_tree_from_tal_and_ta_der_payload_replay_serial_audit(
&store,
&crate::policy::Policy::default(),
&tal_bytes,
&ta_der,
None,
std::path::Path::new("tests/fixtures/missing-payload-archive"),
std::path::Path::new("tests/fixtures/missing-locks.json"),
time::OffsetDateTime::now_utc(),
&TreeRunConfig {
max_depth: Some(0),
max_instances: Some(1),
},
)
.unwrap_err();
assert!(matches!(err, RunTreeFromTalError::Replay(_)), "{err}");
}
#[test]
fn payload_replay_api_root_only_apnic_archive_runs() {
let temp = tempfile::tempdir().expect("tempdir");
let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db");
let (tal_bytes, ta_der, archive_root, locks_path, validation_time) = apnic_replay_inputs();
assert!(
archive_root.is_dir(),
"payload replay archive missing: {}",
archive_root.display()
);
assert!(
locks_path.is_file(),
"payload replay locks missing: {}",
locks_path.display()
);
let out = run_tree_from_tal_and_ta_der_payload_replay_serial_audit(
&store,
&crate::policy::Policy::default(),
&tal_bytes,
&ta_der,
None,
&archive_root,
&locks_path,
validation_time,
&TreeRunConfig {
max_depth: Some(0),
max_instances: Some(1),
},
)
.expect("run replay root-only audit");
assert_eq!(out.tree.instances_processed, 1);
assert_eq!(out.tree.instances_failed, 0);
assert_eq!(out.publication_points.len(), 1);
assert_eq!(out.discovery.trust_anchor.resolved_ta_uri, None);
assert!(!out.downloads.is_empty());
assert!(
out.downloads.iter().all(|d| d.success),
"expected successful replay downloads"
);
}
#[test]
fn payload_replay_api_root_only_apnic_archive_runs_with_timing() {
let temp = tempfile::tempdir().expect("tempdir");
let db_path = temp.path().join("db");
let store = crate::storage::RocksStore::open(&db_path).expect("open db");
let (tal_bytes, ta_der, archive_root, locks_path, validation_time) = apnic_replay_inputs();
assert!(
archive_root.is_dir(),
"payload replay archive missing: {}",
archive_root.display()
);
assert!(
locks_path.is_file(),
"payload replay locks missing: {}",
locks_path.display()
);
let timing = TimingHandle::new(TimingMeta {
recorded_at_utc_rfc3339: "2026-03-13T03:00:00Z".to_string(),
validation_time_utc_rfc3339: "2026-03-13T02:30:00Z".to_string(),
tal_url: None,
db_path: Some(db_path.to_string_lossy().into_owned()),
});
let out = run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing(
&store,
&crate::policy::Policy::default(),
&tal_bytes,
&ta_der,
None,
&archive_root,
&locks_path,
validation_time,
&TreeRunConfig {
max_depth: Some(0),
max_instances: Some(1),
},
&timing,
)
.expect("run replay root-only audit with timing");
assert_eq!(out.tree.instances_processed, 1);
let timing_json = temp.path().join("timing_replay.json");
timing
.write_json(&timing_json, 20)
.expect("write timing json");
let json: serde_json::Value =
serde_json::from_slice(&std::fs::read(&timing_json).expect("read timing json"))
.expect("parse timing json");
let counts = json.get("counts").expect("counts");
assert!(
counts
.get("repo_sync_rrdp_ok_total")
.and_then(|v| v.as_u64())
.unwrap_or(0)
>= 1
);
}
}

45
src/validation/tree_runner.rs
View File

@@ -11,6 +11,7 @@ use crate::data_model::rc::ResourceCertificate;
use crate::data_model::roa::{RoaAfi, RoaObject};
use crate::fetch::rsync::RsyncFetcher;
use crate::policy::Policy;
use crate::replay::archive::ReplayArchiveIndex;
use crate::report::{RfcRef, Warning};
use crate::storage::{
AuditRuleIndexEntry, AuditRuleKind, PackFile, PackTime, RawByHashEntry, RocksStore,
@@ -18,7 +19,7 @@ use crate::storage::{
VcirAuditSummary, VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirOutputType,
VcirRelatedArtifact, VcirSummary,
};
use crate::sync::repo::sync_publication_point;
use crate::sync::repo::{sync_publication_point, sync_publication_point_replay};
use crate::sync::rrdp::Fetcher;
use crate::validation::ca_instance::ca_instance_uris_from_ca_certificate;
use crate::validation::ca_path::{
@@ -34,7 +35,7 @@ use crate::validation::tree::{
CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult, PublicationPointRunner,
};
use std::collections::{HashMap, HashSet};
use std::sync::Mutex;
use std::sync::{Arc, Mutex};
use serde::Deserialize;
use serde_json::json;
@@ -49,6 +50,7 @@ pub struct Rpkiv1PublicationPointRunner<'a> {
pub validation_time: time::OffsetDateTime,
pub timing: Option<TimingHandle>,
pub download_log: Option<DownloadLogHandle>,
pub replay_archive_index: Option<Arc<ReplayArchiveIndex>>,
/// In-run RRDP dedup: when RRDP is enabled, only sync each `rrdp_notification_uri` once per run.
///
/// - If RRDP succeeded for a repo, later publication points referencing that same RRDP repo
@@ -151,16 +153,29 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
.map(|t| t.span_phase("repo_sync_total"));
let _repo_span = self.timing.as_ref().map(|t| t.span_rrdp_repo(repo_key));
match sync_publication_point(
self.store,
self.policy,
effective_notification_uri,
&ca.rsync_base_uri,
self.http_fetcher,
self.rsync_fetcher,
self.timing.as_ref(),
self.download_log.as_ref(),
) {
match if let Some(replay_index) = self.replay_archive_index.as_ref() {
sync_publication_point_replay(
self.store,
replay_index,
effective_notification_uri,
&ca.rsync_base_uri,
self.http_fetcher,
self.rsync_fetcher,
self.timing.as_ref(),
self.download_log.as_ref(),
)
} else {
sync_publication_point(
self.store,
self.policy,
effective_notification_uri,
&ca.rsync_base_uri,
self.http_fetcher,
self.rsync_fetcher,
self.timing.as_ref(),
self.download_log.as_ref(),
)
} {
Ok(res) => {
if self.rsync_dedup && res.source == crate::sync::repo::RepoSyncSource::Rsync {
let base = normalize_rsync_base_uri(&ca.rsync_base_uri);
@@ -2857,6 +2872,7 @@ authorityKeyIdentifier = keyid:always
validation_time,
timing: None,
download_log: None,
replay_archive_index: None,
rrdp_dedup: false,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: false,
@@ -3023,6 +3039,7 @@ authorityKeyIdentifier = keyid:always
validation_time,
timing: None,
download_log: None,
replay_archive_index: None,
rrdp_dedup: false,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,
@@ -3100,6 +3117,7 @@ authorityKeyIdentifier = keyid:always
validation_time,
timing: None,
download_log: None,
replay_archive_index: None,
rrdp_dedup: false,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: false,
@@ -3123,6 +3141,7 @@ authorityKeyIdentifier = keyid:always
validation_time,
timing: None,
download_log: None,
replay_archive_index: None,
rrdp_dedup: false,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: false,
@@ -4107,6 +4126,7 @@ authorityKeyIdentifier = keyid:always
validation_time,
timing: Some(timing.clone()),
download_log: None,
replay_archive_index: None,
rrdp_dedup: true,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: false,
@@ -4133,6 +4153,7 @@ authorityKeyIdentifier = keyid:always
validation_time,
timing: Some(timing),
download_log: None,
replay_archive_index: None,
rrdp_dedup: false,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: true,