20260401 live recorder 扩展到1+N个delta,完成5个RIR录制,以及三方replay,结果三方均对齐vrps和vaps
This commit is contained in:
@@ -3,11 +3,15 @@ use crate::data_model::rc::{ResourceCertKind, ResourceCertificate, SubjectInfoAc
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct CaInstanceUris {
|
||||
/// CA repository base rsync URI (must end with `/`).
|
||||
/// rsync sync base URI used for live/replay publication point fetches (must end with `/`).
|
||||
///
|
||||
/// This is the parent directory of `manifest_rsync_uri`, not necessarily the full
|
||||
/// `id-ad-caRepository` URI. Using the manifest parent keeps sync scope narrow while
|
||||
/// `publication_point_rsync_uri` preserves the full RFC 9286 publication-point value.
|
||||
pub rsync_base_uri: String,
|
||||
/// rsync URI for the manifest object (`.mft`).
|
||||
pub manifest_rsync_uri: String,
|
||||
/// Publication point rsync URI (RFC 9286 terminology). In v1 this equals `rsync_base_uri`.
|
||||
/// Publication point rsync URI (RFC 9286 terminology).
|
||||
pub publication_point_rsync_uri: String,
|
||||
/// Optional RRDP notification URI (https).
|
||||
pub rrdp_notification_uri: Option<String>,
|
||||
@@ -104,8 +108,6 @@ pub fn ca_instance_uris_from_ca_certificate(
|
||||
publication_point_rsync_uri.push('/');
|
||||
}
|
||||
|
||||
let rsync_base_uri = publication_point_rsync_uri.clone();
|
||||
|
||||
let manifest_rsync_uri = manifest.ok_or(CaInstanceUrisError::MissingRpkiManifest)?;
|
||||
if !manifest_rsync_uri.starts_with(&publication_point_rsync_uri) {
|
||||
return Err(CaInstanceUrisError::ManifestNotUnderPublicationPoint {
|
||||
@@ -113,9 +115,16 @@ pub fn ca_instance_uris_from_ca_certificate(
|
||||
publication_point_rsync_uri,
|
||||
});
|
||||
}
|
||||
let manifest_parent = manifest_rsync_uri
|
||||
.rsplit_once('/')
|
||||
.map(|(parent, _)| format!("{parent}/"))
|
||||
.ok_or_else(|| CaInstanceUrisError::ManifestNotUnderPublicationPoint {
|
||||
manifest_rsync_uri: manifest_rsync_uri.clone(),
|
||||
publication_point_rsync_uri: publication_point_rsync_uri.clone(),
|
||||
})?;
|
||||
|
||||
Ok(CaInstanceUris {
|
||||
rsync_base_uri,
|
||||
rsync_base_uri: manifest_parent,
|
||||
manifest_rsync_uri,
|
||||
publication_point_rsync_uri,
|
||||
rrdp_notification_uri: notify,
|
||||
|
||||
@@ -7,7 +7,9 @@ use crate::data_model::ta::TrustAnchor;
|
||||
use crate::replay::archive::ReplayArchiveIndex;
|
||||
use crate::replay::delta_archive::ReplayDeltaArchiveIndex;
|
||||
use crate::replay::delta_fetch_http::PayloadDeltaReplayHttpFetcher;
|
||||
use crate::replay::delta_fetch_rsync::PayloadDeltaReplayRsyncFetcher;
|
||||
use crate::replay::delta_fetch_rsync::{
|
||||
PayloadDeltaReplayCurrentStoreRsyncFetcher, PayloadDeltaReplayRsyncFetcher,
|
||||
};
|
||||
use crate::replay::fetch_http::PayloadReplayHttpFetcher;
|
||||
use crate::replay::fetch_rsync::PayloadReplayRsyncFetcher;
|
||||
use crate::sync::rrdp::Fetcher;
|
||||
@@ -629,6 +631,33 @@ fn build_payload_delta_replay_runner<'a>(
|
||||
}
|
||||
}
|
||||
|
||||
fn build_payload_delta_replay_current_store_runner<'a>(
|
||||
store: &'a crate::storage::RocksStore,
|
||||
policy: &'a crate::policy::Policy,
|
||||
delta_index: Arc<ReplayDeltaArchiveIndex>,
|
||||
http_fetcher: &'a PayloadDeltaReplayHttpFetcher,
|
||||
rsync_fetcher: &'a PayloadDeltaReplayCurrentStoreRsyncFetcher<'a>,
|
||||
validation_time: time::OffsetDateTime,
|
||||
timing: Option<TimingHandle>,
|
||||
download_log: Option<DownloadLogHandle>,
|
||||
) -> Rpkiv1PublicationPointRunner<'a> {
|
||||
Rpkiv1PublicationPointRunner {
|
||||
store,
|
||||
policy,
|
||||
http_fetcher,
|
||||
rsync_fetcher,
|
||||
validation_time,
|
||||
timing,
|
||||
download_log,
|
||||
replay_archive_index: None,
|
||||
replay_delta_index: Some(delta_index),
|
||||
rrdp_dedup: true,
|
||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||
rsync_dedup: true,
|
||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
fn run_payload_delta_replay_audit_inner(
|
||||
store: &crate::storage::RocksStore,
|
||||
policy: &crate::policy::Policy,
|
||||
@@ -808,6 +837,110 @@ pub fn run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit_with_timin
|
||||
)
|
||||
}
|
||||
|
||||
fn run_payload_delta_replay_step_audit_inner(
|
||||
store: &crate::storage::RocksStore,
|
||||
policy: &crate::policy::Policy,
|
||||
discovery: DiscoveredRootCaInstance,
|
||||
delta_payload_archive_root: &std::path::Path,
|
||||
previous_locks_path: &std::path::Path,
|
||||
delta_locks_path: &std::path::Path,
|
||||
validation_time: time::OffsetDateTime,
|
||||
config: &TreeRunConfig,
|
||||
timing: Option<TimingHandle>,
|
||||
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError> {
|
||||
let delta_index = Arc::new(
|
||||
ReplayDeltaArchiveIndex::load(delta_payload_archive_root, delta_locks_path)
|
||||
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?,
|
||||
);
|
||||
delta_index
|
||||
.validate_base_locks_sha256_file(previous_locks_path)
|
||||
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?;
|
||||
|
||||
let root = root_handle_from_trust_anchor(
|
||||
&discovery.trust_anchor,
|
||||
derive_tal_id(&discovery),
|
||||
None,
|
||||
&discovery.ca_instance,
|
||||
);
|
||||
|
||||
let delta_http_fetcher = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone())
|
||||
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?;
|
||||
let delta_rsync_fetcher =
|
||||
PayloadDeltaReplayCurrentStoreRsyncFetcher::new(store, delta_index.clone());
|
||||
let download_log = DownloadLogHandle::new();
|
||||
|
||||
let (tree, publication_points) = if let Some(t) = timing.as_ref() {
|
||||
let _phase = t.span_phase("payload_delta_replay_step_total");
|
||||
let delta_runner = build_payload_delta_replay_current_store_runner(
|
||||
store,
|
||||
policy,
|
||||
delta_index,
|
||||
&delta_http_fetcher,
|
||||
&delta_rsync_fetcher,
|
||||
validation_time,
|
||||
Some(t.clone()),
|
||||
Some(download_log.clone()),
|
||||
);
|
||||
let TreeRunAuditOutput {
|
||||
tree,
|
||||
publication_points,
|
||||
} = run_tree_serial_audit(root, &delta_runner, config)?;
|
||||
(tree, publication_points)
|
||||
} else {
|
||||
let delta_runner = build_payload_delta_replay_current_store_runner(
|
||||
store,
|
||||
policy,
|
||||
delta_index,
|
||||
&delta_http_fetcher,
|
||||
&delta_rsync_fetcher,
|
||||
validation_time,
|
||||
None,
|
||||
Some(download_log.clone()),
|
||||
);
|
||||
let TreeRunAuditOutput {
|
||||
tree,
|
||||
publication_points,
|
||||
} = run_tree_serial_audit(root, &delta_runner, config)?;
|
||||
(tree, publication_points)
|
||||
};
|
||||
let downloads = download_log.snapshot_events();
|
||||
let download_stats = DownloadLogHandle::stats_from_events(&downloads);
|
||||
Ok(RunTreeFromTalAuditOutput {
|
||||
discovery,
|
||||
tree,
|
||||
publication_points,
|
||||
downloads,
|
||||
download_stats,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn run_tree_from_tal_and_ta_der_payload_delta_replay_step_serial_audit(
|
||||
store: &crate::storage::RocksStore,
|
||||
policy: &crate::policy::Policy,
|
||||
tal_bytes: &[u8],
|
||||
ta_der: &[u8],
|
||||
resolved_ta_uri: Option<&Url>,
|
||||
delta_payload_archive_root: &std::path::Path,
|
||||
previous_locks_path: &std::path::Path,
|
||||
delta_locks_path: &std::path::Path,
|
||||
validation_time: time::OffsetDateTime,
|
||||
config: &TreeRunConfig,
|
||||
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError> {
|
||||
let discovery =
|
||||
discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri)?;
|
||||
run_payload_delta_replay_step_audit_inner(
|
||||
store,
|
||||
policy,
|
||||
discovery,
|
||||
delta_payload_archive_root,
|
||||
previous_locks_path,
|
||||
delta_locks_path,
|
||||
validation_time,
|
||||
config,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod replay_api_tests {
|
||||
use super::*;
|
||||
|
||||
@@ -83,6 +83,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
&self,
|
||||
ca: &CaInstanceHandle,
|
||||
) -> Result<PublicationPointRunResult, String> {
|
||||
let publication_point_started = std::time::Instant::now();
|
||||
let _pp_total = self
|
||||
.timing
|
||||
.as_ref()
|
||||
@@ -97,6 +98,15 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
}
|
||||
|
||||
let mut warnings: Vec<Warning> = Vec::new();
|
||||
crate::progress_log::emit(
|
||||
"publication_point_start",
|
||||
serde_json::json!({
|
||||
"manifest_rsync_uri": ca.manifest_rsync_uri,
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"rsync_base_uri": ca.rsync_base_uri,
|
||||
"rrdp_notification_uri": ca.rrdp_notification_uri,
|
||||
}),
|
||||
);
|
||||
|
||||
let attempted_rrdp =
|
||||
self.policy.sync_preference == crate::policy::SyncPreference::RrdpThenRsync;
|
||||
@@ -134,7 +144,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
}
|
||||
|
||||
if !skip_sync_due_to_dedup && effective_notification_uri.is_none() && self.rsync_dedup {
|
||||
let base = normalize_rsync_base_uri(&ca.rsync_base_uri);
|
||||
let base = self.rsync_fetcher.dedup_key(&ca.rsync_base_uri);
|
||||
let hit_ok = self
|
||||
.rsync_repo_cache
|
||||
.lock()
|
||||
@@ -153,8 +163,14 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
let (repo_sync_ok, repo_sync_err): (bool, Option<String>) = if skip_sync_due_to_dedup {
|
||||
(true, None)
|
||||
let repo_sync_started = std::time::Instant::now();
|
||||
let (repo_sync_ok, repo_sync_err, repo_sync_source): (bool, Option<String>, Option<String>) = if skip_sync_due_to_dedup {
|
||||
let source = if effective_notification_uri.is_some() {
|
||||
Some("rrdp_dedup_skip".to_string())
|
||||
} else {
|
||||
Some("rsync_dedup_skip".to_string())
|
||||
};
|
||||
(true, None, source)
|
||||
} else {
|
||||
let repo_key = effective_notification_uri.unwrap_or_else(|| ca.rsync_base_uri.as_str());
|
||||
let _repo_total = self
|
||||
@@ -199,7 +215,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
} {
|
||||
Ok(res) => {
|
||||
if self.rsync_dedup && res.source == crate::sync::repo::RepoSyncSource::Rsync {
|
||||
let base = normalize_rsync_base_uri(&ca.rsync_base_uri);
|
||||
let base = self.rsync_fetcher.dedup_key(&ca.rsync_base_uri);
|
||||
self.rsync_repo_cache
|
||||
.lock()
|
||||
.expect("rsync_repo_cache lock")
|
||||
@@ -229,7 +245,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
}
|
||||
|
||||
warnings.extend(res.warnings);
|
||||
(true, None)
|
||||
(true, None, Some(repo_sync_source_label(res.source).to_string()))
|
||||
}
|
||||
Err(e) => {
|
||||
if attempted_rrdp && self.rrdp_dedup {
|
||||
@@ -247,10 +263,22 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
.with_rfc_refs(&[RfcRef("RFC 8182 §3.4.5"), RfcRef("RFC 9286 §6.6")])
|
||||
.with_context(&ca.rsync_base_uri),
|
||||
);
|
||||
(false, Some(e.to_string()))
|
||||
(false, Some(e.to_string()), None)
|
||||
}
|
||||
}
|
||||
};
|
||||
let repo_sync_duration_ms = repo_sync_started.elapsed().as_millis() as u64;
|
||||
crate::progress_log::emit(
|
||||
"publication_point_repo_sync_done",
|
||||
serde_json::json!({
|
||||
"manifest_rsync_uri": ca.manifest_rsync_uri,
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"repo_sync_ok": repo_sync_ok,
|
||||
"repo_sync_source": repo_sync_source,
|
||||
"repo_sync_error": repo_sync_err,
|
||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||
}),
|
||||
);
|
||||
|
||||
let fresh_publication_point = {
|
||||
let _manifest_total = self
|
||||
@@ -334,23 +362,71 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
let audit = build_publication_point_audit_from_snapshot(
|
||||
ca,
|
||||
PublicationPointSource::Fresh,
|
||||
repo_sync_source.as_deref(),
|
||||
Some(repo_sync_duration_ms),
|
||||
repo_sync_err.as_deref(),
|
||||
&pack,
|
||||
&warnings,
|
||||
&objects,
|
||||
&child_audits,
|
||||
);
|
||||
|
||||
Ok(PublicationPointRunResult {
|
||||
let result = PublicationPointRunResult {
|
||||
source: PublicationPointSource::Fresh,
|
||||
snapshot: Some(pack),
|
||||
warnings,
|
||||
objects,
|
||||
audit,
|
||||
discovered_children,
|
||||
})
|
||||
};
|
||||
let total_duration_ms = publication_point_started.elapsed().as_millis() as u64;
|
||||
crate::progress_log::emit(
|
||||
"publication_point_finish",
|
||||
serde_json::json!({
|
||||
"manifest_rsync_uri": ca.manifest_rsync_uri,
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"source": "fresh",
|
||||
"repo_sync_source": repo_sync_source,
|
||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||
"total_duration_ms": total_duration_ms,
|
||||
"warning_count": result.warnings.len(),
|
||||
"vrp_count": result.objects.vrps.len(),
|
||||
"vap_count": result.objects.aspas.len(),
|
||||
"router_key_count": result.objects.router_keys.len(),
|
||||
"child_count": result.discovered_children.len(),
|
||||
}),
|
||||
);
|
||||
if (total_duration_ms as f64) / 1000.0 >= crate::progress_log::slow_threshold_secs() {
|
||||
crate::progress_log::emit(
|
||||
"publication_point_slow",
|
||||
serde_json::json!({
|
||||
"manifest_rsync_uri": ca.manifest_rsync_uri,
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"source": "fresh",
|
||||
"repo_sync_source": repo_sync_source,
|
||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||
"total_duration_ms": total_duration_ms,
|
||||
}),
|
||||
);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
Err(fresh_err) => match self.policy.ca_failed_fetch_policy {
|
||||
crate::policy::CaFailedFetchPolicy::StopAllOutput => Err(format!("{fresh_err}")),
|
||||
crate::policy::CaFailedFetchPolicy::StopAllOutput => {
|
||||
let total_duration_ms = publication_point_started.elapsed().as_millis() as u64;
|
||||
crate::progress_log::emit(
|
||||
"publication_point_finish",
|
||||
serde_json::json!({
|
||||
"manifest_rsync_uri": ca.manifest_rsync_uri,
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"source": "error",
|
||||
"repo_sync_source": repo_sync_source,
|
||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||
"total_duration_ms": total_duration_ms,
|
||||
"error": fresh_err.to_string(),
|
||||
}),
|
||||
);
|
||||
Err(format!("{fresh_err}"))
|
||||
}
|
||||
crate::policy::CaFailedFetchPolicy::ReuseCurrentInstanceVcir => {
|
||||
let projection = project_current_instance_vcir_on_failed_fetch(
|
||||
self.store,
|
||||
@@ -363,34 +439,60 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
let audit = build_publication_point_audit_from_vcir(
|
||||
ca,
|
||||
projection.source,
|
||||
repo_sync_source.as_deref(),
|
||||
Some(repo_sync_duration_ms),
|
||||
repo_sync_err.as_deref(),
|
||||
projection.vcir.as_ref(),
|
||||
projection.snapshot.as_ref(),
|
||||
&warnings,
|
||||
&projection.objects,
|
||||
&projection.child_audits,
|
||||
);
|
||||
Ok(PublicationPointRunResult {
|
||||
let result = PublicationPointRunResult {
|
||||
source: projection.source,
|
||||
snapshot: projection.snapshot,
|
||||
warnings,
|
||||
objects: projection.objects,
|
||||
audit,
|
||||
discovered_children: projection.discovered_children,
|
||||
})
|
||||
};
|
||||
let total_duration_ms = publication_point_started.elapsed().as_millis() as u64;
|
||||
crate::progress_log::emit(
|
||||
"publication_point_finish",
|
||||
serde_json::json!({
|
||||
"manifest_rsync_uri": ca.manifest_rsync_uri,
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"source": source_label(result.source),
|
||||
"repo_sync_source": repo_sync_source,
|
||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||
"total_duration_ms": total_duration_ms,
|
||||
"warning_count": result.warnings.len(),
|
||||
"vrp_count": result.objects.vrps.len(),
|
||||
"vap_count": result.objects.aspas.len(),
|
||||
"router_key_count": result.objects.router_keys.len(),
|
||||
"child_count": result.discovered_children.len(),
|
||||
}),
|
||||
);
|
||||
if (total_duration_ms as f64) / 1000.0 >= crate::progress_log::slow_threshold_secs() {
|
||||
crate::progress_log::emit(
|
||||
"publication_point_slow",
|
||||
serde_json::json!({
|
||||
"manifest_rsync_uri": ca.manifest_rsync_uri,
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"source": source_label(result.source),
|
||||
"repo_sync_source": repo_sync_source,
|
||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||
"total_duration_ms": total_duration_ms,
|
||||
}),
|
||||
);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_rsync_base_uri(s: &str) -> String {
|
||||
if s.ends_with('/') {
|
||||
s.to_string()
|
||||
} else {
|
||||
format!("{s}/")
|
||||
}
|
||||
}
|
||||
|
||||
struct ChildDiscoveryOutput {
|
||||
children: Vec<DiscoveredChildCaInstance>,
|
||||
audits: Vec<ObjectAuditEntry>,
|
||||
@@ -1044,6 +1146,13 @@ fn source_label(source: PublicationPointSource) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
fn repo_sync_source_label(source: crate::sync::repo::RepoSyncSource) -> &'static str {
|
||||
match source {
|
||||
crate::sync::repo::RepoSyncSource::Rrdp => "rrdp",
|
||||
crate::sync::repo::RepoSyncSource::Rsync => "rsync",
|
||||
}
|
||||
}
|
||||
|
||||
fn kind_from_vcir_artifact_kind(kind: VcirArtifactKind) -> AuditObjectKind {
|
||||
match kind {
|
||||
VcirArtifactKind::Mft => AuditObjectKind::Manifest,
|
||||
@@ -1068,6 +1177,9 @@ fn audit_result_from_vcir_status(status: VcirArtifactValidationStatus) -> AuditO
|
||||
fn build_publication_point_audit_from_snapshot(
|
||||
ca: &CaInstanceHandle,
|
||||
source: PublicationPointSource,
|
||||
repo_sync_source: Option<&str>,
|
||||
repo_sync_duration_ms: Option<u64>,
|
||||
repo_sync_error: Option<&str>,
|
||||
pack: &PublicationPointSnapshot,
|
||||
runner_warnings: &[Warning],
|
||||
objects: &crate::validation::objects::ObjectsOutput,
|
||||
@@ -1158,6 +1270,9 @@ fn build_publication_point_audit_from_snapshot(
|
||||
publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(),
|
||||
rrdp_notification_uri: ca.rrdp_notification_uri.clone(),
|
||||
source: source_label(source),
|
||||
repo_sync_source: repo_sync_source.map(ToString::to_string),
|
||||
repo_sync_duration_ms,
|
||||
repo_sync_error: repo_sync_error.map(ToString::to_string),
|
||||
this_update_rfc3339_utc: pack.this_update.rfc3339_utc.clone(),
|
||||
next_update_rfc3339_utc: pack.next_update.rfc3339_utc.clone(),
|
||||
verified_at_rfc3339_utc: pack.verified_at.rfc3339_utc.clone(),
|
||||
@@ -1169,6 +1284,9 @@ fn build_publication_point_audit_from_snapshot(
|
||||
fn build_publication_point_audit_from_vcir(
|
||||
ca: &CaInstanceHandle,
|
||||
source: PublicationPointSource,
|
||||
repo_sync_source: Option<&str>,
|
||||
repo_sync_duration_ms: Option<u64>,
|
||||
repo_sync_error: Option<&str>,
|
||||
vcir: Option<&ValidatedCaInstanceResult>,
|
||||
pack: Option<&PublicationPointSnapshot>,
|
||||
runner_warnings: &[Warning],
|
||||
@@ -1179,6 +1297,9 @@ fn build_publication_point_audit_from_vcir(
|
||||
return build_publication_point_audit_from_snapshot(
|
||||
ca,
|
||||
source,
|
||||
repo_sync_source,
|
||||
repo_sync_duration_ms,
|
||||
repo_sync_error,
|
||||
pack,
|
||||
runner_warnings,
|
||||
objects,
|
||||
@@ -1200,6 +1321,9 @@ fn build_publication_point_audit_from_vcir(
|
||||
publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(),
|
||||
rrdp_notification_uri: ca.rrdp_notification_uri.clone(),
|
||||
source: source_label(source),
|
||||
repo_sync_source: repo_sync_source.map(ToString::to_string),
|
||||
repo_sync_duration_ms,
|
||||
repo_sync_error: repo_sync_error.map(ToString::to_string),
|
||||
this_update_rfc3339_utc: String::new(),
|
||||
next_update_rfc3339_utc: String::new(),
|
||||
verified_at_rfc3339_utc: String::new(),
|
||||
@@ -1278,6 +1402,9 @@ fn build_publication_point_audit_from_vcir(
|
||||
publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(),
|
||||
rrdp_notification_uri: ca.rrdp_notification_uri.clone(),
|
||||
source: source_label(source),
|
||||
repo_sync_source: repo_sync_source.map(ToString::to_string),
|
||||
repo_sync_duration_ms,
|
||||
repo_sync_error: repo_sync_error.map(ToString::to_string),
|
||||
this_update_rfc3339_utc: vcir
|
||||
.validated_manifest_meta
|
||||
.validated_manifest_this_update
|
||||
@@ -3896,6 +4023,112 @@ authorityKeyIdentifier = keyid:always
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runner_rsync_dedup_skips_second_sync_for_same_module_scope() {
|
||||
let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0");
|
||||
assert!(fixture_dir.is_dir(), "fixture directory must exist");
|
||||
|
||||
let first_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string();
|
||||
let second_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/sub/".to_string();
|
||||
let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft";
|
||||
let manifest_rsync_uri = format!("{first_base_uri}{manifest_file}");
|
||||
|
||||
let fixture_manifest_bytes =
|
||||
std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture");
|
||||
let fixture_manifest =
|
||||
crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes)
|
||||
.expect("decode manifest fixture");
|
||||
let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60);
|
||||
|
||||
let store_dir = tempfile::tempdir().expect("store dir");
|
||||
let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
|
||||
let policy = Policy {
|
||||
sync_preference: crate::policy::SyncPreference::RsyncOnly,
|
||||
..Policy::default()
|
||||
};
|
||||
|
||||
let issuer_ca_der = std::fs::read(
|
||||
std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(
|
||||
"tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer",
|
||||
),
|
||||
)
|
||||
.expect("read issuer ca fixture");
|
||||
let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca");
|
||||
|
||||
let handle = CaInstanceHandle {
|
||||
depth: 0,
|
||||
tal_id: "test-tal".to_string(),
|
||||
parent_manifest_rsync_uri: None,
|
||||
ca_certificate_der: issuer_ca_der,
|
||||
ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()),
|
||||
effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(),
|
||||
effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(),
|
||||
rsync_base_uri: first_base_uri.clone(),
|
||||
manifest_rsync_uri: manifest_rsync_uri.clone(),
|
||||
publication_point_rsync_uri: first_base_uri.clone(),
|
||||
rrdp_notification_uri: None,
|
||||
};
|
||||
let second_handle = CaInstanceHandle {
|
||||
rsync_base_uri: second_base_uri.clone(),
|
||||
publication_point_rsync_uri: second_base_uri.clone(),
|
||||
..handle.clone()
|
||||
};
|
||||
|
||||
struct ModuleScopeRsyncFetcher {
|
||||
inner: LocalDirRsyncFetcher,
|
||||
calls: Arc<AtomicUsize>,
|
||||
}
|
||||
impl RsyncFetcher for ModuleScopeRsyncFetcher {
|
||||
fn fetch_objects(
|
||||
&self,
|
||||
rsync_base_uri: &str,
|
||||
) -> Result<Vec<(String, Vec<u8>)>, RsyncFetchError> {
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
self.inner.fetch_objects(rsync_base_uri)
|
||||
}
|
||||
|
||||
fn dedup_key(&self, _rsync_base_uri: &str) -> String {
|
||||
"rsync://rpki.cernet.net/repo/".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let rsync = ModuleScopeRsyncFetcher {
|
||||
inner: LocalDirRsyncFetcher::new(&fixture_dir),
|
||||
calls: calls.clone(),
|
||||
};
|
||||
|
||||
let runner = Rpkiv1PublicationPointRunner {
|
||||
store: &store,
|
||||
policy: &policy,
|
||||
http_fetcher: &NeverHttpFetcher,
|
||||
rsync_fetcher: &rsync,
|
||||
validation_time,
|
||||
timing: None,
|
||||
download_log: None,
|
||||
replay_archive_index: None,
|
||||
replay_delta_index: None,
|
||||
rrdp_dedup: false,
|
||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||
rsync_dedup: true,
|
||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||
};
|
||||
|
||||
let first = runner.run_publication_point(&handle).expect("first run ok");
|
||||
assert_eq!(first.source, PublicationPointSource::Fresh);
|
||||
|
||||
let second = runner
|
||||
.run_publication_point(&second_handle)
|
||||
.expect("second run ok");
|
||||
assert!(matches!(
|
||||
second.source,
|
||||
PublicationPointSource::Fresh | PublicationPointSource::VcirCurrentInstance
|
||||
));
|
||||
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1, "module-scope dedup should skip second sync");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runner_when_repo_sync_fails_uses_current_instance_vcir_and_keeps_children_empty_for_fixture()
|
||||
{
|
||||
@@ -4035,6 +4268,9 @@ authorityKeyIdentifier = keyid:always
|
||||
let audit = build_publication_point_audit_from_snapshot(
|
||||
&ca,
|
||||
pp.source,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
&pp.snapshot,
|
||||
&[],
|
||||
&objects,
|
||||
@@ -4103,6 +4339,9 @@ authorityKeyIdentifier = keyid:always
|
||||
let audit = build_publication_point_audit_from_snapshot(
|
||||
&issuer,
|
||||
pp.source,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
&pp.snapshot,
|
||||
&[],
|
||||
&objects,
|
||||
@@ -4857,6 +5096,9 @@ authorityKeyIdentifier = keyid:always
|
||||
let audit = build_publication_point_audit_from_vcir(
|
||||
&ca,
|
||||
PublicationPointSource::VcirCurrentInstance,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some(&vcir),
|
||||
None,
|
||||
&runner_warnings,
|
||||
@@ -4916,6 +5158,9 @@ authorityKeyIdentifier = keyid:always
|
||||
PublicationPointSource::FailedFetchNoCache,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
&[Warning::new("runner warning")],
|
||||
&crate::validation::objects::ObjectsOutput {
|
||||
vrps: Vec::new(),
|
||||
@@ -5155,7 +5400,7 @@ authorityKeyIdentifier = keyid:always
|
||||
.expect("rsync dedup run");
|
||||
assert_eq!(fourth.source, PublicationPointSource::Fresh);
|
||||
assert_eq!(
|
||||
normalize_rsync_base_uri("rsync://example.test/repo"),
|
||||
crate::fetch::rsync::normalize_rsync_base_uri("rsync://example.test/repo"),
|
||||
"rsync://example.test/repo/"
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user