NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260427 拆分独立repo-bytes.db数据库文件，cir materialize 移除旧兼容

Browse Source
This commit is contained in:
yuyr
2026-04-27 16:01:33 +08:00
parent 944ea6ca00
commit eaa375c5ec
53 changed files with 950 additions and 595 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/validation/manifest.rs
View File

@@ -4,7 +4,7 @@ use crate::data_model::signed_object::SignedObjectVerifyError;
use crate::policy::{CaFailedFetchPolicy, Policy};
use crate::report::{RfcRef, Warning};
use crate::storage::{PackFile, PackTime, RocksStore, StorageError, VcirArtifactRole};
use crate::validation::cert_path::{validate_signed_object_ee_cert_path_fast, CertPathError};
use crate::validation::cert_path::{CertPathError, validate_signed_object_ee_cert_path_fast};
use crate::validation::publication_point::PublicationPointSnapshot;
use sha2::Digest;
use std::cmp::Ordering;
@@ -398,10 +398,11 @@ pub fn process_manifest_publication_point_after_repo_sync(
Err(ManifestProcessError::StopAllOutput(fresh_err))
}
CaFailedFetchPolicy::ReuseCurrentInstanceVcir => {
let mut warnings =
vec![Warning::new(format!("manifest failed fetch: {fresh_err}"))
let mut warnings = vec![
Warning::new(format!("manifest failed fetch: {fresh_err}"))
.with_rfc_refs(&[RfcRef("RFC 9286 §6.6")])
.with_context(manifest_rsync_uri)];
.with_context(manifest_rsync_uri),
];
match load_current_instance_vcir_publication_point(
store,
@@ -740,6 +741,10 @@ pub(crate) fn try_build_fresh_publication_point_with_timing(
.external_raw_store_ref()
.cloned()
.map(std::sync::Arc::new);
let external_repo_bytes = store
.external_repo_bytes_ref()
.cloned()
.map(std::sync::Arc::new);
for entry in &entries {
let rsync_uri =
join_rsync_dir_and_file(publication_point_rsync_uri, entry.file_name.as_str());
@@ -770,7 +775,16 @@ pub(crate) fn try_build_fresh_publication_point_with_timing(
return Err(ManifestFreshError::HashMismatch { rsync_uri });
}
if let (Some(_), Some(raw_store)) =
if let (Some(_), Some(repo_bytes)) =
(current_index_guard.as_ref(), external_repo_bytes.as_ref())
{
files.push(PackFile::from_lazy_repo_bytes(
rsync_uri,
current_object.current_hash_hex,
current_object.current_hash,
repo_bytes.clone(),
));
} else if let (Some(_), Some(raw_store)) =
(current_index_guard.as_ref(), external_raw_store.as_ref())
{
files.push(PackFile::from_lazy_external_raw_store(
@@ -1327,10 +1341,12 @@ mod tests {
.apply_repository_view_entries(&entries)
.expect("apply current index");
assert!(store
.get_repository_view_entry(&manifest_rsync_uri)
.expect("get repository view")
.is_none());
assert!(
store
.get_repository_view_entry(&manifest_rsync_uri)
.expect("get repository view")
.is_none()
);
let (fresh, _timing) = try_build_fresh_publication_point_with_timing(
&store,

4
src/validation/objects.rs
View File

@@ -1,5 +1,5 @@
use crate::analysis::timing::TimingHandle;
use crate::audit::{sha256_hex_from_32, AuditObjectKind, AuditObjectResult, ObjectAuditEntry};
use crate::audit::{AuditObjectKind, AuditObjectResult, ObjectAuditEntry, sha256_hex_from_32};
use crate::data_model::aspa::{AspaDecodeError, AspaObject, AspaValidateError};
use crate::data_model::manifest::ManifestObject;
use crate::data_model::rc::{
@@ -15,7 +15,7 @@ use crate::parallel::object_worker::{
use crate::policy::{Policy, SignedObjectFailurePolicy};
use crate::report::{RfcRef, Warning};
use crate::storage::{PackFile, PackTime, VcirLocalOutput, VcirOutputType};
use crate::validation::cert_path::{validate_signed_object_ee_cert_path_fast, CertPathError};
use crate::validation::cert_path::{CertPathError, validate_signed_object_ee_cert_path_fast};
use crate::validation::manifest::PublicationPointData;
use crate::validation::publication_point::PublicationPointSnapshot;
use std::sync::{Arc, Mutex};

8
src/validation/run_tree_from_tal.rs
View File

@@ -23,13 +23,13 @@ use crate::replay::fetch_http::PayloadReplayHttpFetcher;
use crate::replay::fetch_rsync::PayloadReplayRsyncFetcher;
use crate::sync::rrdp::Fetcher;
use crate::validation::from_tal::{
discover_root_ca_instance_from_tal_and_ta_der, discover_root_ca_instance_from_tal_url,
discover_root_ca_instance_from_tal_with_fetchers, DiscoveredRootCaInstance, FromTalError,
DiscoveredRootCaInstance, FromTalError, discover_root_ca_instance_from_tal_and_ta_der,
discover_root_ca_instance_from_tal_url, discover_root_ca_instance_from_tal_with_fetchers,
};
use crate::validation::objects::ParallelRoaWorkerPool;
use crate::validation::tree::{
run_tree_serial, run_tree_serial_audit, run_tree_serial_audit_multi_root, CaInstanceHandle,
TreeRunAuditOutput, TreeRunConfig, TreeRunError, TreeRunOutput,
CaInstanceHandle, TreeRunAuditOutput, TreeRunConfig, TreeRunError, TreeRunOutput,
run_tree_serial, run_tree_serial_audit, run_tree_serial_audit_multi_root,
};
use crate::validation::tree_parallel::{
run_tree_parallel_phase2_audit, run_tree_parallel_phase2_audit_multi_root,

10
src/validation/tree_parallel.rs
View File

@@ -8,13 +8,13 @@ use crate::parallel::types::RepoIdentity;
use crate::policy::SignedObjectFailurePolicy;
use crate::report::Warning;
use crate::validation::objects::{
prepare_publication_point_for_parallel_roa, reduce_parallel_roa_stage, ObjectsOutput,
OwnedRoaTask, ParallelObjectsPrepare, ParallelObjectsStage,
ObjectsOutput, OwnedRoaTask, ParallelObjectsPrepare, ParallelObjectsStage,
prepare_publication_point_for_parallel_roa, reduce_parallel_roa_stage,
};
use crate::validation::tree::{
run_tree_serial_audit_multi_root, CaInstanceHandle, DiscoveredChildCaInstance,
PublicationPointRunResult, PublicationPointRunner, TreeRunAuditOutput, TreeRunConfig,
TreeRunError, TreeRunOutput,
CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult, PublicationPointRunner,
TreeRunAuditOutput, TreeRunConfig, TreeRunError, TreeRunOutput,
run_tree_serial_audit_multi_root,
};
use crate::validation::tree_runner::{FreshPublicationPointStage, Rpkiv1PublicationPointRunner};

182
src/validation/tree_runner.rs
View File

@@ -1,7 +1,7 @@
use crate::analysis::timing::TimingHandle;
use crate::audit::{
sha256_hex, sha256_hex_from_32, AuditObjectKind, AuditObjectResult, AuditWarning,
ObjectAuditEntry, PublicationPointAudit,
AuditObjectKind, AuditObjectResult, AuditWarning, ObjectAuditEntry, PublicationPointAudit,
sha256_hex, sha256_hex_from_32,
};
use crate::audit_downloads::DownloadLogHandle;
use crate::ccr::CcrAccumulator;
@@ -34,18 +34,18 @@ use crate::sync::repo::{
use crate::sync::rrdp::Fetcher;
use crate::validation::ca_instance::ca_instance_uris_from_ca_certificate;
use crate::validation::ca_path::{
validate_subordinate_ca_cert_with_prevalidated_issuer_and_resources, CaPathError,
IssuerEffectiveResourcesIndex, ValidatedSubordinateCaLite,
CaPathError, IssuerEffectiveResourcesIndex, ValidatedSubordinateCaLite,
validate_subordinate_ca_cert_with_prevalidated_issuer_and_resources,
};
use crate::validation::manifest::{
process_manifest_publication_point_fresh_after_repo_sync_with_timing,
FreshPublicationPointTimingBreakdown, FreshValidatedPublicationPoint, ManifestFreshError,
PublicationPointData, PublicationPointSource,
process_manifest_publication_point_fresh_after_repo_sync_with_timing,
};
use crate::validation::objects::{
AspaAttestation, ParallelRoaWorkerPool, RouterKeyPayload, Vrp,
process_publication_point_for_issuer, process_publication_point_for_issuer_parallel_roa,
process_publication_point_for_issuer_parallel_roa_with_pool, AspaAttestation,
ParallelRoaWorkerPool, RouterKeyPayload, Vrp,
process_publication_point_for_issuer_parallel_roa_with_pool,
};
use crate::validation::publication_point::PublicationPointSnapshot;
use crate::validation::tree::{
@@ -228,9 +228,11 @@ impl<'a> Rpkiv1PublicationPointRunner<'a> {
Vec::new(),
Vec::new(),
Vec::new(),
vec![Warning::new(format!("child CA discovery failed: {e}"))
.with_rfc_refs(&[RfcRef("RFC 6487 §7.2")])
.with_context(&ca.manifest_rsync_uri)],
vec![
Warning::new(format!("child CA discovery failed: {e}"))
.with_rfc_refs(&[RfcRef("RFC 6487 §7.2")])
.with_context(&ca.manifest_rsync_uri),
],
),
};
let child_discovery_ms = child_discovery_started.elapsed().as_millis() as u64;
@@ -1946,9 +1948,11 @@ fn project_current_instance_vcir_on_failed_fetch(
fresh_err: &ManifestFreshError,
validation_time: time::OffsetDateTime,
) -> Result<VcirReuseProjection, String> {
let mut warnings = vec![Warning::new(format!("manifest failed fetch: {fresh_err}"))
.with_rfc_refs(&[RfcRef("RFC 9286 §6.6")])
.with_context(&ca.manifest_rsync_uri)];
let mut warnings = vec![
Warning::new(format!("manifest failed fetch: {fresh_err}"))
.with_rfc_refs(&[RfcRef("RFC 9286 §6.6")])
.with_context(&ca.manifest_rsync_uri),
];
let Some(vcir) = store
.get_vcir(&ca.manifest_rsync_uri)
@@ -3236,8 +3240,8 @@ mod tests {
use crate::validation::tree::PublicationPointRunner;
use std::process::Command;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
struct NeverHttpFetcher;
impl Fetcher for NeverHttpFetcher {
@@ -3666,8 +3670,8 @@ authorityKeyIdentifier = keyid:always
}
}
fn cernet_publication_point_snapshot_for_vcir_tests(
) -> (PublicationPointSnapshot, Vec<u8>, time::OffsetDateTime) {
fn cernet_publication_point_snapshot_for_vcir_tests()
-> (PublicationPointSnapshot, Vec<u8>, time::OffsetDateTime) {
let dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0");
let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/";
@@ -4013,18 +4017,22 @@ authorityKeyIdentifier = keyid:always
.get_raw_by_hash_entry(&issuer_hash)
.expect("load issuer raw entry")
.expect("issuer raw entry present");
assert!(issuer_entry
.origin_uris
.iter()
.any(|uri| uri.ends_with("BfycW4hQb3wNP4YsiJW-1n6fjro.cer")));
assert!(
issuer_entry
.origin_uris
.iter()
.any(|uri| uri.ends_with("BfycW4hQb3wNP4YsiJW-1n6fjro.cer"))
);
let first_output = objects
.local_outputs_cache
.first()
.expect("first local output");
assert!(store
.get_raw_by_hash_entry(&first_output.source_ee_cert_hash)
.expect("load source ee raw")
.is_none());
assert!(
store
.get_raw_by_hash_entry(&first_output.source_ee_cert_hash)
.expect("load source ee raw")
.is_none()
);
}
#[test]
@@ -4100,9 +4108,11 @@ authorityKeyIdentifier = keyid:always
.expect("rebuild vcir local outputs");
assert!(!local_outputs.is_empty());
assert_eq!(local_outputs.len(), objects.vrps.len());
assert!(local_outputs
.iter()
.all(|output| output.output_type == VcirOutputType::Vrp));
assert!(
local_outputs
.iter()
.all(|output| output.output_type == VcirOutputType::Vrp)
);
}
#[test]
@@ -4275,10 +4285,15 @@ authorityKeyIdentifier = keyid:always
pack.manifest_bytes.len() as u64
);
let first_output = vcir.local_outputs.first().expect("local outputs stored");
assert!(store
.get_audit_rule_index_entry(crate::storage::AuditRuleKind::Roa, &first_output.rule_hash)
.expect("get audit rule index entry")
.is_some());
assert!(
store
.get_audit_rule_index_entry(
crate::storage::AuditRuleKind::Roa,
&first_output.rule_hash
)
.expect("get audit rule index entry")
.is_some()
);
}
#[test]
@@ -4454,12 +4469,16 @@ authorityKeyIdentifier = keyid:always
&objects,
&[],
);
assert!(artifacts
.iter()
.any(|artifact| artifact.artifact_role == VcirArtifactRole::Manifest));
assert!(artifacts
.iter()
.any(|artifact| artifact.artifact_role == VcirArtifactRole::TrustAnchorCert));
assert!(
artifacts
.iter()
.any(|artifact| artifact.artifact_role == VcirArtifactRole::Manifest)
);
assert!(
artifacts
.iter()
.any(|artifact| artifact.artifact_role == VcirArtifactRole::TrustAnchorCert)
);
assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref()
== Some("rsync://example.test/repo/issuer/issuer.crl")
&& artifact.artifact_role == VcirArtifactRole::CurrentCrl));
@@ -5402,11 +5421,13 @@ authorityKeyIdentifier = keyid:always
assert!(out.children.is_empty());
assert_eq!(out.audits.len(), 1);
assert!(matches!(out.audits[0].result, AuditObjectResult::Ok));
assert!(out.audits[0]
.detail
.as_deref()
.unwrap_or("")
.contains("validated BGPsec router certificate"));
assert!(
out.audits[0]
.detail
.as_deref()
.unwrap_or("")
.contains("validated BGPsec router certificate")
);
}
#[test]
@@ -5450,11 +5471,13 @@ authorityKeyIdentifier = keyid:always
assert!(out.children.is_empty());
assert_eq!(out.audits.len(), 1);
assert!(matches!(out.audits[0].result, AuditObjectResult::Skipped));
assert!(out.audits[0]
.detail
.as_deref()
.unwrap_or("")
.contains("not a CA resource certificate or BGPsec router certificate"));
assert!(
out.audits[0]
.detail
.as_deref()
.unwrap_or("")
.contains("not a CA resource certificate or BGPsec router certificate")
);
}
#[test]
@@ -5498,11 +5521,13 @@ authorityKeyIdentifier = keyid:always
assert!(out.children.is_empty());
assert_eq!(out.audits.len(), 1);
assert!(matches!(out.audits[0].result, AuditObjectResult::Error));
assert!(out.audits[0]
.detail
.as_deref()
.unwrap_or("")
.contains("router certificate validation failed"));
assert!(
out.audits[0]
.detail
.as_deref()
.unwrap_or("")
.contains("router certificate validation failed")
);
}
#[test]
@@ -5637,11 +5662,13 @@ authorityKeyIdentifier = keyid:always
assert!(out.children.is_empty());
assert_eq!(out.audits.len(), 1);
assert!(matches!(out.audits[0].result, AuditObjectResult::Error));
assert!(out.audits[0]
.detail
.as_deref()
.unwrap_or("")
.contains("issuer CA decode failed"));
assert!(
out.audits[0]
.detail
.as_deref()
.unwrap_or("")
.contains("issuer CA decode failed")
);
}
#[test]
@@ -6187,8 +6214,8 @@ authorityKeyIdentifier = keyid:always
}
#[test]
fn build_publication_point_audit_from_vcir_uses_vcir_metadata_and_overlays_child_and_object_audits(
) {
fn build_publication_point_audit_from_vcir_uses_vcir_metadata_and_overlays_child_and_object_audits()
{
let now = time::OffsetDateTime::now_utc();
let child_cert_hash = sha256_hex(b"child-cert");
let mut vcir = sample_vcir_for_projection(now, &child_cert_hash);
@@ -6434,22 +6461,27 @@ authorityKeyIdentifier = keyid:always
.expect("reconstruct pack with partial related artifacts");
assert_eq!(pack.manifest_bytes, manifest_bytes);
assert_eq!(pack.files.len(), 3, "crl + child cert + roa only");
assert!(pack
.files
.iter()
.any(|file| file.rsync_uri.ends_with("issuer.crl")));
assert!(pack
.files
.iter()
.any(|file| file.rsync_uri.ends_with("child.cer")));
assert!(pack
.files
.iter()
.any(|file| file.rsync_uri.ends_with("a.roa")));
assert!(!pack
.files
.iter()
.any(|file| file.rsync_uri.ends_with("issuer.cer")));
assert!(
pack.files
.iter()
.any(|file| file.rsync_uri.ends_with("issuer.crl"))
);
assert!(
pack.files
.iter()
.any(|file| file.rsync_uri.ends_with("child.cer"))
);
assert!(
pack.files
.iter()
.any(|file| file.rsync_uri.ends_with("a.roa"))
);
assert!(
!pack
.files
.iter()
.any(|file| file.rsync_uri.ends_with("issuer.cer"))
);
assert!(warnings.iter().any(|warning| {
warning
.message