NASP/rpki
63
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260420_2 完成输出report和ccr优化，snapshot耗时优化到70多秒

Browse Source
This commit is contained in:
yuyr 2026-04-20 18:28:59 +08:00
parent f6a601e16c
commit 542bd7be80
21 changed files with 2111 additions and 344 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
scripts/compare/run_perf_compare_quick_remote.sh
View File

@ -10,6 +10,7 @@ Usage:
[--ssh-target <user@host>] \
[--rpki-client-bin <path>] \
[--libtls-path <path>] \
[--rp-run-mode <serial|parallel>] \
[--ours-extra-args '<args>'] \
[--dry-run]
EOF
@ -21,6 +22,7 @@ REMOTE_ROOT=""
SSH_TARGET="${SSH_TARGET:-root@47.251.56.108}"
RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}"
LIBTLS_PATH="${LIBTLS_PATH:-/home/yuyr/dev/rpki-client-9.7/.deps/libtls/root/usr/lib/x86_64-linux-gnu/libtls.so.28.0.0}"
RP_RUN_MODE="${RP_RUN_MODE:-serial}"
OURS_EXTRA_ARGS="${OURS_EXTRA_ARGS:-}"
DRY_RUN=0
@ -31,6 +33,7 @@ while [[ $# -gt 0 ]]; do
--ssh-target) SSH_TARGET="$2"; shift 2 ;;
--rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
--libtls-path) LIBTLS_PATH="$2"; shift 2 ;;
--rp-run-mode) RP_RUN_MODE="$2"; shift 2 ;;
--ours-extra-args) OURS_EXTRA_ARGS="$2"; shift 2 ;;
--dry-run) DRY_RUN=1; shift ;;
-h|--help) usage; exit 0 ;;
@ -39,6 +42,7 @@ while [[ $# -gt 0 ]]; do
done
[[ -n "$RUN_ROOT" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
[[ "$RP_RUN_MODE" == "serial" || "$RP_RUN_MODE" == "parallel" ]] || { echo "invalid --rp-run-mode: $RP_RUN_MODE" >&2; usage; exit 2; }
[[ "$DRY_RUN" -eq 1 || -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
[[ "$DRY_RUN" -eq 1 || -f "$LIBTLS_PATH" ]] || { echo "libtls not found: $LIBTLS_PATH" >&2; exit 2; }
@ -64,6 +68,7 @@ scope=APNIC+ARIN mixed release two-step synchronized compare
run_root=$RUN_ROOT
remote_root=$REMOTE_ROOT
ssh_target=$SSH_TARGET
rp_run_mode=$RP_RUN_MODE
ours_extra_args=$OURS_EXTRA_ARGS
EOF2
exit 0
@ -76,12 +81,10 @@ cleanup_remote() {
}
trap cleanup_remote EXIT
if [[ ! -x "$ROOT_DIR/target/release/rpki" || ! -x "$ROOT_DIR/target/release/ccr_to_compare_views" ]]; then
(
cd "$ROOT_DIR"
cargo build --release --bin rpki --bin ccr_to_compare_views
)
fi
(
cd "$ROOT_DIR"
cargo build --release --bin rpki --bin ccr_to_compare_views
)
ssh "$SSH_TARGET" "set -e; id -u _rpki-client >/dev/null 2>&1 || useradd -r -M -s /usr/sbin/nologin _rpki-client || true; rm -rf '$REMOTE_ROOT'; mkdir -p '$REMOTE_ROOT/bin' '$REMOTE_ROOT/lib' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client' '$REMOTE_ROOT/steps/step-001/ours' '$REMOTE_ROOT/steps/step-001/rpki-client' '$REMOTE_ROOT/steps/step-002/ours' '$REMOTE_ROOT/steps/step-002/rpki-client'"
scp "$ROOT_DIR/target/release/rpki" "$APNIC_TAL" "$APNIC_TA" "$ARIN_TAL" "$ARIN_TA" "$SSH_TARGET:$REMOTE_ROOT/"
@ -93,12 +96,13 @@ run_step() {
local kind="$2"
local local_step="$RUN_ROOT/steps/$step_id"
ssh "$SSH_TARGET" bash -s -- "$REMOTE_ROOT" "$step_id" "$kind" "$OURS_EXTRA_ARGS" <<'EOS'
ssh "$SSH_TARGET" bash -s -- "$REMOTE_ROOT" "$step_id" "$kind" "$OURS_EXTRA_ARGS" "$RP_RUN_MODE" <<'EOS'
set -euo pipefail
REMOTE_ROOT="$1"
STEP_ID="$2"
KIND="$3"
OURS_EXTRA_ARGS="$4"
RP_RUN_MODE="$5"
cd "$REMOTE_ROOT"
mkdir -p "steps/$STEP_ID/ours" "steps/$STEP_ID/rpki-client"
@ -121,7 +125,7 @@ print(time.time() + 3.0)
PY
)"
(
run_ours() {
python3 - <<'PY' "$START_EPOCH"
import sys, time
x = float(sys.argv[1])
@ -166,10 +170,9 @@ json.dump(
indent=2,
)
PY
) &
OURS_PID=$!
}
(
run_client() {
cd state/rpki-client
python3 - <<'PY' "$START_EPOCH"
import sys, time
@ -213,11 +216,19 @@ json.dump(
indent=2,
)
PY
) &
CLIENT_PID=$!
}
wait "$OURS_PID"
wait "$CLIENT_PID"
if [[ "$RP_RUN_MODE" == "parallel" ]]; then
run_ours &
OURS_PID=$!
run_client &
CLIENT_PID=$!
wait "$OURS_PID"
wait "$CLIENT_PID"
else
run_ours
run_client
fi
EOS
for rel in result.ccr round-result.json run.log stage-timing.json; do
@ -275,13 +286,14 @@ PY
run_step step-001 snapshot
run_step step-002 delta
python3 - <<'PY' "$RUN_ROOT/steps/step-001/step-summary.json" "$RUN_ROOT/steps/step-002/step-summary.json" "$RUN_ROOT/summary.json" "$OURS_EXTRA_ARGS"
python3 - <<'PY' "$RUN_ROOT/steps/step-001/step-summary.json" "$RUN_ROOT/steps/step-002/step-summary.json" "$RUN_ROOT/summary.json" "$RP_RUN_MODE" "$OURS_EXTRA_ARGS"
import json, sys
steps = [json.load(open(p)) for p in sys.argv[1:3]]
summary = {
"workflowName": "性能对比测试快速版",
"scope": "APNIC+ARIN mixed release two-step synchronized compare",
"oursExtraArgs": sys.argv[4],
"rpRunMode": sys.argv[4],
"oursExtraArgs": sys.argv[5],
"steps": steps,
}
json.dump(summary, open(sys.argv[3], "w"), indent=2, ensure_ascii=False)

32
src/audit_trace.rs
View File

@ -342,8 +342,8 @@ mod tests {
use crate::audit::sha256_hex;
use crate::data_model::roa::RoaObject;
use crate::storage::{
PackTime, ValidatedManifestMeta, VcirAuditSummary, VcirChildEntry, VcirInstanceGate,
VcirRelatedArtifact, VcirSummary,
PackTime, ValidatedManifestMeta, VcirAuditSummary, VcirCcrManifestProjection,
VcirChildEntry, VcirInstanceGate, VcirRelatedArtifact, VcirSummary,
};
use base64::Engine as _;
@ -357,6 +357,19 @@ mod tests {
let now = time::OffsetDateTime::now_utc();
let next = PackTime::from_utc_offset_datetime(now + time::Duration::hours(1));
let local_outputs: Vec<VcirLocalOutput> = local_output.into_iter().collect();
let ccr_manifest_projection = VcirCcrManifestProjection {
manifest_rsync_uri: manifest_rsync_uri.to_string(),
manifest_sha256: vec![0x44; 32],
manifest_size: 2048,
manifest_ee_aki: vec![0x55; 20],
manifest_number_be: vec![1],
manifest_this_update: PackTime::from_utc_offset_datetime(now),
manifest_sia_locations_der: vec![vec![
0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05,
b'r', b's', b'y', b'n', b'c',
]],
subordinate_skis: vec![vec![0x33; 20]],
};
ValidatedCaInstanceResult {
manifest_rsync_uri: manifest_rsync_uri.to_string(),
parent_manifest_rsync_uri: parent_manifest_rsync_uri.map(str::to_string),
@ -372,6 +385,7 @@ mod tests {
validated_manifest_this_update: PackTime::from_utc_offset_datetime(now),
validated_manifest_next_update: next.clone(),
},
ccr_manifest_projection,
instance_gate: VcirInstanceGate {
manifest_next_update: next.clone(),
current_crl_next_update: next.clone(),
@ -569,14 +583,12 @@ mod tests {
);
assert!(trace.source_object_raw.raw_present);
assert!(trace.source_ee_cert_raw.raw_present);
assert!(
trace.chain_leaf_to_root[0]
.related_artifacts
.iter()
.any(|artifact| {
artifact.uri.as_deref() == Some(leaf_manifest) && artifact.raw.raw_present
})
);
assert!(trace.chain_leaf_to_root[0]
.related_artifacts
.iter()
.any(|artifact| {
artifact.uri.as_deref() == Some(leaf_manifest) && artifact.raw.raw_present
}));
}
#[test]

415
src/ccr/accumulator.rs Normal file
View File

@ -0,0 +1,415 @@
use std::collections::BTreeMap;
use crate::ccr::build::{
build_aspa_payload_state, build_roa_payload_state, build_router_key_state_from_runtime,
build_trust_anchor_state,
};
use crate::ccr::encode::encode_manifest_state_payload_der;
use crate::ccr::hash::compute_state_hash;
use crate::ccr::model::{
CcrDigestAlgorithm, ManifestInstance, ManifestState, RpkiCanonicalCacheRepresentation,
};
use crate::data_model::common::BigUnsigned;
use crate::data_model::ta::TrustAnchor;
use crate::storage::VcirCcrManifestProjection;
use crate::validation::objects::{AspaAttestation, RouterKeyPayload, Vrp};
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CcrManifestContribution {
pub manifest_rsync_uri: String,
pub hash: Vec<u8>,
pub size: u64,
pub aki: Vec<u8>,
pub manifest_number_be: Vec<u8>,
pub this_update: time::OffsetDateTime,
pub locations_der: Vec<Vec<u8>>,
pub subordinate_skis: Vec<Vec<u8>>,
}
impl CcrManifestContribution {
fn from_projection(projection: &VcirCcrManifestProjection) -> Result<Self, String> {
let this_update = projection
.manifest_this_update
.parse()
.map_err(|e| format!("parse projection manifest_this_update failed: {e}"))?;
Ok(Self {
manifest_rsync_uri: projection.manifest_rsync_uri.clone(),
hash: projection.manifest_sha256.clone(),
size: projection.manifest_size,
aki: projection.manifest_ee_aki.clone(),
manifest_number_be: projection.manifest_number_be.clone(),
this_update,
locations_der: projection.manifest_sia_locations_der.clone(),
subordinate_skis: projection.subordinate_skis.clone(),
})
}
fn to_manifest_instance(&self) -> ManifestInstance {
ManifestInstance {
hash: self.hash.clone(),
size: self.size,
aki: self.aki.clone(),
manifest_number: BigUnsigned {
bytes_be: self.manifest_number_be.clone(),
},
this_update: self.this_update,
locations: self.locations_der.clone(),
subordinates: self.subordinate_skis.clone(),
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CcrAccumulator {
trust_anchors: Vec<TrustAnchor>,
manifests_by_hash: BTreeMap<Vec<u8>, CcrManifestContribution>,
most_recent_update: time::OffsetDateTime,
}
impl CcrAccumulator {
pub fn new(trust_anchors: Vec<TrustAnchor>) -> Self {
Self {
trust_anchors,
manifests_by_hash: BTreeMap::new(),
most_recent_update: time::OffsetDateTime::UNIX_EPOCH,
}
}
pub fn append_manifest_projection(
&mut self,
projection: &VcirCcrManifestProjection,
) -> Result<(), String> {
let contribution = CcrManifestContribution::from_projection(projection)?;
match self.manifests_by_hash.get(contribution.hash.as_slice()) {
Some(existing) if existing != &contribution => {
return Err(format!(
"duplicate manifest hash with conflicting content for URI: {}",
contribution.manifest_rsync_uri
));
}
Some(_) => {}
None => {
self.manifests_by_hash
.insert(contribution.hash.clone(), contribution.clone());
}
}
if contribution.this_update > self.most_recent_update {
self.most_recent_update = contribution.this_update;
}
Ok(())
}
pub fn finish(
&self,
produced_at: time::OffsetDateTime,
vrps: &[Vrp],
aspas: &[AspaAttestation],
router_keys: &[RouterKeyPayload],
) -> Result<RpkiCanonicalCacheRepresentation, String> {
let manifest_instances = self
.manifests_by_hash
.values()
.map(CcrManifestContribution::to_manifest_instance)
.collect::<Vec<_>>();
let manifest_payload_der = encode_manifest_state_payload_der(&manifest_instances)
.map_err(|e| format!("manifest state encoding failed: {e}"))?;
let manifest_state = ManifestState {
mis: manifest_instances,
most_recent_update: self.most_recent_update,
hash: compute_state_hash(&manifest_payload_der),
};
let vrp_state = build_roa_payload_state(vrps).map_err(|e| e.to_string())?;
let aspa_state = build_aspa_payload_state(aspas).map_err(|e| e.to_string())?;
let ta_state = build_trust_anchor_state(&self.trust_anchors).map_err(|e| e.to_string())?;
let router_key_state =
build_router_key_state_from_runtime(router_keys).map_err(|e| e.to_string())?;
Ok(RpkiCanonicalCacheRepresentation {
version: 0,
hash_alg: CcrDigestAlgorithm::Sha256,
produced_at,
mfts: Some(manifest_state),
vrps: Some(vrp_state),
vaps: Some(aspa_state),
tas: Some(ta_state),
rks: Some(router_key_state),
})
}
pub fn manifest_count(&self) -> usize {
self.manifests_by_hash.len()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ccr::export::build_ccr_from_run;
use crate::ccr::verify::verify_content_info;
use crate::data_model::manifest::ManifestObject;
use crate::data_model::rc::SubjectInfoAccess;
use crate::data_model::roa::{IpPrefix, RoaAfi};
use crate::data_model::ta::TrustAnchor;
use crate::data_model::tal::Tal;
use crate::storage::{
PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, ValidatedManifestMeta,
VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary,
VcirCcrManifestProjection, VcirChildEntry, VcirInstanceGate, VcirRelatedArtifact,
VcirSummary,
};
use sha2::Digest;
fn sample_trust_anchor() -> TrustAnchor {
let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let tal_bytes = std::fs::read(base.join("tests/fixtures/tal/apnic-rfc7730-https.tal"))
.expect("read tal");
let ta_der = std::fs::read(base.join("tests/fixtures/ta/apnic-ta.cer")).expect("read ta");
let tal = Tal::decode_bytes(&tal_bytes).expect("decode tal");
TrustAnchor::bind_der(tal, &ta_der, None).expect("bind ta")
}
fn sample_vcir_and_manifest(
store: &RocksStore,
) -> (
ValidatedCaInstanceResult,
Vec<Vrp>,
Vec<AspaAttestation>,
Vec<RouterKeyPayload>,
) {
let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let manifest_der = std::fs::read(
base.join(
"tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft",
),
)
.expect("read manifest");
let manifest = ManifestObject::decode_der(&manifest_der).expect("decode manifest");
let manifest_hash = hex::encode(sha2::Sha256::digest(&manifest_der));
let mut raw = RawByHashEntry::from_bytes(manifest_hash.clone(), manifest_der.clone());
raw.origin_uris
.push("rsync://example.test/repo/current.mft".to_string());
raw.object_type = Some("mft".to_string());
raw.encoding = Some("der".to_string());
store.put_raw_by_hash_entry(&raw).expect("put raw");
let projection = VcirCcrManifestProjection {
manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(),
manifest_sha256: sha2::Sha256::digest(&manifest_der).to_vec(),
manifest_size: manifest_der.len() as u64,
manifest_ee_aki: manifest.signed_object.signed_data.certificates[0]
.resource_cert
.tbs
.extensions
.authority_key_identifier
.clone()
.expect("manifest aki"),
manifest_number_be: manifest.manifest.manifest_number.bytes_be.clone(),
manifest_this_update: PackTime::from_utc_offset_datetime(manifest.manifest.this_update),
manifest_sia_locations_der: match manifest.signed_object.signed_data.certificates[0]
.resource_cert
.tbs
.extensions
.subject_info_access
.as_ref()
.expect("manifest sia")
{
SubjectInfoAccess::Ee(ee_sia) => ee_sia
.access_descriptions
.iter()
.map(|ad| {
let oid = encode_oid_for_test(&ad.access_method_oid)
.expect("encode access method oid");
let uri = encode_tlv_for_test(0x86, ad.access_location.as_bytes().to_vec());
encode_sequence_for_test(&[oid, uri])
})
.collect(),
SubjectInfoAccess::Ca(_) => panic!("manifest ee sia should not be CA variant"),
},
subordinate_skis: vec![vec![0x33; 20]],
};
let vcir = ValidatedCaInstanceResult {
manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(),
parent_manifest_rsync_uri: None,
tal_id: "apnic".to_string(),
ca_subject_name: "CN=test".to_string(),
ca_ski: "11".repeat(20),
issuer_ski: "22".repeat(20),
last_successful_validation_time: PackTime::from_utc_offset_datetime(
manifest.manifest.this_update,
),
current_manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(),
current_crl_rsync_uri: "rsync://example.test/repo/current.crl".to_string(),
validated_manifest_meta: ValidatedManifestMeta {
validated_manifest_number: manifest.manifest.manifest_number.bytes_be.clone(),
validated_manifest_this_update: PackTime::from_utc_offset_datetime(
manifest.manifest.this_update,
),
validated_manifest_next_update: PackTime::from_utc_offset_datetime(
manifest.manifest.next_update,
),
},
ccr_manifest_projection: projection.clone(),
instance_gate: VcirInstanceGate {
manifest_next_update: PackTime::from_utc_offset_datetime(
manifest.manifest.next_update,
),
current_crl_next_update: PackTime::from_utc_offset_datetime(
manifest.manifest.next_update,
),
self_ca_not_after: PackTime::from_utc_offset_datetime(
manifest.manifest.next_update,
),
instance_effective_until: PackTime::from_utc_offset_datetime(
manifest.manifest.next_update,
),
},
child_entries: vec![VcirChildEntry {
child_manifest_rsync_uri: "rsync://example.test/repo/child.mft".to_string(),
child_cert_rsync_uri: "rsync://example.test/repo/child.cer".to_string(),
child_cert_hash: "aa".repeat(32),
child_ski: "33".repeat(20),
child_rsync_base_uri: "rsync://example.test/repo/".to_string(),
child_publication_point_rsync_uri: "rsync://example.test/repo/".to_string(),
child_rrdp_notification_uri: None,
child_effective_ip_resources: None,
child_effective_as_resources: None,
accepted_at_validation_time: PackTime::from_utc_offset_datetime(
manifest.manifest.this_update,
),
}],
local_outputs: Vec::new(),
related_artifacts: vec![VcirRelatedArtifact {
artifact_role: VcirArtifactRole::Manifest,
artifact_kind: VcirArtifactKind::Mft,
uri: Some("rsync://example.test/repo/current.mft".to_string()),
sha256: manifest_hash,
object_type: Some("mft".to_string()),
validation_status: VcirArtifactValidationStatus::Accepted,
}],
summary: VcirSummary {
local_vrp_count: 0,
local_aspa_count: 0,
local_router_key_count: 0,
child_count: 1,
accepted_object_count: 1,
rejected_object_count: 0,
},
audit_summary: VcirAuditSummary {
failed_fetch_eligible: true,
last_failed_fetch_reason: None,
warning_count: 0,
audit_flags: Vec::new(),
},
};
let vrps = vec![Vrp {
asn: 64496,
prefix: IpPrefix {
afi: RoaAfi::Ipv4,
prefix_len: 8,
addr: [10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
},
max_length: 8,
}];
let aspas = vec![AspaAttestation {
customer_as_id: 64496,
provider_as_ids: vec![64497],
}];
let router_keys = vec![RouterKeyPayload {
as_id: 64496,
ski: vec![0x11; 20],
spki_der: vec![0x30, 0x00],
source_object_uri: "rsync://example.test/repo/router.cer".to_string(),
source_object_hash: hex::encode([0x11; 32]),
source_ee_cert_hash: hex::encode([0x11; 32]),
item_effective_until: PackTime::from_utc_offset_datetime(
time::OffsetDateTime::now_utc() + time::Duration::hours(1),
),
}];
(vcir, vrps, aspas, router_keys)
}
fn encode_oid_for_test(oid: &str) -> Result<Vec<u8>, String> {
let arcs = oid
.split('.')
.map(|part| part.parse::<u64>().map_err(|_| format!("bad oid: {oid}")))
.collect::<Result<Vec<_>, _>>()?;
if arcs.len() < 2 {
return Err(format!("bad oid: {oid}"));
}
let mut body = Vec::new();
body.push((arcs[0] * 40 + arcs[1]) as u8);
for arc in &arcs[2..] {
encode_base128_for_test(*arc, &mut body);
}
Ok(encode_tlv_for_test(0x06, body))
}
fn encode_base128_for_test(mut value: u64, out: &mut Vec<u8>) {
let mut tmp = vec![(value & 0x7F) as u8];
value >>= 7;
while value > 0 {
tmp.push(((value & 0x7F) as u8) | 0x80);
value >>= 7;
}
tmp.reverse();
out.extend_from_slice(&tmp);
}
fn encode_sequence_for_test(elements: &[Vec<u8>]) -> Vec<u8> {
let total_len: usize = elements.iter().map(Vec::len).sum();
let mut buf = Vec::with_capacity(total_len);
for element in elements {
buf.extend_from_slice(element);
}
encode_tlv_for_test(0x30, buf)
}
fn encode_tlv_for_test(tag: u8, value: Vec<u8>) -> Vec<u8> {
let mut out = Vec::with_capacity(1 + 9 + value.len());
out.push(tag);
if value.len() < 0x80 {
out.push(value.len() as u8);
} else {
out.push(0x81);
out.push(value.len() as u8);
}
out.extend_from_slice(&value);
out
}
#[test]
fn accumulator_finish_matches_builder_on_fresh_vcir_inputs() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let (vcir, vrps, aspas, router_keys) = sample_vcir_and_manifest(&store);
store.put_vcir(&vcir).expect("put vcir");
let trust_anchor = sample_trust_anchor();
let builder_ccr = build_ccr_from_run(
&store,
&[trust_anchor.clone()],
&vrps,
&aspas,
&router_keys,
time::OffsetDateTime::now_utc(),
)
.expect("build ccr from run");
let mut accumulator = CcrAccumulator::new(vec![trust_anchor]);
accumulator
.append_manifest_projection(&vcir.ccr_manifest_projection)
.expect("append manifest projection");
let accumulated_ccr = accumulator
.finish(time::OffsetDateTime::now_utc(), &vrps, &aspas, &router_keys)
.expect("finish accumulator");
assert_eq!(builder_ccr.mfts, accumulated_ccr.mfts);
assert_eq!(builder_ccr.vrps, accumulated_ccr.vrps);
assert_eq!(builder_ccr.vaps, accumulated_ccr.vaps);
assert_eq!(builder_ccr.tas, accumulated_ccr.tas);
assert_eq!(builder_ccr.rks, accumulated_ccr.rks);
let ci = crate::ccr::model::CcrContentInfo::new(accumulated_ccr);
verify_content_info(&ci).expect("verify accumulated ccr");
}
}

119
src/ccr/build.rs
View File

@ -1,5 +1,6 @@
use std::collections::{BTreeMap, BTreeSet};
use serde::Serialize;
use sha2::Digest;
use crate::ccr::encode::{
@ -18,7 +19,7 @@ use crate::data_model::roa::RoaAfi;
use crate::data_model::router_cert::BgpsecRouterCertificate;
use crate::data_model::ta::TrustAnchor;
use crate::storage::{RocksStore, ValidatedCaInstanceResult, VcirArtifactRole};
use crate::validation::objects::{AspaAttestation, Vrp};
use crate::validation::objects::{AspaAttestation, RouterKeyPayload, Vrp};
#[derive(Debug, thiserror::Error)]
pub enum CcrBuildError {
@ -83,6 +84,19 @@ pub enum CcrBuildError {
RouterKeyEncode(String),
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
pub struct ManifestStateBuildBreakdown {
pub vcir_count: usize,
pub unique_manifest_count: usize,
pub find_manifest_artifact_ms: u64,
pub load_manifest_blob_ms: u64,
pub decode_manifest_der_ms: u64,
pub build_manifest_instance_ms: u64,
pub dedup_manifest_instance_ms: u64,
pub encode_manifest_state_ms: u64,
pub total_ms: u64,
}
pub fn build_roa_payload_state(vrps: &[Vrp]) -> Result<RoaPayloadState, CcrBuildError> {
let mut grouped: BTreeMap<u32, BTreeSet<RoaPayloadKey>> = BTreeMap::new();
for vrp in vrps {
@ -174,10 +188,28 @@ pub fn build_manifest_state_from_vcirs(
store: &RocksStore,
vcirs: &[ValidatedCaInstanceResult],
) -> Result<ManifestState, CcrBuildError> {
build_manifest_state_from_vcirs_with_breakdown(store, vcirs).map(|(state, _)| state)
}
pub fn build_manifest_state_from_vcirs_with_breakdown(
store: &RocksStore,
vcirs: &[ValidatedCaInstanceResult],
) -> Result<(ManifestState, ManifestStateBuildBreakdown), CcrBuildError> {
let total_started = std::time::Instant::now();
let mut breakdown = ManifestStateBuildBreakdown {
vcir_count: vcirs.len(),
..ManifestStateBuildBreakdown::default()
};
let mut find_manifest_artifact_duration = std::time::Duration::ZERO;
let mut load_manifest_blob_duration = std::time::Duration::ZERO;
let mut decode_manifest_der_duration = std::time::Duration::ZERO;
let mut build_manifest_instance_duration = std::time::Duration::ZERO;
let mut dedup_manifest_instance_duration = std::time::Duration::ZERO;
let mut mis_by_hash: BTreeMap<Vec<u8>, ManifestInstance> = BTreeMap::new();
let mut most_recent_update = time::OffsetDateTime::UNIX_EPOCH;
for vcir in vcirs {
let started = std::time::Instant::now();
let manifest_artifact = vcir
.related_artifacts
.iter()
@ -188,7 +220,9 @@ pub fn build_manifest_state_from_vcirs(
.ok_or_else(|| {
CcrBuildError::MissingManifestArtifact(vcir.current_manifest_rsync_uri.clone())
})?;
find_manifest_artifact_duration += started.elapsed();
let started = std::time::Instant::now();
let raw_bytes = store
.get_blob_bytes(&manifest_artifact.sha256)
.map_err(|e| CcrBuildError::LoadManifestRawBytes {
@ -199,13 +233,17 @@ pub fn build_manifest_state_from_vcirs(
manifest_rsync_uri: vcir.current_manifest_rsync_uri.clone(),
sha256_hex: manifest_artifact.sha256.clone(),
})?;
load_manifest_blob_duration += started.elapsed();
let started = std::time::Instant::now();
let manifest =
ManifestObject::decode_der(&raw_bytes).map_err(|e| CcrBuildError::ManifestDecode {
manifest_rsync_uri: vcir.current_manifest_rsync_uri.clone(),
detail: e.to_string(),
})?;
decode_manifest_der_duration += started.elapsed();
let started = std::time::Instant::now();
let ee = &manifest.signed_object.signed_data.certificates[0].resource_cert;
let aki = ee
.tbs
@ -263,7 +301,9 @@ pub fn build_manifest_state_from_vcirs(
locations,
subordinates,
};
build_manifest_instance_duration += started.elapsed();
let started = std::time::Instant::now();
match mis_by_hash.get(instance.hash.as_slice()) {
Some(existing) if existing != &instance => {
return Err(CcrBuildError::DuplicateManifestHashConflict(
@ -275,16 +315,27 @@ pub fn build_manifest_state_from_vcirs(
mis_by_hash.insert(instance.hash.clone(), instance);
}
}
dedup_manifest_instance_duration += started.elapsed();
}
let started = std::time::Instant::now();
let mis: Vec<ManifestInstance> = mis_by_hash.into_values().collect();
let payload_der = encode_manifest_state_payload_der(&mis)
.map_err(|e| CcrBuildError::ManifestEncode(e.to_string()))?;
Ok(ManifestState {
let state = ManifestState {
mis,
most_recent_update,
hash: compute_state_hash(&payload_der),
})
};
breakdown.encode_manifest_state_ms = started.elapsed().as_millis() as u64;
breakdown.find_manifest_artifact_ms = find_manifest_artifact_duration.as_millis() as u64;
breakdown.load_manifest_blob_ms = load_manifest_blob_duration.as_millis() as u64;
breakdown.decode_manifest_der_ms = decode_manifest_der_duration.as_millis() as u64;
breakdown.build_manifest_instance_ms = build_manifest_instance_duration.as_millis() as u64;
breakdown.dedup_manifest_instance_ms = dedup_manifest_instance_duration.as_millis() as u64;
breakdown.unique_manifest_count = state.mis.len();
breakdown.total_ms = total_started.elapsed().as_millis() as u64;
Ok((state, breakdown))
}
fn collect_subordinate_ski_bytes(
@ -371,6 +422,36 @@ pub fn build_router_key_state(
})
}
pub fn build_router_key_state_from_runtime(
router_keys: &[RouterKeyPayload],
) -> Result<RouterKeyState, CcrBuildError> {
let mut grouped: BTreeMap<u32, BTreeSet<RouterKey>> = BTreeMap::new();
for router_key in router_keys {
grouped
.entry(router_key.as_id)
.or_default()
.insert(RouterKey {
ski: router_key.ski.clone(),
spki_der: router_key.spki_der.clone(),
});
}
let rksets: Vec<RouterKeySet> = grouped
.into_iter()
.map(|(as_id, router_keys)| RouterKeySet {
as_id,
router_keys: router_keys.into_iter().collect(),
})
.collect();
let payload_der = encode_router_key_state_payload_der(&rksets)
.map_err(|e| CcrBuildError::RouterKeyEncode(e.to_string()))?;
Ok(RouterKeyState {
rksets,
hash: compute_state_hash(&payload_der),
})
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
struct RoaPayloadKey {
afi: u16,
@ -578,6 +659,37 @@ mod tests {
let hash = hex::encode(sha2::Sha256::digest(manifest_der));
let now = manifest.manifest.this_update;
let next = manifest.manifest.next_update;
let projection = crate::storage::VcirCcrManifestProjection {
manifest_rsync_uri: manifest_uri.to_string(),
manifest_sha256: sha2::Sha256::digest(manifest_der).to_vec(),
manifest_size: manifest_der.len() as u64,
manifest_ee_aki: manifest.signed_object.signed_data.certificates[0]
.resource_cert
.tbs
.extensions
.authority_key_identifier
.clone()
.expect("manifest aki"),
manifest_number_be: manifest.manifest.manifest_number.bytes_be.clone(),
manifest_this_update: crate::storage::PackTime::from_utc_offset_datetime(now),
manifest_sia_locations_der: match manifest.signed_object.signed_data.certificates[0]
.resource_cert
.tbs
.extensions
.subject_info_access
.as_ref()
.expect("manifest sia")
{
SubjectInfoAccess::Ee(ee_sia) => ee_sia
.access_descriptions
.iter()
.map(encode_access_description_der)
.collect::<Result<Vec<_>, _>>()
.expect("encode access descriptions"),
SubjectInfoAccess::Ca(_) => panic!("manifest ee sia should not be CA variant"),
},
subordinate_skis: vec![hex::decode(child_ski_hex).expect("decode child ski")],
};
crate::storage::ValidatedCaInstanceResult {
manifest_rsync_uri: manifest_uri.to_string(),
parent_manifest_rsync_uri: None,
@ -599,6 +711,7 @@ mod tests {
next,
),
},
ccr_manifest_projection: projection,
instance_gate: crate::storage::VcirInstanceGate {
manifest_next_update: crate::storage::PackTime::from_utc_offset_datetime(next),
current_crl_next_update: crate::storage::PackTime::from_utc_offset_datetime(next),

6
src/ccr/decode.rs
View File

@ -1,7 +1,7 @@
use crate::ccr::model::{
AspaPayloadSet, AspaPayloadState, CCR_VERSION_V0, CcrContentInfo, CcrDigestAlgorithm,
ManifestInstance, ManifestState, RoaPayloadSet, RoaPayloadState, RouterKey, RouterKeySet,
RouterKeyState, RpkiCanonicalCacheRepresentation, TrustAnchorState,
AspaPayloadSet, AspaPayloadState, CcrContentInfo, CcrDigestAlgorithm, ManifestInstance,
ManifestState, RoaPayloadSet, RoaPayloadState, RouterKey, RouterKeySet, RouterKeyState,
RpkiCanonicalCacheRepresentation, TrustAnchorState, CCR_VERSION_V0,
};
use crate::data_model::common::{BigUnsigned, DerReader};
use crate::data_model::oid::{OID_CT_RPKI_CCR, OID_CT_RPKI_CCR_RAW, OID_SHA256, OID_SHA256_RAW};

2
src/ccr/dump.rs
View File

@ -1,4 +1,4 @@
use crate::ccr::decode::{CcrDecodeError, decode_content_info};
use crate::ccr::decode::{decode_content_info, CcrDecodeError};
use serde_json::json;
#[derive(Debug, thiserror::Error)]

6
src/ccr/encode.rs
View File

@ -1,7 +1,7 @@
use crate::ccr::model::{
AspaPayloadSet, AspaPayloadState, CCR_VERSION_V0, CcrContentInfo, CcrDigestAlgorithm,
ManifestInstance, ManifestState, RoaPayloadSet, RoaPayloadState, RouterKey, RouterKeySet,
RouterKeyState, RpkiCanonicalCacheRepresentation, TrustAnchorState,
AspaPayloadSet, AspaPayloadState, CcrContentInfo, CcrDigestAlgorithm, ManifestInstance,
ManifestState, RoaPayloadSet, RoaPayloadState, RouterKey, RouterKeySet, RouterKeyState,
RpkiCanonicalCacheRepresentation, TrustAnchorState, CCR_VERSION_V0,
};
use crate::data_model::common::BigUnsigned;
use crate::data_model::oid::{OID_CT_RPKI_CCR_RAW, OID_SHA256_RAW};

140
src/ccr/export.rs
View File

@ -1,12 +1,14 @@
use crate::ccr::build::{
CcrBuildError, build_aspa_payload_state, build_manifest_state_from_vcirs,
build_roa_payload_state, build_trust_anchor_state,
build_aspa_payload_state, build_manifest_state_from_vcirs_with_breakdown,
build_roa_payload_state, build_router_key_state_from_runtime, build_trust_anchor_state,
CcrBuildError, ManifestStateBuildBreakdown,
};
use crate::ccr::encode::{CcrEncodeError, encode_content_info};
use crate::ccr::encode::{encode_content_info, CcrEncodeError};
use crate::ccr::model::{CcrContentInfo, CcrDigestAlgorithm, RpkiCanonicalCacheRepresentation};
use crate::data_model::ta::TrustAnchor;
use crate::storage::RocksStore;
use crate::validation::objects::{AspaAttestation, RouterKeyPayload, Vrp};
use serde::Serialize;
use std::path::Path;
#[derive(Debug, thiserror::Error)]
@ -24,6 +26,19 @@ pub enum CcrExportError {
Write(String, String),
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
pub struct CcrBuildBreakdown {
pub vcir_count: usize,
pub list_vcirs_ms: u64,
pub manifest_state_ms: u64,
pub manifest_state_breakdown: ManifestStateBuildBreakdown,
pub roa_payload_state_ms: u64,
pub aspa_payload_state_ms: u64,
pub trust_anchor_state_ms: u64,
pub router_key_state_ms: u64,
pub total_ms: u64,
}
pub fn build_ccr_from_run(
store: &RocksStore,
trust_anchors: &[TrustAnchor],
@ -32,63 +47,63 @@ pub fn build_ccr_from_run(
router_keys: &[RouterKeyPayload],
produced_at: time::OffsetDateTime,
) -> Result<RpkiCanonicalCacheRepresentation, CcrExportError> {
build_ccr_from_run_with_breakdown(store, trust_anchors, vrps, aspas, router_keys, produced_at)
.map(|(ccr, _)| ccr)
}
pub fn build_ccr_from_run_with_breakdown(
store: &RocksStore,
trust_anchors: &[TrustAnchor],
vrps: &[Vrp],
aspas: &[AspaAttestation],
router_keys: &[RouterKeyPayload],
produced_at: time::OffsetDateTime,
) -> Result<(RpkiCanonicalCacheRepresentation, CcrBuildBreakdown), CcrExportError> {
let total_started = std::time::Instant::now();
let mut breakdown = CcrBuildBreakdown::default();
let started = std::time::Instant::now();
let vcirs = store
.list_vcirs()
.map_err(|e| CcrExportError::ListVcirs(e.to_string()))?;
breakdown.list_vcirs_ms = started.elapsed().as_millis() as u64;
breakdown.vcir_count = vcirs.len();
let mfts = Some(build_manifest_state_from_vcirs(store, &vcirs)?);
let vrps = Some(build_roa_payload_state(vrps)?);
let vaps = Some(build_aspa_payload_state(aspas)?);
let tas = Some(build_trust_anchor_state(trust_anchors)?);
let rks = Some(build_router_key_state_from_runtime(router_keys)?);
let started = std::time::Instant::now();
let (mfts, manifest_state_breakdown) =
build_manifest_state_from_vcirs_with_breakdown(store, &vcirs)?;
breakdown.manifest_state_ms = started.elapsed().as_millis() as u64;
breakdown.manifest_state_breakdown = manifest_state_breakdown;
Ok(RpkiCanonicalCacheRepresentation {
let started = std::time::Instant::now();
let vrps = build_roa_payload_state(vrps)?;
breakdown.roa_payload_state_ms = started.elapsed().as_millis() as u64;
let started = std::time::Instant::now();
let vaps = build_aspa_payload_state(aspas)?;
breakdown.aspa_payload_state_ms = started.elapsed().as_millis() as u64;
let started = std::time::Instant::now();
let tas = build_trust_anchor_state(trust_anchors)?;
breakdown.trust_anchor_state_ms = started.elapsed().as_millis() as u64;
let started = std::time::Instant::now();
let rks = build_router_key_state_from_runtime(router_keys)?;
breakdown.router_key_state_ms = started.elapsed().as_millis() as u64;
let ccr = RpkiCanonicalCacheRepresentation {
version: 0,
hash_alg: CcrDigestAlgorithm::Sha256,
produced_at,
mfts,
vrps,
vaps,
tas,
rks,
})
}
mfts: Some(mfts),
vrps: Some(vrps),
vaps: Some(vaps),
tas: Some(tas),
rks: Some(rks),
};
breakdown.total_ms = total_started.elapsed().as_millis() as u64;
fn build_router_key_state_from_runtime(
router_keys: &[RouterKeyPayload],
) -> Result<crate::ccr::model::RouterKeyState, CcrBuildError> {
use crate::ccr::model::{RouterKey, RouterKeySet};
use std::collections::{BTreeMap, BTreeSet};
let mut grouped: BTreeMap<u32, BTreeSet<RouterKey>> = BTreeMap::new();
for router_key in router_keys {
grouped
.entry(router_key.as_id)
.or_default()
.insert(RouterKey {
ski: router_key.ski.clone(),
spki_der: router_key.spki_der.clone(),
});
}
let rksets = grouped
.into_iter()
.map(|(as_id, router_keys)| RouterKeySet {
as_id,
router_keys: router_keys.into_iter().collect(),
})
.collect::<Vec<_>>();
build_router_key_state_from_sets(&rksets)
}
fn build_router_key_state_from_sets(
rksets: &[crate::ccr::model::RouterKeySet],
) -> Result<crate::ccr::model::RouterKeyState, CcrBuildError> {
let der = crate::ccr::encode::encode_router_key_state_payload_der(rksets)
.map_err(|e| CcrBuildError::RouterKeyEncode(e.to_string()))?;
Ok(crate::ccr::model::RouterKeyState {
rksets: rksets.to_vec(),
hash: crate::ccr::compute_state_hash(&der),
})
Ok((ccr, breakdown))
}
pub fn write_ccr_file(
@ -115,7 +130,8 @@ mod tests {
use crate::storage::{
PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, ValidatedManifestMeta,
VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary,
VcirChildEntry, VcirInstanceGate, VcirRelatedArtifact, VcirSummary,
VcirCcrManifestProjection, VcirChildEntry, VcirInstanceGate, VcirRelatedArtifact,
VcirSummary,
};
use crate::validation::objects::{AspaAttestation, RouterKeyPayload, Vrp};
use sha2::Digest;
@ -140,6 +156,25 @@ mod tests {
raw.object_type = Some("mft".to_string());
raw.encoding = Some("der".to_string());
store.put_raw_by_hash_entry(&raw).expect("put raw");
let projection = VcirCcrManifestProjection {
manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(),
manifest_sha256: sha2::Sha256::digest(&manifest_der).to_vec(),
manifest_size: manifest_der.len() as u64,
manifest_ee_aki: manifest.signed_object.signed_data.certificates[0]
.resource_cert
.tbs
.extensions
.authority_key_identifier
.clone()
.expect("manifest aki"),
manifest_number_be: manifest.manifest.manifest_number.bytes_be.clone(),
manifest_this_update: PackTime::from_utc_offset_datetime(manifest.manifest.this_update),
manifest_sia_locations_der: vec![vec![
0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05,
b'r', b's', b'y', b'n', b'c',
]],
subordinate_skis: vec![vec![0x33; 20]],
};
ValidatedCaInstanceResult {
manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(),
parent_manifest_rsync_uri: None,
@ -161,6 +196,7 @@ mod tests {
manifest.manifest.next_update,
),
},
ccr_manifest_projection: projection,
instance_gate: VcirInstanceGate {
manifest_next_update: PackTime::from_utc_offset_datetime(
manifest.manifest.next_update,

31
src/ccr/mod.rs
View File

@ -1,4 +1,6 @@
#[cfg(feature = "full")]
pub mod accumulator;
#[cfg(feature = "full")]
pub mod build;
pub mod decode;
pub mod dump;
@ -11,15 +13,22 @@ pub mod model;
pub mod verify;
#[cfg(feature = "full")]
pub use build::{
CcrBuildError, build_aspa_payload_state, build_manifest_state_from_vcirs,
build_roa_payload_state, build_trust_anchor_state,
};
pub use decode::{CcrDecodeError, decode_content_info};
pub use dump::{CcrDumpError, dump_content_info_json, dump_content_info_json_value};
pub use encode::{CcrEncodeError, encode_content_info};
pub use accumulator::{CcrAccumulator, CcrManifestContribution};
#[cfg(feature = "full")]
pub use export::{CcrExportError, build_ccr_from_run, write_ccr_file};
pub use build::{
build_aspa_payload_state, build_manifest_state_from_vcirs,
build_manifest_state_from_vcirs_with_breakdown, build_roa_payload_state,
build_router_key_state_from_runtime, build_trust_anchor_state, CcrBuildError,
ManifestStateBuildBreakdown,
};
pub use decode::{decode_content_info, CcrDecodeError};
pub use dump::{dump_content_info_json, dump_content_info_json_value, CcrDumpError};
pub use encode::{encode_content_info, CcrEncodeError};
#[cfg(feature = "full")]
pub use export::{
build_ccr_from_run, build_ccr_from_run_with_breakdown, write_ccr_file, CcrBuildBreakdown,
CcrExportError,
};
pub use hash::{compute_state_hash, verify_state_hash};
pub use model::{
AspaPayloadSet, AspaPayloadState, CcrContentInfo, CcrDigestAlgorithm, ManifestInstance,
@ -28,7 +37,7 @@ pub use model::{
};
#[cfg(feature = "full")]
pub use verify::{
CcrVerifyError, CcrVerifySummary, extract_vrp_rows, verify_against_report_json_path,
verify_against_vcir_store, verify_against_vcir_store_path, verify_content_info,
verify_content_info_bytes,
extract_vrp_rows, verify_against_report_json_path, verify_against_vcir_store,
verify_against_vcir_store_path, verify_content_info, verify_content_info_bytes, CcrVerifyError,
CcrVerifySummary,
};

2
src/ccr/model.rs
View File

@ -1,4 +1,4 @@
use crate::data_model::common::{BigUnsigned, der_take_tlv};
use crate::data_model::common::{der_take_tlv, BigUnsigned};
use crate::data_model::oid::{OID_CT_RPKI_CCR, OID_SHA256};
pub const CCR_VERSION_V0: u32 = 0;

26
src/ccr/verify.rs
View File

@ -1,4 +1,4 @@
use crate::ccr::decode::{CcrDecodeError, decode_content_info};
use crate::ccr::decode::{decode_content_info, CcrDecodeError};
use crate::ccr::encode::{
encode_aspa_payload_state_payload_der, encode_manifest_state_payload_der,
encode_roa_payload_state_payload_der, encode_router_key_state_payload_der,
@ -430,8 +430,9 @@ mod tests {
use crate::data_model::roa::{IpPrefix, RoaAfi};
use crate::storage::{
PackTime, ValidatedCaInstanceResult, ValidatedManifestMeta, VcirArtifactKind,
VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, VcirChildEntry,
VcirInstanceGate, VcirRelatedArtifact, VcirSummary,
VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary,
VcirCcrManifestProjection, VcirChildEntry, VcirInstanceGate, VcirRelatedArtifact,
VcirSummary,
};
use crate::validation::objects::{AspaAttestation, Vrp};
@ -505,6 +506,22 @@ mod tests {
})
}
fn sample_ccr_manifest_projection(manifest_rsync_uri: &str) -> VcirCcrManifestProjection {
VcirCcrManifestProjection {
manifest_rsync_uri: manifest_rsync_uri.to_string(),
manifest_sha256: vec![0x44; 32],
manifest_size: 2048,
manifest_ee_aki: vec![0x55; 20],
manifest_number_be: vec![1],
manifest_this_update: PackTime::from_utc_offset_datetime(sample_time()),
manifest_sia_locations_der: vec![vec![
0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05,
b'r', b's', b'y', b'n', b'c',
]],
subordinate_skis: vec![vec![0x33; 20]],
}
}
#[test]
fn verify_detects_each_state_hash_mismatch() {
let mut ci = sample_content_info();
@ -599,6 +616,9 @@ mod tests {
validated_manifest_this_update: PackTime::from_utc_offset_datetime(sample_time()),
validated_manifest_next_update: PackTime::from_utc_offset_datetime(sample_time()),
},
ccr_manifest_projection: sample_ccr_manifest_projection(
"rsync://example.test/current.mft",
),
instance_gate: VcirInstanceGate {
manifest_next_update: PackTime::from_utc_offset_datetime(sample_time()),
current_crl_next_update: PackTime::from_utc_offset_datetime(sample_time()),

623
src/cli.rs
View File

@ -1,11 +1,14 @@
use crate::ccr::{build_ccr_from_run, write_ccr_file};
use crate::cir::{CirTalBinding, export_cir_from_run_multi};
use crate::ccr::{
build_ccr_from_run_with_breakdown, write_ccr_file, CcrAccumulator, CcrBuildBreakdown,
};
use crate::cir::{export_cir_from_run_multi, CirTalBinding};
use std::io::BufWriter;
use std::path::{Path, PathBuf};
use crate::analysis::timing::{TimingHandle, TimingMeta, TimingMetaUpdate};
use crate::audit::{
AspaOutput, AuditRepoSyncStats, AuditReportV2, AuditRunMeta, AuditWarning, TreeSummary,
VrpOutput, format_roa_ip_prefix,
format_roa_ip_prefix, AspaOutput, AuditRepoSyncStats, AuditReportV2, AuditRunMeta,
AuditWarning, TreeSummary, VrpOutput,
};
use crate::fetch::http::{BlockingHttpFetcher, HttpFetcherConfig};
use crate::fetch::rsync::LocalDirRsyncFetcher;
@ -15,7 +18,7 @@ use crate::parallel::types::TalInputSpec;
use crate::policy::Policy;
use crate::storage::RocksStore;
use crate::validation::run_tree_from_tal::{
RunTreeFromTalAuditOutput, run_tree_from_multiple_tals_parallel_phase1_audit,
run_tree_from_multiple_tals_parallel_phase1_audit,
run_tree_from_multiple_tals_parallel_phase2_audit,
run_tree_from_tal_and_ta_der_parallel_phase1_audit,
run_tree_from_tal_and_ta_der_parallel_phase2_audit,
@ -27,6 +30,7 @@ use crate::validation::run_tree_from_tal::{
run_tree_from_tal_and_ta_der_serial_audit_with_timing,
run_tree_from_tal_url_parallel_phase1_audit, run_tree_from_tal_url_parallel_phase2_audit,
run_tree_from_tal_url_serial_audit, run_tree_from_tal_url_serial_audit_with_timing,
RunTreeFromTalAuditOutput,
};
use crate::validation::tree::TreeRunConfig;
use serde::Serialize;
@ -38,6 +42,7 @@ struct RunStageTiming {
report_build_ms: u64,
report_write_ms: Option<u64>,
ccr_build_ms: Option<u64>,
ccr_build_breakdown: Option<CcrBuildBreakdown>,
ccr_write_ms: Option<u64>,
cir_build_cir_ms: Option<u64>,
cir_write_cir_ms: Option<u64>,
@ -71,6 +76,7 @@ pub struct CliArgs {
pub raw_store_db: Option<PathBuf>,
pub policy_path: Option<PathBuf>,
pub report_json_path: Option<PathBuf>,
pub report_json_compact: bool,
pub ccr_out_path: Option<PathBuf>,
pub cir_enabled: bool,
pub cir_out_path: Option<PathBuf>,
@ -114,6 +120,7 @@ Options:
--raw-store-db <path> External raw-by-hash store DB path (optional)
--policy <path> Policy TOML path (optional)
--report-json <path> Write full audit report as JSON (optional)
--report-json-compact Write report JSON without pretty-printing (requires --report-json)
--ccr-out <path> Write CCR DER ContentInfo to this path (optional)
--cir-enable Export CIR after the run completes
--cir-out <path> Write CIR DER to this path (requires --cir-enable)
@ -175,6 +182,7 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
let mut raw_store_db: Option<PathBuf> = None;
let mut policy_path: Option<PathBuf> = None;
let mut report_json_path: Option<PathBuf> = None;
let mut report_json_compact: bool = false;
let mut ccr_out_path: Option<PathBuf> = None;
let mut cir_enabled: bool = false;
let mut cir_out_path: Option<PathBuf> = None;
@ -298,6 +306,9 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
let v = argv.get(i).ok_or("--report-json requires a value")?;
report_json_path = Some(PathBuf::from(v));
}
"--report-json-compact" => {
report_json_compact = true;
}
"--ccr-out" => {
i += 1;
let v = argv.get(i).ok_or("--ccr-out requires a value")?;
@ -514,6 +525,12 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
if cir_enabled && cir_out_path.is_none() {
return Err(format!("--cir-enable requires --cir-out\n\n{}", usage()));
}
if report_json_compact && report_json_path.is_none() {
return Err(format!(
"--report-json-compact requires --report-json\n\n{}",
usage()
));
}
if cir_static_root.is_some() {
return Err(format!(
"--cir-static-root is no longer supported; CIR export now writes only .cir files\n\n{}",
@ -660,6 +677,7 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
raw_store_db,
policy_path,
report_json_path,
report_json_compact,
ccr_out_path,
cir_enabled,
cir_out_path,
@ -698,11 +716,21 @@ fn read_policy(path: Option<&Path>) -> Result<Policy, String> {
}
}
fn write_json(path: &Path, report: &AuditReportV2) -> Result<(), String> {
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum ReportJsonFormat {
Pretty,
Compact,
}
fn write_json(path: &Path, report: &AuditReportV2, format: ReportJsonFormat) -> Result<(), String> {
let f = std::fs::File::create(path)
.map_err(|e| format!("create report file failed: {}: {e}", path.display()))?;
serde_json::to_writer_pretty(f, report)
.map_err(|e| format!("write report json failed: {e}"))?;
let writer = BufWriter::new(f);
match format {
ReportJsonFormat::Pretty => serde_json::to_writer_pretty(writer, report),
ReportJsonFormat::Compact => serde_json::to_writer(writer, report),
}
.map_err(|e| format!("write report json failed: {e}"))?;
Ok(())
}
@ -746,10 +774,85 @@ fn print_summary(report: &AuditReportV2) {
);
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct PostValidationShared {
discovery: crate::validation::from_tal::DiscoveredRootCaInstance,
discoveries: Arc<[crate::validation::from_tal::DiscoveredRootCaInstance]>,
instances_processed: usize,
instances_failed: usize,
tree_warnings: Arc<[crate::report::Warning]>,
vrps: Arc<[crate::validation::objects::Vrp]>,
aspas: Arc<[crate::validation::objects::AspaAttestation]>,
router_keys: Arc<[crate::validation::objects::RouterKeyPayload]>,
publication_points: Arc<[crate::audit::PublicationPointAudit]>,
downloads: Arc<[crate::audit::AuditDownloadEvent]>,
download_stats: crate::audit::AuditDownloadStats,
current_repo_objects: Arc<[crate::current_repo_index::CurrentRepoObject]>,
ccr_accumulator: Option<CcrAccumulator>,
}
impl PostValidationShared {
fn from_run_output(out: RunTreeFromTalAuditOutput) -> Self {
let RunTreeFromTalAuditOutput {
discovery,
discoveries,
tree,
publication_points,
downloads,
download_stats,
current_repo_objects,
ccr_accumulator,
} = out;
let crate::validation::tree::TreeRunOutput {
instances_processed,
instances_failed,
warnings,
vrps,
aspas,
router_keys,
} = tree;
Self {
discovery,
discoveries: discoveries.into(),
instances_processed,
instances_failed,
tree_warnings: warnings.into(),
vrps: vrps.into(),
aspas: aspas.into(),
router_keys: router_keys.into(),
publication_points: publication_points.into(),
downloads: downloads.into(),
download_stats,
current_repo_objects: current_repo_objects.into(),
ccr_accumulator,
}
}
fn trust_anchors(&self) -> Vec<crate::data_model::ta::TrustAnchor> {
if self.discoveries.is_empty() {
vec![self.discovery.trust_anchor.clone()]
} else {
self.discoveries
.iter()
.map(|item| item.trust_anchor.clone())
.collect()
}
}
fn current_repo_objects(&self) -> Option<&[crate::current_repo_index::CurrentRepoObject]> {
if self.current_repo_objects.is_empty() {
None
} else {
Some(self.current_repo_objects.as_ref())
}
}
}
fn build_report(
policy: &Policy,
validation_time: time::OffsetDateTime,
out: RunTreeFromTalAuditOutput,
shared: &PostValidationShared,
) -> AuditReportV2 {
use time::format_description::well_known::Rfc3339;
let validation_time_rfc3339_utc = validation_time
@ -757,8 +860,7 @@ fn build_report(
.format(&Rfc3339)
.expect("format validation_time");
let vrps = out
.tree
let vrps = shared
.vrps
.iter()
.map(|v| VrpOutput {
@ -768,8 +870,7 @@ fn build_report(
})
.collect::<Vec<_>>();
let aspas = out
.tree
let aspas = shared
.aspas
.iter()
.map(|a| AspaOutput {
@ -778,7 +879,7 @@ fn build_report(
})
.collect::<Vec<_>>();
let repo_sync_stats = build_repo_sync_stats(&out.publication_points);
let repo_sync_stats = build_repo_sync_stats(shared.publication_points.as_ref());
AuditReportV2 {
format_version: 2,
@ -787,19 +888,137 @@ fn build_report(
},
policy: policy.clone(),
tree: TreeSummary {
instances_processed: out.tree.instances_processed,
instances_failed: out.tree.instances_failed,
warnings: out.tree.warnings.iter().map(AuditWarning::from).collect(),
instances_processed: shared.instances_processed,
instances_failed: shared.instances_failed,
warnings: shared
.tree_warnings
.iter()
.map(AuditWarning::from)
.collect(),
},
publication_points: out.publication_points,
publication_points: shared.publication_points.iter().cloned().collect(),
vrps,
aspas,
downloads: out.downloads,
download_stats: out.download_stats,
downloads: shared.downloads.iter().cloned().collect(),
download_stats: shared.download_stats.clone(),
repo_sync_stats,
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct ReportTaskOutput {
report: AuditReportV2,
report_build_ms: u64,
report_write_ms: Option<u64>,
}
fn run_report_task(
policy: &Policy,
validation_time: time::OffsetDateTime,
shared: &PostValidationShared,
report_json_path: Option<&Path>,
report_json_format: ReportJsonFormat,
) -> Result<ReportTaskOutput, String> {
let report_started = std::time::Instant::now();
let report = build_report(policy, validation_time, shared);
let report_build_ms = report_started.elapsed().as_millis() as u64;
let report_write_ms = if let Some(path) = report_json_path {
let started = std::time::Instant::now();
write_json(path, &report, report_json_format)?;
Some(started.elapsed().as_millis() as u64)
} else {
None
};
Ok(ReportTaskOutput {
report,
report_build_ms,
report_write_ms,
})
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct CcrTaskOutput {
ccr_build_ms: Option<u64>,
ccr_build_breakdown: Option<CcrBuildBreakdown>,
ccr_write_ms: Option<u64>,
}
fn run_ccr_task(
store: &RocksStore,
shared: &PostValidationShared,
ccr_out_path: Option<&Path>,
produced_at: time::OffsetDateTime,
) -> Result<CcrTaskOutput, String> {
let mut ccr_build_ms = None;
let mut ccr_build_breakdown = None;
let mut ccr_write_ms = None;
if let Some(path) = ccr_out_path {
let started = std::time::Instant::now();
let (ccr, build_breakdown) = if let Some(accumulator) = shared.ccr_accumulator.as_ref() {
(
accumulator
.finish(
produced_at,
shared.vrps.as_ref(),
shared.aspas.as_ref(),
shared.router_keys.as_ref(),
)
.map_err(|e| e.to_string())?,
None,
)
} else {
let trust_anchors = shared.trust_anchors();
let (ccr, build_breakdown) = build_ccr_from_run_with_breakdown(
store,
&trust_anchors,
shared.vrps.as_ref(),
shared.aspas.as_ref(),
shared.router_keys.as_ref(),
produced_at,
)
.map_err(|e| e.to_string())?;
(ccr, Some(build_breakdown))
};
ccr_build_ms = Some(started.elapsed().as_millis() as u64);
ccr_build_breakdown = build_breakdown;
let started = std::time::Instant::now();
write_ccr_file(path, &ccr).map_err(|e| e.to_string())?;
ccr_write_ms = Some(started.elapsed().as_millis() as u64);
eprintln!("wrote CCR: {}", path.display());
}
Ok(CcrTaskOutput {
ccr_build_ms,
ccr_build_breakdown,
ccr_write_ms,
})
}
fn write_stage_timing(
report_json_path: Option<&Path>,
stage_timing: &RunStageTiming,
) -> Result<(), String> {
if let Some(path) = report_json_path {
if let Some(parent) = path.parent() {
let stage_timing_path = parent.join("stage-timing.json");
std::fs::write(
&stage_timing_path,
serde_json::to_vec_pretty(stage_timing).map_err(|e| e.to_string())?,
)
.map_err(|e| {
format!(
"write stage timing failed: {}: {e}",
stage_timing_path.display()
)
})?;
eprintln!("analysis: wrote {}", stage_timing_path.display());
}
}
Ok(())
}
fn resolve_cir_export_tal_uris(args: &CliArgs) -> Result<Vec<String>, String> {
if !args.cir_tal_uris.is_empty() {
return Ok(args.cir_tal_uris.clone());
@ -1468,37 +1687,39 @@ pub fn run(argv: &[String]) -> Result<(), String> {
};
let validation_ms = validation_started.elapsed().as_millis() as u64;
let shared = PostValidationShared::from_run_output(out);
if let Some((_out_dir, t)) = timing.as_ref() {
t.record_count("instances_processed", out.tree.instances_processed as u64);
t.record_count("instances_failed", out.tree.instances_failed as u64);
t.record_count("instances_processed", shared.instances_processed as u64);
t.record_count("instances_failed", shared.instances_failed as u64);
}
let publication_points = out.publication_points.len();
let publication_point_repo_sync_ms_total: u64 = out
let publication_points = shared.publication_points.len();
let publication_point_repo_sync_ms_total: u64 = shared
.publication_points
.iter()
.map(|pp| pp.repo_sync_duration_ms.unwrap_or(0))
.sum();
let download_event_count = out.download_stats.events_total;
let download_event_count = shared.download_stats.events_total;
let rrdp_download_ms_total: u64 = ["rrdp_notification", "rrdp_snapshot", "rrdp_delta"]
.iter()
.map(|key| {
out.download_stats
shared
.download_stats
.by_kind
.get(*key)
.map(|item| item.duration_ms_total)
.unwrap_or(0)
})
.sum();
let rsync_download_ms_total = out
let rsync_download_ms_total = shared
.download_stats
.by_kind
.get("rsync")
.map(|item| item.duration_ms_total)
.unwrap_or(0);
let repo_sync_ms_total = rrdp_download_ms_total + rsync_download_ms_total;
let download_bytes_total: u64 = out
let download_bytes_total: u64 = shared
.download_stats
.by_kind
.values()
@ -1517,51 +1738,66 @@ pub fn run(argv: &[String]) -> Result<(), String> {
None
};
let mut ccr_build_ms = None;
let mut ccr_write_ms = None;
if let Some(path) = args.ccr_out_path.as_deref() {
let started = std::time::Instant::now();
let trust_anchors = if out.discoveries.is_empty() {
vec![out.discovery.trust_anchor.clone()]
} else {
out.discoveries
.iter()
.map(|item| item.trust_anchor.clone())
.collect::<Vec<_>>()
};
let ccr = build_ccr_from_run(
store.as_ref(),
&trust_anchors,
&out.tree.vrps,
&out.tree.aspas,
&out.tree.router_keys,
time::OffsetDateTime::now_utc(),
)
.map_err(|e| e.to_string())?;
ccr_build_ms = Some(started.elapsed().as_millis() as u64);
let started = std::time::Instant::now();
write_ccr_file(path, &ccr).map_err(|e| e.to_string())?;
ccr_write_ms = Some(started.elapsed().as_millis() as u64);
eprintln!("wrote CCR: {}", path.display());
}
let report_json_format = if args.report_json_compact {
ReportJsonFormat::Compact
} else {
ReportJsonFormat::Pretty
};
let ccr_produced_at = time::OffsetDateTime::now_utc();
let (report_result, ccr_result) = std::thread::scope(|scope| {
let report_handle = scope.spawn(|| {
run_report_task(
&policy,
validation_time,
&shared,
args.report_json_path.as_deref(),
report_json_format,
)
});
let ccr_handle = scope.spawn(|| {
run_ccr_task(
store.as_ref(),
&shared,
args.ccr_out_path.as_deref(),
ccr_produced_at,
)
});
let report_result = report_handle
.join()
.map_err(|_| "report task panicked".to_string())
.and_then(|result| result);
let ccr_result = ccr_handle
.join()
.map_err(|_| "ccr task panicked".to_string())
.and_then(|result| result);
(report_result, ccr_result)
});
let report_output = report_result?;
let ccr_output = ccr_result?;
let report = report_output.report;
let report_build_ms = report_output.report_build_ms;
let report_write_ms = report_output.report_write_ms;
let ccr_build_ms = ccr_output.ccr_build_ms;
let ccr_build_breakdown = ccr_output.ccr_build_breakdown;
let ccr_write_ms = ccr_output.ccr_write_ms;
let mut cir_build_cir_ms = None;
let mut cir_write_cir_ms = None;
let mut cir_total_ms = None;
if args.cir_enabled {
let cir_tal_uris = resolve_cir_export_tal_uris(&args)?;
if cir_tal_uris.len() != out.discoveries.len() {
if cir_tal_uris.len() != shared.discoveries.len() {
return Err(format!(
"CIR export TAL URI count ({}) does not match discovery count ({})",
cir_tal_uris.len(),
out.discoveries.len()
shared.discoveries.len()
));
}
let cir_out_path = args
.cir_out_path
.as_deref()
.expect("validated by parse_args for cir");
let tal_bindings = out
let tal_bindings = shared
.discoveries
.iter()
.zip(cir_tal_uris.iter())
@ -1574,14 +1810,10 @@ pub fn run(argv: &[String]) -> Result<(), String> {
store.as_ref(),
&tal_bindings,
validation_time,
&out.publication_points,
shared.publication_points.as_ref(),
cir_out_path,
time::OffsetDateTime::now_utc().date(),
if out.current_repo_objects.is_empty() {
None
} else {
Some(out.current_repo_objects.as_slice())
},
shared.current_repo_objects(),
)
.map_err(|e| e.to_string())?;
cir_build_cir_ms = Some(summary.timing.build_cir_ms);
@ -1597,49 +1829,26 @@ pub fn run(argv: &[String]) -> Result<(), String> {
summary.timing.total_ms
);
}
let report_started = std::time::Instant::now();
let report = build_report(&policy, validation_time, out);
let report_build_ms = report_started.elapsed().as_millis() as u64;
let mut report_write_ms = None;
if let Some(p) = args.report_json_path.as_deref() {
let started = std::time::Instant::now();
write_json(p, &report)?;
report_write_ms = Some(started.elapsed().as_millis() as u64);
if let Some(parent) = p.parent() {
let stage_timing = RunStageTiming {
validation_ms,
report_build_ms,
report_write_ms,
ccr_build_ms,
ccr_write_ms,
cir_build_cir_ms,
cir_write_cir_ms,
cir_total_ms,
total_ms: total_started.elapsed().as_millis() as u64,
publication_points,
repo_sync_ms_total,
publication_point_repo_sync_ms_total,
download_event_count,
rrdp_download_ms_total,
rsync_download_ms_total,
download_bytes_total,
};
let stage_timing_path = parent.join("stage-timing.json");
std::fs::write(
&stage_timing_path,
serde_json::to_vec_pretty(&stage_timing).map_err(|e| e.to_string())?,
)
.map_err(|e| {
format!(
"write stage timing failed: {}: {e}",
stage_timing_path.display()
)
})?;
eprintln!("analysis: wrote {}", stage_timing_path.display());
}
}
let stage_timing = RunStageTiming {
validation_ms,
report_build_ms,
report_write_ms,
ccr_build_ms,
ccr_build_breakdown,
ccr_write_ms,
cir_build_cir_ms,
cir_write_cir_ms,
cir_total_ms,
total_ms: total_started.elapsed().as_millis() as u64,
publication_points,
repo_sync_ms_total,
publication_point_repo_sync_ms_total,
download_event_count,
rrdp_download_ms_total,
rsync_download_ms_total,
download_bytes_total,
};
write_stage_timing(args.report_json_path.as_deref(), &stage_timing)?;
if let Some((out_dir, t)) = timing.as_ref() {
t.record_count("vrps", report.vrps.len() as u64);
@ -1773,6 +1982,43 @@ mod tests {
);
}
#[test]
fn parse_accepts_report_json_compact_when_report_json_is_set() {
let argv = vec![
"rpki".to_string(),
"--db".to_string(),
"db".to_string(),
"--tal-url".to_string(),
"https://example.test/x.tal".to_string(),
"--report-json".to_string(),
"out/report.json".to_string(),
"--report-json-compact".to_string(),
];
let args = parse_args(&argv).expect("parse args");
assert_eq!(
args.report_json_path.as_deref(),
Some(std::path::Path::new("out/report.json"))
);
assert!(args.report_json_compact);
}
#[test]
fn parse_rejects_report_json_compact_without_report_json() {
let argv = vec![
"rpki".to_string(),
"--db".to_string(),
"db".to_string(),
"--tal-url".to_string(),
"https://example.test/x.tal".to_string(),
"--report-json-compact".to_string(),
];
let err = parse_args(&argv).expect_err("compact flag without report path should fail");
assert!(
err.contains("--report-json-compact requires --report-json"),
"{err}"
);
}
#[test]
fn parse_accepts_external_raw_store_db() {
let argv = vec![
@ -2485,8 +2731,7 @@ mod tests {
assert!(err.contains("read policy file failed"), "{err}");
}
#[test]
fn build_report_and_helpers_work_on_synthetic_output() {
fn synthetic_post_validation_shared() -> PostValidationShared {
let tal_bytes = std::fs::read(
std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures/tal/apnic-rfc7730-https.tal"),
@ -2506,11 +2751,9 @@ mod tests {
let tree = crate::validation::tree::TreeRunOutput {
instances_processed: 1,
instances_failed: 0,
warnings: vec![
crate::report::Warning::new("synthetic warning")
.with_rfc_refs(&[crate::report::RfcRef("RFC 6487 §4.8.8.1")])
.with_context("rsync://example.test/repo/pp/"),
],
warnings: vec![crate::report::Warning::new("synthetic warning")
.with_rfc_refs(&[crate::report::RfcRef("RFC 6487 §4.8.8.1")])
.with_context("rsync://example.test/repo/pp/")],
vrps: vec![crate::validation::objects::Vrp {
asn: 64496,
prefix: crate::data_model::roa::IpPrefix {
@ -2528,10 +2771,13 @@ mod tests {
};
let mut pp1 = crate::audit::PublicationPointAudit::default();
pp1.source = "fresh".to_string();
pp1.rrdp_notification_uri = Some("https://example.test/n1.xml".to_string());
let mut pp2 = crate::audit::PublicationPointAudit::default();
pp2.source = "fresh".to_string();
pp2.rrdp_notification_uri = Some("https://example.test/n1.xml".to_string());
let mut pp3 = crate::audit::PublicationPointAudit::default();
pp3.source = "fresh".to_string();
pp3.rrdp_notification_uri = Some("https://example.test/n2.xml".to_string());
let out = crate::validation::run_tree_from_tal::RunTreeFromTalAuditOutput {
@ -2542,11 +2788,54 @@ mod tests {
downloads: Vec::new(),
download_stats: crate::audit::AuditDownloadStats::default(),
current_repo_objects: Vec::new(),
ccr_accumulator: None,
};
PostValidationShared::from_run_output(out)
}
fn sample_cli_ccr_accumulator() -> CcrAccumulator {
let tal_bytes = std::fs::read(
std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures/tal/apnic-rfc7730-https.tal"),
)
.expect("read tal fixture");
let ta_der = std::fs::read(
std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures/ta/apnic-ta.cer"),
)
.expect("read ta fixture");
let discovery = crate::validation::from_tal::discover_root_ca_instance_from_tal_and_ta_der(
&tal_bytes, &ta_der, None,
)
.expect("discover root");
let mut accumulator = CcrAccumulator::new(vec![discovery.trust_anchor.clone()]);
let projection = crate::storage::VcirCcrManifestProjection {
manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(),
manifest_sha256: vec![0x44; 32],
manifest_size: 2048,
manifest_ee_aki: vec![0x55; 20],
manifest_number_be: vec![1],
manifest_this_update: crate::storage::PackTime::from_utc_offset_datetime(
time::OffsetDateTime::now_utc(),
),
manifest_sia_locations_der: vec![vec![
0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05,
b'r', b's', b'y', b'n', b'c',
]],
subordinate_skis: vec![vec![0x33; 20]],
};
accumulator
.append_manifest_projection(&projection)
.expect("append manifest projection");
accumulator
}
#[test]
fn build_report_and_helpers_work_on_synthetic_output() {
let shared = synthetic_post_validation_shared();
let policy = Policy::default();
let validation_time = time::OffsetDateTime::now_utc();
let report = build_report(&policy, validation_time, out);
let report = build_report(&policy, validation_time, &shared);
assert_eq!(unique_rrdp_repos(&report), 2);
assert_eq!(report.vrps.len(), 1);
@ -2555,6 +2844,92 @@ mod tests {
print_summary(&report);
}
#[test]
fn run_report_task_and_stage_timing_work() {
let shared = synthetic_post_validation_shared();
let policy = Policy::default();
let validation_time = time::OffsetDateTime::now_utc();
let dir = tempfile::tempdir().expect("tmpdir");
let report_path = dir.path().join("report.json");
let report_output = run_report_task(
&policy,
validation_time,
&shared,
Some(&report_path),
ReportJsonFormat::Compact,
)
.expect("run report task");
assert_eq!(report_output.report.vrps.len(), 1);
assert_eq!(report_output.report.aspas.len(), 1);
assert!(report_output.report_write_ms.is_some());
let report_json = std::fs::read_to_string(&report_path).expect("read report json");
assert!(!report_json.contains('\n'), "{report_json}");
let stage_timing = RunStageTiming {
validation_ms: 1,
report_build_ms: report_output.report_build_ms,
report_write_ms: report_output.report_write_ms,
ccr_build_ms: Some(2),
ccr_build_breakdown: None,
ccr_write_ms: Some(3),
cir_build_cir_ms: Some(4),
cir_write_cir_ms: Some(5),
cir_total_ms: Some(6),
total_ms: 7,
publication_points: shared.publication_points.len(),
repo_sync_ms_total: 8,
publication_point_repo_sync_ms_total: 9,
download_event_count: 10,
rrdp_download_ms_total: 11,
rsync_download_ms_total: 12,
download_bytes_total: 13,
};
write_stage_timing(Some(&report_path), &stage_timing).expect("write stage timing");
let stage_timing_json =
std::fs::read_to_string(dir.path().join("stage-timing.json")).expect("read timing");
assert!(stage_timing_json.contains("\"validation_ms\""));
assert!(stage_timing_json.contains("\"ccr_build_ms\""));
}
#[test]
fn run_ccr_task_uses_accumulator_when_phase2_output_contains_reuse_sources() {
let mut shared = synthetic_post_validation_shared();
shared.ccr_accumulator = Some(sample_cli_ccr_accumulator());
let mut publication_points = shared
.publication_points
.iter()
.cloned()
.collect::<Vec<_>>();
publication_points[1].source = "vcir_current_instance".to_string();
publication_points[2].source = "failed_no_cache".to_string();
shared.publication_points = publication_points.into();
let dir = tempfile::tempdir().expect("tmpdir");
let ccr_path = dir.path().join("result.ccr");
let store = RocksStore::open(&dir.path().join("db")).expect("open empty store");
let output = run_ccr_task(
&store,
&shared,
Some(&ccr_path),
time::OffsetDateTime::now_utc(),
)
.expect("run ccr task");
assert!(output.ccr_build_ms.is_some());
assert!(output.ccr_build_breakdown.is_none());
let der = std::fs::read(&ccr_path).expect("read ccr");
let ci = crate::ccr::decode_content_info(&der).expect("decode ccr");
assert_eq!(
ci.content
.mfts
.as_ref()
.map(|manifest_state| manifest_state.mis.len()),
Some(1)
);
}
#[test]
fn write_json_writes_report() {
let report = AuditReportV2 {
@ -2577,11 +2952,19 @@ mod tests {
};
let dir = tempfile::tempdir().expect("tmpdir");
let p = dir.path().join("report.json");
write_json(&p, &report).expect("write json");
let s = std::fs::read_to_string(&p).expect("read report");
assert!(s.contains("\"format_version\""));
assert!(s.contains("\"policy\""));
let pretty_path = dir.path().join("report-pretty.json");
write_json(&pretty_path, &report, ReportJsonFormat::Pretty).expect("write pretty json");
let pretty = std::fs::read_to_string(&pretty_path).expect("read pretty report");
assert!(pretty.contains("\"format_version\""));
assert!(pretty.contains("\"policy\""));
assert!(pretty.contains("\n \"format_version\""), "{pretty}");
let compact_path = dir.path().join("report-compact.json");
write_json(&compact_path, &report, ReportJsonFormat::Compact).expect("write compact json");
let compact = std::fs::read_to_string(&compact_path).expect("read compact report");
assert!(compact.contains("\"format_version\""));
assert!(compact.contains("\"policy\""));
assert!(!compact.contains('\n'), "{compact}");
}
#[test]

357
src/storage.rs
View File

@ -2,13 +2,14 @@ use std::collections::HashSet;
use std::path::Path;
use rocksdb::{
ColumnFamily, ColumnFamilyDescriptor, DB, DBCompressionType, Direction, IteratorMode, Options,
WriteBatch,
ColumnFamily, ColumnFamilyDescriptor, DBCompressionType, Direction, IteratorMode, Options,
WriteBatch, DB,
};
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use sha2::Digest;
use crate::blob_store::{ExternalRawStoreDb, RawObjectStore};
use crate::data_model::common::der_take_tlv;
use crate::data_model::rc::{AsResourceSet, IpResourceSet};
pub const CF_REPOSITORY_VIEW: &str = "repository_view";
@ -231,6 +232,69 @@ impl ValidatedManifestMeta {
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct VcirCcrManifestProjection {
pub manifest_rsync_uri: String,
pub manifest_sha256: Vec<u8>,
pub manifest_size: u64,
pub manifest_ee_aki: Vec<u8>,
pub manifest_number_be: Vec<u8>,
pub manifest_this_update: PackTime,
pub manifest_sia_locations_der: Vec<Vec<u8>>,
pub subordinate_skis: Vec<Vec<u8>>,
}
impl VcirCcrManifestProjection {
pub fn validate_internal(&self) -> StorageResult<()> {
validate_non_empty(
"vcir.ccr_manifest_projection.manifest_rsync_uri",
&self.manifest_rsync_uri,
)?;
validate_sha256_digest_bytes(
"vcir.ccr_manifest_projection.manifest_sha256",
&self.manifest_sha256,
)?;
if self.manifest_size < 1000 {
return Err(StorageError::InvalidData {
entity: "vcir.ccr_manifest_projection.manifest_size",
detail: format!("must be >= 1000, got {}", self.manifest_size),
});
}
validate_fixed_len_bytes(
"vcir.ccr_manifest_projection.manifest_ee_aki",
&self.manifest_ee_aki,
20,
)?;
validate_manifest_number_be(
"vcir.ccr_manifest_projection.manifest_number_be",
&self.manifest_number_be,
)?;
parse_time(
"vcir.ccr_manifest_projection.manifest_this_update",
&self.manifest_this_update,
)?;
if self.manifest_sia_locations_der.is_empty() {
return Err(StorageError::InvalidData {
entity: "vcir.ccr_manifest_projection.manifest_sia_locations_der",
detail: "must contain at least one AccessDescription".to_string(),
});
}
for location in &self.manifest_sia_locations_der {
validate_full_der_with_tag(
"vcir.ccr_manifest_projection.manifest_sia_locations_der[]",
location,
Some(0x30),
)?;
}
validate_sorted_unique_fixed_len_bytes(
"vcir.ccr_manifest_projection.subordinate_skis",
&self.subordinate_skis,
20,
)?;
Ok(())
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct VcirInstanceGate {
pub manifest_next_update: PackTime,
@ -469,6 +533,7 @@ pub struct ValidatedCaInstanceResult {
pub current_manifest_rsync_uri: String,
pub current_crl_rsync_uri: String,
pub validated_manifest_meta: ValidatedManifestMeta,
pub ccr_manifest_projection: VcirCcrManifestProjection,
pub instance_gate: VcirInstanceGate,
pub child_entries: Vec<VcirChildEntry>,
pub local_outputs: Vec<VcirLocalOutput>,
@ -497,6 +562,7 @@ impl ValidatedCaInstanceResult {
)?;
validate_non_empty("vcir.current_crl_rsync_uri", &self.current_crl_rsync_uri)?;
self.validated_manifest_meta.validate_internal()?;
self.ccr_manifest_projection.validate_internal()?;
self.instance_gate.validate_internal()?;
let expected_manifest_next = self
@ -1554,6 +1620,75 @@ fn validate_manifest_number_be(field: &'static str, value: &[u8]) -> StorageResu
Ok(())
}
fn validate_sha256_digest_bytes(field: &'static str, value: &[u8]) -> StorageResult<()> {
if value.len() != 32 {
return Err(StorageError::InvalidData {
entity: field,
detail: format!("must be 32 bytes, got {}", value.len()),
});
}
Ok(())
}
fn validate_fixed_len_bytes(
field: &'static str,
value: &[u8],
expected_len: usize,
) -> StorageResult<()> {
if value.len() != expected_len {
return Err(StorageError::InvalidData {
entity: field,
detail: format!("must be {expected_len} bytes, got {}", value.len()),
});
}
Ok(())
}
fn validate_sorted_unique_fixed_len_bytes(
field: &'static str,
values: &[Vec<u8>],
expected_len: usize,
) -> StorageResult<()> {
for value in values {
validate_fixed_len_bytes(field, value, expected_len)?;
}
for window in values.windows(2) {
if window[0] >= window[1] {
return Err(StorageError::InvalidData {
entity: field,
detail: "must be strictly sorted and unique".to_string(),
});
}
}
Ok(())
}
fn validate_full_der_with_tag(
field: &'static str,
der: &[u8],
expected_tag: Option<u8>,
) -> StorageResult<()> {
let (tag, _value, rem) = der_take_tlv(der).map_err(|detail| StorageError::InvalidData {
entity: field,
detail,
})?;
if !rem.is_empty() {
return Err(StorageError::InvalidData {
entity: field,
detail: "trailing bytes after DER object".to_string(),
});
}
if let Some(expected_tag) = expected_tag {
if tag != expected_tag {
return Err(StorageError::InvalidData {
entity: field,
detail: format!("unexpected tag 0x{tag:02X}, expected 0x{expected_tag:02X}"),
});
}
}
Ok(())
}
fn parse_time(field: &'static str, value: &PackTime) -> StorageResult<time::OffsetDateTime> {
value.parse().map_err(|detail| StorageError::InvalidData {
entity: field,
@ -1761,10 +1896,31 @@ mod tests {
}
}
fn sample_ccr_manifest_projection(
manifest_rsync_uri: &str,
manifest_this_update: PackTime,
subordinate_skis: Vec<Vec<u8>>,
) -> VcirCcrManifestProjection {
VcirCcrManifestProjection {
manifest_rsync_uri: manifest_rsync_uri.to_string(),
manifest_sha256: vec![0x11; 32],
manifest_size: 4096,
manifest_ee_aki: vec![0x22; 20],
manifest_number_be: vec![3],
manifest_this_update,
manifest_sia_locations_der: vec![vec![
0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05,
b'r', b's', b'y', b'n', b'c',
]],
subordinate_skis,
}
}
fn sample_vcir(manifest_rsync_uri: &str) -> ValidatedCaInstanceResult {
let roa_bytes = b"roa-object".to_vec();
let ee_bytes = b"ee-cert".to_vec();
let child_bytes = b"child-cert".to_vec();
let child_ski = "1234567890abcdef1234567890abcdef12345678".to_string();
ValidatedCaInstanceResult {
manifest_rsync_uri: manifest_rsync_uri.to_string(),
parent_manifest_rsync_uri: Some(
@ -1782,6 +1938,11 @@ mod tests {
validated_manifest_this_update: pack_time(0),
validated_manifest_next_update: pack_time(24),
},
ccr_manifest_projection: sample_ccr_manifest_projection(
manifest_rsync_uri,
pack_time(0),
vec![hex::decode(&child_ski).expect("decode child ski")],
),
instance_gate: VcirInstanceGate {
manifest_next_update: pack_time(24),
current_crl_next_update: pack_time(12),
@ -1792,7 +1953,7 @@ mod tests {
child_manifest_rsync_uri: "rsync://example.test/repo/child/child.mft".to_string(),
child_cert_rsync_uri: "rsync://example.test/repo/child/child.cer".to_string(),
child_cert_hash: sha256_hex(&child_bytes),
child_ski: "1234567890abcdef1234567890abcdef12345678".to_string(),
child_ski,
child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(),
child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(),
child_rrdp_notification_uri: Some(
@ -1863,6 +2024,51 @@ mod tests {
}
}
#[test]
fn vcir_ccr_manifest_projection_validate_accepts_valid_projection() {
let projection = sample_ccr_manifest_projection(
"rsync://example.test/repo/current.mft",
pack_time(0),
vec![vec![0x33; 20], vec![0x44; 20]],
);
projection.validate_internal().expect("valid projection");
}
#[test]
fn vcir_ccr_manifest_projection_validate_rejects_invalid_fields() {
let mut bad_hash = sample_ccr_manifest_projection(
"rsync://example.test/repo/current.mft",
pack_time(0),
vec![vec![0x33; 20]],
);
bad_hash.manifest_sha256 = vec![0x11; 31];
assert!(matches!(
bad_hash.validate_internal(),
Err(StorageError::InvalidData { .. })
));
let mut bad_locations = sample_ccr_manifest_projection(
"rsync://example.test/repo/current.mft",
pack_time(0),
vec![vec![0x33; 20]],
);
bad_locations.manifest_sia_locations_der = vec![vec![0x04, 0x00]];
assert!(matches!(
bad_locations.validate_internal(),
Err(StorageError::InvalidData { .. })
));
let bad_subordinates = sample_ccr_manifest_projection(
"rsync://example.test/repo/current.mft",
pack_time(0),
vec![vec![0x44; 20], vec![0x33; 20]],
);
assert!(matches!(
bad_subordinates.validate_internal(),
Err(StorageError::InvalidData { .. })
));
}
fn sample_audit_rule_entry(kind: AuditRuleKind) -> AuditRuleIndexEntry {
AuditRuleIndexEntry {
kind,
@ -1964,12 +2170,10 @@ mod tests {
store
.delete_repository_view_entry(&entry1.rsync_uri)
.expect("delete repository view entry1");
assert!(
store
.get_repository_view_entry(&entry1.rsync_uri)
.expect("get deleted repository view entry1")
.is_none()
);
assert!(store
.get_repository_view_entry(&entry1.rsync_uri)
.expect("get deleted repository view entry1")
.is_none());
let raw = sample_raw_by_hash_entry(b"raw-der-object".to_vec());
store
@ -2026,12 +2230,10 @@ mod tests {
store.get_blob_bytes(&hash).expect("get blob bytes"),
Some(bytes.clone())
);
assert!(
store
.get_raw_by_hash_entry(&hash)
.expect("get raw entry")
.is_none()
);
assert!(store
.get_raw_by_hash_entry(&hash)
.expect("get raw entry")
.is_none());
}
#[test]
@ -2100,10 +2302,7 @@ mod tests {
let batch = store
.get_blob_bytes_batch(&[blob_hash.clone(), raw.sha256_hex.clone(), "00".repeat(32)])
.expect("get blob bytes batch");
assert_eq!(
batch,
vec![Some(blob_bytes), Some(raw.bytes.clone()), None]
);
assert_eq!(batch, vec![Some(blob_bytes), Some(raw.bytes.clone()), None]);
}
#[test]
@ -2156,12 +2355,10 @@ mod tests {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
assert!(
store
.get_blob_bytes_batch(&[])
.expect("empty blob batch request")
.is_empty()
);
assert!(store
.get_blob_bytes_batch(&[])
.expect("empty blob batch request")
.is_empty());
}
#[test]
@ -2232,7 +2429,10 @@ mod tests {
.delete_raw_by_hash_entry(&raw.sha256_hex)
.expect("delete external raw entry");
assert!(store.get_raw_by_hash_entry(&raw.sha256_hex).unwrap().is_none());
assert!(store
.get_raw_by_hash_entry(&raw.sha256_hex)
.unwrap()
.is_none());
assert!(store.get_blob_bytes(&raw.sha256_hex).unwrap().is_none());
}
@ -2296,12 +2496,10 @@ mod tests {
store
.delete_vcir(&vcir.manifest_rsync_uri)
.expect("delete vcir");
assert!(
store
.get_vcir(&vcir.manifest_rsync_uri)
.expect("get deleted vcir")
.is_none()
);
assert!(store
.get_vcir(&vcir.manifest_rsync_uri)
.expect("get deleted vcir")
.is_none());
}
#[test]
@ -2356,12 +2554,10 @@ mod tests {
store
.delete_audit_rule_index_entry(AuditRuleKind::Roa, &roa.rule_hash)
.expect("delete roa audit rule entry");
assert!(
store
.get_audit_rule_index_entry(AuditRuleKind::Roa, &roa.rule_hash)
.expect("get deleted roa audit rule entry")
.is_none()
);
assert!(store
.get_audit_rule_index_entry(AuditRuleKind::Roa, &roa.rule_hash)
.expect("get deleted roa audit rule entry")
.is_none());
let mut invalid = sample_audit_rule_entry(AuditRuleKind::Roa);
invalid.rule_hash = "bad".to_string();
@ -2395,15 +2591,10 @@ mod tests {
store
.replace_vcir_and_audit_rule_indexes(None, &previous)
.expect("store previous vcir");
assert!(
store
.get_audit_rule_index_entry(
AuditRuleKind::Roa,
&previous.local_outputs[0].rule_hash
)
.expect("get old audit entry")
.is_some()
);
assert!(store
.get_audit_rule_index_entry(AuditRuleKind::Roa, &previous.local_outputs[0].rule_hash)
.expect("get old audit entry")
.is_some());
let mut current = sample_vcir("rsync://example.test/repo/current.mft");
current.local_outputs = vec![VcirLocalOutput {
@ -2429,24 +2620,14 @@ mod tests {
.expect("get replaced vcir")
.expect("vcir exists");
assert_eq!(got, current);
assert!(
store
.get_audit_rule_index_entry(
AuditRuleKind::Roa,
&previous.local_outputs[0].rule_hash
)
.expect("get deleted old audit entry")
.is_none()
);
assert!(
store
.get_audit_rule_index_entry(
AuditRuleKind::Aspa,
&current.local_outputs[0].rule_hash
)
.expect("get new audit entry")
.is_some()
);
assert!(store
.get_audit_rule_index_entry(AuditRuleKind::Roa, &previous.local_outputs[0].rule_hash)
.expect("get deleted old audit entry")
.is_none());
assert!(store
.get_audit_rule_index_entry(AuditRuleKind::Aspa, &current.local_outputs[0].rule_hash)
.expect("get new audit entry")
.is_some());
}
#[test]
@ -2588,12 +2769,10 @@ mod tests {
store
.delete_rrdp_uri_owner_record(&member1.rsync_uri)
.expect("delete uri owner record");
assert!(
store
.get_rrdp_uri_owner_record(&member1.rsync_uri)
.expect("get deleted uri owner")
.is_none()
);
assert!(store
.get_rrdp_uri_owner_record(&member1.rsync_uri)
.expect("get deleted uri owner")
.is_none());
let mut invalid_source =
sample_rrdp_source_record("https://invalid.example/notification.xml");
@ -2724,21 +2903,15 @@ mod tests {
]
);
assert!(
store
.is_current_rrdp_source_member(notify_uri, &present_a.rsync_uri)
.expect("current a")
);
assert!(
!store
.is_current_rrdp_source_member(notify_uri, &withdrawn_b.rsync_uri)
.expect("withdrawn b")
);
assert!(
!store
.is_current_rrdp_source_member(notify_uri, &other_source.rsync_uri)
.expect("other source")
);
assert!(store
.is_current_rrdp_source_member(notify_uri, &present_a.rsync_uri)
.expect("current a"));
assert!(!store
.is_current_rrdp_source_member(notify_uri, &withdrawn_b.rsync_uri)
.expect("withdrawn b"));
assert!(!store
.is_current_rrdp_source_member(notify_uri, &other_source.rsync_uri)
.expect("other source"));
}
#[test]
@ -2904,12 +3077,10 @@ mod tests {
assert_eq!(got.current_hash_hex, hash);
assert_eq!(got.current_hash, compute_sha256_32(&bytes));
assert_eq!(got.bytes, bytes);
assert!(
store
.get_raw_by_hash_entry(&got.current_hash_hex)
.expect("get raw entry")
.is_none()
);
assert!(store
.get_raw_by_hash_entry(&got.current_hash_hex)
.expect("get raw entry")
.is_none());
}
#[test]

36
src/validation/manifest.rs
View File

@ -4,7 +4,7 @@ use crate::data_model::signed_object::SignedObjectVerifyError;
use crate::policy::{CaFailedFetchPolicy, Policy};
use crate::report::{RfcRef, Warning};
use crate::storage::{PackFile, PackTime, RocksStore, StorageError, VcirArtifactRole};
use crate::validation::cert_path::{CertPathError, validate_signed_object_ee_cert_path_fast};
use crate::validation::cert_path::{validate_signed_object_ee_cert_path_fast, CertPathError};
use crate::validation::publication_point::PublicationPointSnapshot;
use sha2::Digest;
use std::cmp::Ordering;
@ -398,11 +398,10 @@ pub fn process_manifest_publication_point_after_repo_sync(
Err(ManifestProcessError::StopAllOutput(fresh_err))
}
CaFailedFetchPolicy::ReuseCurrentInstanceVcir => {
let mut warnings = vec![
Warning::new(format!("manifest failed fetch: {fresh_err}"))
let mut warnings =
vec![Warning::new(format!("manifest failed fetch: {fresh_err}"))
.with_rfc_refs(&[RfcRef("RFC 9286 §6.6")])
.with_context(manifest_rsync_uri),
];
.with_context(manifest_rsync_uri)];
match load_current_instance_vcir_publication_point(
store,
@ -932,7 +931,8 @@ mod tests {
use crate::storage::{
PackFile, PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult,
ValidatedManifestMeta, VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus,
VcirAuditSummary, VcirInstanceGate, VcirRelatedArtifact, VcirSummary,
VcirAuditSummary, VcirCcrManifestProjection, VcirInstanceGate, VcirRelatedArtifact,
VcirSummary,
};
use std::path::Path;
@ -1039,6 +1039,19 @@ mod tests {
) -> ValidatedCaInstanceResult {
let gate_time =
PackTime::from_utc_offset_datetime(validation_time + time::Duration::hours(1));
let ccr_manifest_projection = VcirCcrManifestProjection {
manifest_rsync_uri: manifest_rsync_uri.to_string(),
manifest_sha256: hex::decode(manifest_sha256).expect("decode manifest sha256"),
manifest_size: 2048,
manifest_ee_aki: vec![0x11; 20],
manifest_number_be: vec![1],
manifest_this_update: PackTime::from_utc_offset_datetime(validation_time),
manifest_sia_locations_der: vec![vec![
0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05,
b'r', b's', b'y', b'n', b'c',
]],
subordinate_skis: Vec::new(),
};
ValidatedCaInstanceResult {
manifest_rsync_uri: manifest_rsync_uri.to_string(),
parent_manifest_rsync_uri: None,
@ -1054,6 +1067,7 @@ mod tests {
validated_manifest_this_update: PackTime::from_utc_offset_datetime(validation_time),
validated_manifest_next_update: gate_time.clone(),
},
ccr_manifest_projection,
instance_gate: VcirInstanceGate {
manifest_next_update: gate_time.clone(),
current_crl_next_update: gate_time.clone(),
@ -1313,12 +1327,10 @@ mod tests {
.apply_repository_view_entries(&entries)
.expect("apply current index");
assert!(
store
.get_repository_view_entry(&manifest_rsync_uri)
.expect("get repository view")
.is_none()
);
assert!(store
.get_repository_view_entry(&manifest_rsync_uri)
.expect("get repository view")
.is_none());
let (fresh, _timing) = try_build_fresh_publication_point_with_timing(
&store,

1
src/validation/run.rs
View File

@ -77,6 +77,7 @@ pub fn run_publication_point_once(
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let result = runner

43
src/validation/run_tree_from_tal.rs
View File

@ -3,6 +3,7 @@ use url::Url;
use crate::analysis::timing::TimingHandle;
use crate::audit::PublicationPointAudit;
use crate::audit_downloads::DownloadLogHandle;
use crate::ccr::CcrAccumulator;
use crate::current_repo_index::{CurrentRepoIndexHandle, CurrentRepoObject};
use crate::data_model::ta::TrustAnchor;
use crate::parallel::config::{ParallelPhase1Config, ParallelPhase2Config};
@ -90,6 +91,7 @@ pub struct RunTreeFromTalAuditOutput {
pub downloads: Vec<crate::audit::AuditDownloadEvent>,
pub download_stats: crate::audit::AuditDownloadStats,
pub current_repo_objects: Vec<CurrentRepoObject>,
pub ccr_accumulator: Option<CcrAccumulator>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
@ -118,6 +120,7 @@ fn make_live_runner<'a>(
current_repo_index: Option<CurrentRepoIndexHandle>,
repo_sync_runtime: Option<Arc<dyn RepoSyncRuntime>>,
parallel_phase2_config: Option<ParallelPhase2Config>,
ccr_accumulator: Option<CcrAccumulator>,
) -> Rpkiv1PublicationPointRunner<'a> {
let parallel_roa_worker_pool = parallel_phase2_config
.as_ref()
@ -140,6 +143,7 @@ fn make_live_runner<'a>(
repo_sync_runtime,
parallel_phase2_config,
parallel_roa_worker_pool,
ccr_accumulator: ccr_accumulator.map(Mutex::new),
}
}
@ -291,6 +295,7 @@ pub fn run_tree_from_tal_url_serial(
None,
None,
None,
None,
);
let root = root_handle_from_trust_anchor(
@ -327,6 +332,7 @@ pub fn run_tree_from_tal_url_serial_audit(
None,
None,
None,
None,
);
let root = root_handle_from_trust_anchor(
@ -350,6 +356,7 @@ pub fn run_tree_from_tal_url_serial_audit(
downloads,
download_stats,
current_repo_objects: Vec::new(),
ccr_accumulator: None,
})
}
@ -379,6 +386,7 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing(
None,
None,
None,
None,
);
let root = root_handle_from_trust_anchor(
@ -403,6 +411,7 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing(
downloads,
download_stats,
current_repo_objects: Vec::new(),
ccr_accumulator: None,
})
}
@ -446,6 +455,7 @@ where
Some(current_repo_index),
Some(runtime),
phase2_config,
phase2_enabled.then(|| CcrAccumulator::new(vec![discovery.trust_anchor.clone()])),
);
let root = root_handle_from_trust_anchor(
@ -472,6 +482,9 @@ where
downloads,
download_stats,
current_repo_objects: snapshot_current_repo_objects(Some(&current_repo_index_for_output)),
ccr_accumulator: phase2_enabled
.then(|| runner.ccr_accumulator_snapshot())
.flatten(),
})
}
@ -532,6 +545,14 @@ where
Some(current_repo_index),
Some(runtime),
phase2_config,
phase2_enabled.then(|| {
CcrAccumulator::new(
discoveries
.iter()
.map(|item| item.trust_anchor.clone())
.collect::<Vec<_>>(),
)
}),
);
let TreeRunAuditOutput {
@ -552,6 +573,9 @@ where
downloads,
download_stats,
current_repo_objects: snapshot_current_repo_objects(Some(&current_repo_index_for_output)),
ccr_accumulator: phase2_enabled
.then(|| runner.ccr_accumulator_snapshot())
.flatten(),
})
}
@ -792,6 +816,7 @@ pub fn run_tree_from_tal_and_ta_der_serial(
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let root = root_handle_from_trust_anchor(
@ -842,6 +867,7 @@ pub fn run_tree_from_tal_bytes_serial_audit(
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let root = root_handle_from_trust_anchor(
@ -865,6 +891,7 @@ pub fn run_tree_from_tal_bytes_serial_audit(
downloads,
download_stats,
current_repo_objects: Vec::new(),
ccr_accumulator: None,
})
}
@ -908,6 +935,7 @@ pub fn run_tree_from_tal_bytes_serial_audit_with_timing(
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let root = root_handle_from_trust_anchor(
@ -933,6 +961,7 @@ pub fn run_tree_from_tal_bytes_serial_audit_with_timing(
downloads,
download_stats,
current_repo_objects: Vec::new(),
ccr_accumulator: None,
})
}
@ -969,6 +998,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit(
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let root = root_handle_from_trust_anchor(
@ -992,6 +1022,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit(
downloads,
download_stats,
current_repo_objects: Vec::new(),
ccr_accumulator: None,
})
}
@ -1031,6 +1062,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing(
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let root = root_handle_from_trust_anchor(
@ -1055,6 +1087,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing(
downloads,
download_stats,
current_repo_objects: Vec::new(),
ccr_accumulator: None,
})
}
@ -1100,6 +1133,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial(
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let root = root_handle_from_trust_anchor(
@ -1156,6 +1190,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit(
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let root = root_handle_from_trust_anchor(
@ -1179,6 +1214,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit(
downloads,
download_stats,
current_repo_objects: Vec::new(),
ccr_accumulator: None,
})
}
@ -1228,6 +1264,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing(
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let root = root_handle_from_trust_anchor(
@ -1252,6 +1289,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing(
downloads,
download_stats,
current_repo_objects: Vec::new(),
ccr_accumulator: None,
})
}
@ -1283,6 +1321,7 @@ fn build_payload_replay_runner<'a>(
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
}
}
@ -1314,6 +1353,7 @@ fn build_payload_delta_replay_runner<'a>(
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
}
}
@ -1345,6 +1385,7 @@ fn build_payload_delta_replay_current_store_runner<'a>(
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
}
}
@ -1461,6 +1502,7 @@ fn run_payload_delta_replay_audit_inner(
downloads,
download_stats,
current_repo_objects: Vec::new(),
ccr_accumulator: None,
})
}
@ -1605,6 +1647,7 @@ fn run_payload_delta_replay_step_audit_inner(
downloads,
download_stats,
current_repo_objects: Vec::new(),
ccr_accumulator: None,
})
}

494
src/validation/tree_runner.rs
View File

@ -4,11 +4,12 @@ use crate::audit::{
ObjectAuditEntry, PublicationPointAudit,
};
use crate::audit_downloads::DownloadLogHandle;
use crate::ccr::CcrAccumulator;
use crate::current_repo_index::CurrentRepoIndexHandle;
use crate::data_model::aspa::AspaObject;
use crate::data_model::crl::RpkixCrl;
use crate::data_model::manifest::ManifestObject;
use crate::data_model::rc::ResourceCertificate;
use crate::data_model::rc::{AccessDescription, ResourceCertificate, SubjectInfoAccess};
use crate::data_model::roa::{RoaAfi, RoaObject};
use crate::data_model::router_cert::{
BgpsecRouterCertificate, BgpsecRouterCertificateDecodeError, BgpsecRouterCertificatePathError,
@ -23,8 +24,9 @@ use crate::replay::delta_archive::ReplayDeltaArchiveIndex;
use crate::report::{RfcRef, Warning};
use crate::storage::{
PackFile, PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, VcirArtifactKind,
VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, VcirChildEntry,
VcirInstanceGate, VcirLocalOutput, VcirOutputType, VcirRelatedArtifact, VcirSummary,
VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, VcirCcrManifestProjection,
VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirOutputType, VcirRelatedArtifact,
VcirSummary,
};
use crate::sync::repo::{
sync_publication_point, sync_publication_point_replay, sync_publication_point_replay_delta,
@ -49,6 +51,7 @@ use crate::validation::publication_point::PublicationPointSnapshot;
use crate::validation::tree::{
CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult, PublicationPointRunner,
};
use sha2::Digest;
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, Mutex};
@ -134,9 +137,47 @@ pub struct Rpkiv1PublicationPointRunner<'a> {
pub repo_sync_runtime: Option<Arc<dyn RepoSyncRuntime>>,
pub parallel_phase2_config: Option<ParallelPhase2Config>,
pub parallel_roa_worker_pool: Option<ParallelRoaWorkerPool>,
pub ccr_accumulator: Option<Mutex<CcrAccumulator>>,
}
impl<'a> Rpkiv1PublicationPointRunner<'a> {
pub(crate) fn ccr_accumulator_snapshot(&self) -> Option<CcrAccumulator> {
self.ccr_accumulator
.as_ref()
.and_then(|accumulator| accumulator.lock().ok().map(|guard| guard.clone()))
}
pub(crate) fn append_ccr_manifest_projection(
&self,
projection: &VcirCcrManifestProjection,
) -> Result<(), String> {
if let Some(accumulator) = self.ccr_accumulator.as_ref() {
accumulator
.lock()
.map_err(|_| "lock CCR accumulator failed".to_string())?
.append_manifest_projection(projection)?;
}
Ok(())
}
fn append_ccr_manifest_projection_from_reuse(
&self,
projection: &VcirReuseProjection,
) -> Result<(), String> {
match projection.source {
PublicationPointSource::Fresh => Err(
"invalid reuse projection source: fresh does not belong to failed-fetch reuse"
.to_string(),
),
PublicationPointSource::VcirCurrentInstance => self.append_ccr_manifest_projection(
projection.ccr_manifest_projection.as_ref().ok_or_else(|| {
"vcir current-instance reuse is missing CCR manifest projection".to_string()
})?,
),
PublicationPointSource::FailedFetchNoCache => Ok(()),
}
}
pub(crate) fn stage_fresh_publication_point_after_repo_ready(
&self,
ca: &CaInstanceHandle,
@ -237,6 +278,14 @@ impl<'a> Rpkiv1PublicationPointRunner<'a> {
.map_err(|e| format!("persist VCIR failed: {e}"))?;
let persist_vcir_ms = persist_vcir_started.elapsed().as_millis() as u64;
if self.ccr_accumulator.is_some() {
let child_entries =
build_vcir_child_entries(&discovered_children, self.validation_time)?;
let ccr_manifest_projection =
build_vcir_ccr_manifest_projection_from_fresh(ca, &pack, &child_entries)?;
self.append_ccr_manifest_projection(&ccr_manifest_projection)?;
}
let audit_build_started = std::time::Instant::now();
let audit = build_publication_point_audit_from_snapshot(
ca,
@ -745,6 +794,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
self.validation_time,
)
.map_err(|e| format!("failed fetch VCIR projection failed: {e}"))?;
self.append_ccr_manifest_projection_from_reuse(&projection)?;
let projection_ms = projection_started.elapsed().as_millis() as u64;
warnings.extend(projection.warnings.clone());
let audit_build_started = std::time::Instant::now();
@ -888,6 +938,7 @@ enum CachedIssuerCrl {
struct VcirReuseProjection {
source: PublicationPointSource,
vcir: Option<ValidatedCaInstanceResult>,
ccr_manifest_projection: Option<VcirCcrManifestProjection>,
snapshot: Option<PublicationPointSnapshot>,
objects: crate::validation::objects::ObjectsOutput,
child_audits: Vec<ObjectAuditEntry>,
@ -1872,6 +1923,19 @@ fn empty_objects_output() -> crate::validation::objects::ObjectsOutput {
}
}
fn reuse_ccr_manifest_projection_from_vcir(
ca: &CaInstanceHandle,
vcir: &ValidatedCaInstanceResult,
) -> Result<VcirCcrManifestProjection, String> {
if vcir.ccr_manifest_projection.manifest_rsync_uri != ca.manifest_rsync_uri {
return Err(format!(
"vcir CCR manifest projection URI mismatch: expected {}, got {}",
ca.manifest_rsync_uri, vcir.ccr_manifest_projection.manifest_rsync_uri
));
}
Ok(vcir.ccr_manifest_projection.clone())
}
fn project_current_instance_vcir_on_failed_fetch(
store: &RocksStore,
ca: &CaInstanceHandle,
@ -1896,6 +1960,7 @@ fn project_current_instance_vcir_on_failed_fetch(
return Ok(VcirReuseProjection {
source: PublicationPointSource::FailedFetchNoCache,
vcir: None,
ccr_manifest_projection: None,
snapshot: None,
objects: empty_objects_output(),
child_audits: Vec::new(),
@ -1915,6 +1980,7 @@ fn project_current_instance_vcir_on_failed_fetch(
return Ok(VcirReuseProjection {
source: PublicationPointSource::FailedFetchNoCache,
vcir: Some(vcir),
ccr_manifest_projection: None,
snapshot: None,
objects: empty_objects_output(),
child_audits: Vec::new(),
@ -1936,6 +2002,7 @@ fn project_current_instance_vcir_on_failed_fetch(
return Ok(VcirReuseProjection {
source: PublicationPointSource::FailedFetchNoCache,
vcir: Some(vcir),
ccr_manifest_projection: None,
snapshot: None,
objects: empty_objects_output(),
child_audits: Vec::new(),
@ -1950,6 +2017,7 @@ fn project_current_instance_vcir_on_failed_fetch(
.with_context(&ca.manifest_rsync_uri),
);
let ccr_manifest_projection = reuse_ccr_manifest_projection_from_vcir(ca, &vcir)?;
let snapshot = reconstruct_snapshot_from_vcir(store, ca, &vcir, &mut warnings);
let objects = build_objects_output_from_vcir(&vcir, validation_time, &mut warnings);
let (discovered_children, child_audits) =
@ -1958,6 +2026,7 @@ fn project_current_instance_vcir_on_failed_fetch(
Ok(VcirReuseProjection {
source: PublicationPointSource::VcirCurrentInstance,
vcir: Some(vcir),
ccr_manifest_projection: Some(ccr_manifest_projection),
snapshot,
objects,
child_audits,
@ -2510,6 +2579,8 @@ fn build_vcir_from_fresh_result_with_timing(
child_audits,
);
timing.related_artifacts_ms = related_artifacts_started.elapsed().as_millis() as u64;
let ccr_manifest_projection =
build_vcir_ccr_manifest_projection_from_fresh(ca, pack, &child_entries)?;
let accepted_object_count = related_artifacts
.iter()
.filter(|artifact| artifact.validation_status == VcirArtifactValidationStatus::Accepted)
@ -2552,6 +2623,7 @@ fn build_vcir_from_fresh_result_with_timing(
validated_manifest_this_update: pack.this_update.clone(),
validated_manifest_next_update: pack.next_update.clone(),
},
ccr_manifest_projection,
instance_gate: VcirInstanceGate {
manifest_next_update: pack.next_update.clone(),
current_crl_next_update: PackTime::from_utc_offset_datetime(
@ -2598,6 +2670,138 @@ fn build_vcir_from_fresh_result_with_timing(
Ok((vcir, timing))
}
fn build_vcir_ccr_manifest_projection_from_fresh(
ca: &CaInstanceHandle,
pack: &PublicationPointSnapshot,
child_entries: &[VcirChildEntry],
) -> Result<VcirCcrManifestProjection, String> {
let manifest = ManifestObject::decode_der(&pack.manifest_bytes)
.map_err(|e| format!("decode manifest for VCIR CCR projection failed: {e}"))?;
let ee = &manifest.signed_object.signed_data.certificates[0].resource_cert;
let manifest_ee_aki = ee
.tbs
.extensions
.authority_key_identifier
.clone()
.ok_or_else(|| "manifest EE certificate missing AuthorityKeyIdentifier".to_string())?;
let manifest_sia_locations_der = match ee
.tbs
.extensions
.subject_info_access
.as_ref()
.ok_or_else(|| "manifest EE certificate missing Subject Information Access".to_string())?
{
SubjectInfoAccess::Ee(ee_sia) => ee_sia
.access_descriptions
.iter()
.map(encode_access_description_der_for_vcir_ccr_projection)
.collect::<Result<Vec<_>, _>>()?,
SubjectInfoAccess::Ca(_) => {
return Err(
"manifest EE certificate Subject Information Access has CA variant".to_string(),
);
}
};
let mut subordinate_skis = child_entries
.iter()
.map(|child| {
hex::decode(&child.child_ski)
.map_err(|e| format!("decode child_ski for VCIR CCR projection failed: {e}"))
})
.collect::<Result<Vec<_>, _>>()?;
subordinate_skis.sort();
subordinate_skis.dedup();
Ok(VcirCcrManifestProjection {
manifest_rsync_uri: ca.manifest_rsync_uri.clone(),
manifest_sha256: sha2::Sha256::digest(&pack.manifest_bytes).to_vec(),
manifest_size: pack.manifest_bytes.len() as u64,
manifest_ee_aki,
manifest_number_be: pack.manifest_number_be.clone(),
manifest_this_update: pack.this_update.clone(),
manifest_sia_locations_der,
subordinate_skis,
})
}
fn encode_access_description_der_for_vcir_ccr_projection(
access_description: &AccessDescription,
) -> Result<Vec<u8>, String> {
let oid = encode_oid_der_for_vcir_ccr_projection(&access_description.access_method_oid)?;
let uri = encode_tlv_for_vcir_ccr_projection(
0x86,
access_description.access_location.as_bytes().to_vec(),
);
Ok(encode_sequence_for_vcir_ccr_projection(&[oid, uri]))
}
fn encode_oid_der_for_vcir_ccr_projection(oid: &str) -> Result<Vec<u8>, String> {
let arcs = oid
.split('.')
.map(|part| {
part.parse::<u64>()
.map_err(|_| format!("unsupported accessMethod OID: {oid}"))
})
.collect::<Result<Vec<_>, _>>()?;
if arcs.len() < 2 {
return Err(format!("unsupported accessMethod OID: {oid}"));
}
if arcs[0] > 2 || (arcs[0] < 2 && arcs[1] >= 40) {
return Err(format!("unsupported accessMethod OID: {oid}"));
}
let mut body = Vec::new();
body.push((arcs[0] * 40 + arcs[1]) as u8);
for arc in &arcs[2..] {
encode_base128_for_vcir_ccr_projection(*arc, &mut body);
}
Ok(encode_tlv_for_vcir_ccr_projection(0x06, body))
}
fn encode_base128_for_vcir_ccr_projection(mut value: u64, out: &mut Vec<u8>) {
let mut tmp = vec![(value & 0x7F) as u8];
value >>= 7;
while value > 0 {
tmp.push(((value & 0x7F) as u8) | 0x80);
value >>= 7;
}
tmp.reverse();
out.extend_from_slice(&tmp);
}
fn encode_sequence_for_vcir_ccr_projection(elements: &[Vec<u8>]) -> Vec<u8> {
let total_len: usize = elements.iter().map(Vec::len).sum();
let mut buf = Vec::with_capacity(total_len);
for element in elements {
buf.extend_from_slice(element);
}
encode_tlv_for_vcir_ccr_projection(0x30, buf)
}
fn encode_tlv_for_vcir_ccr_projection(tag: u8, value: Vec<u8>) -> Vec<u8> {
let mut out = Vec::with_capacity(1 + 9 + value.len());
out.push(tag);
encode_length_for_vcir_ccr_projection(value.len(), &mut out);
out.extend_from_slice(&value);
out
}
fn encode_length_for_vcir_ccr_projection(len: usize, out: &mut Vec<u8>) {
if len < 0x80 {
out.push(len as u8);
return;
}
let mut bytes = Vec::new();
let mut value = len;
while value > 0 {
bytes.push((value & 0xFF) as u8);
value >>= 8;
}
bytes.reverse();
out.push(0x80 | (bytes.len() as u8));
out.extend_from_slice(&bytes);
}
struct CurrentCrlRef<'a> {
file: &'a PackFile,
crl: RpkixCrl,
@ -3048,6 +3252,32 @@ mod tests {
}
}
fn sample_runner_with_ccr_accumulator<'a>(
store: &'a RocksStore,
policy: &'a Policy,
) -> Rpkiv1PublicationPointRunner<'a> {
Rpkiv1PublicationPointRunner {
store,
policy,
http_fetcher: &NeverHttpFetcher,
rsync_fetcher: &FailingRsyncFetcher,
validation_time: time::OffsetDateTime::now_utc(),
timing: None,
download_log: None,
replay_archive_index: None,
replay_delta_index: None,
rrdp_dedup: false,
rrdp_repo_cache: Mutex::new(HashMap::new()),
rsync_dedup: false,
rsync_repo_cache: Mutex::new(HashMap::new()),
current_repo_index: None,
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: Some(Mutex::new(CcrAccumulator::new(Vec::new()))),
}
}
fn openssl_available() -> bool {
Command::new("openssl")
.arg("version")
@ -3509,6 +3739,19 @@ authorityKeyIdentifier = keyid:always
let router_hash = sha256_hex(b"router-bytes");
let ee_hash = sha256_hex(b"ee-cert-bytes");
let gate_until = PackTime::from_utc_offset_datetime(now + time::Duration::hours(1));
let ccr_manifest_projection = VcirCcrManifestProjection {
manifest_rsync_uri: manifest_uri.clone(),
manifest_sha256: hex::decode(&manifest_hash).expect("decode manifest hash"),
manifest_size: 2048,
manifest_ee_aki: vec![0x11; 20],
manifest_number_be: vec![1],
manifest_this_update: PackTime::from_utc_offset_datetime(now),
manifest_sia_locations_der: vec![vec![
0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05,
b'r', b's', b'y', b'n', b'c',
]],
subordinate_skis: vec![vec![0x33; 20]],
};
ValidatedCaInstanceResult {
manifest_rsync_uri: manifest_uri.clone(),
parent_manifest_rsync_uri: None,
@ -3524,6 +3767,7 @@ authorityKeyIdentifier = keyid:always
validated_manifest_this_update: PackTime::from_utc_offset_datetime(now),
validated_manifest_next_update: gate_until.clone(),
},
ccr_manifest_projection,
instance_gate: VcirInstanceGate {
manifest_next_update: gate_until.clone(),
current_crl_next_update: gate_until.clone(),
@ -3901,6 +4145,22 @@ authorityKeyIdentifier = keyid:always
.expect("vcir exists");
assert_eq!(vcir.manifest_rsync_uri, pack.manifest_rsync_uri);
assert_eq!(vcir.summary.local_vrp_count as usize, objects.vrps.len());
assert_eq!(
vcir.ccr_manifest_projection.manifest_rsync_uri,
pack.manifest_rsync_uri
);
assert_eq!(
vcir.ccr_manifest_projection.manifest_number_be,
pack.manifest_number_be
);
assert_eq!(
vcir.ccr_manifest_projection.manifest_this_update,
pack.this_update
);
assert_eq!(
vcir.ccr_manifest_projection.manifest_size,
pack.manifest_bytes.len() as u64
);
let first_output = vcir.local_outputs.first().expect("local outputs stored");
assert!(store
.get_audit_rule_index_entry(crate::storage::AuditRuleKind::Roa, &first_output.rule_hash)
@ -3908,6 +4168,81 @@ authorityKeyIdentifier = keyid:always
.is_some());
}
#[test]
fn build_vcir_ccr_manifest_projection_from_fresh_real_snapshot_matches_manifest_contents() {
let (pack, issuer_ca_der, validation_time) =
cernet_publication_point_snapshot_for_vcir_tests();
let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca");
let ca = CaInstanceHandle {
depth: 0,
tal_id: "test-tal".to_string(),
parent_manifest_rsync_uri: None,
ca_certificate_der: issuer_ca_der,
ca_certificate_rsync_uri: Some(
"rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(),
),
effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(),
effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(),
rsync_base_uri: pack.publication_point_rsync_uri.clone(),
manifest_rsync_uri: pack.manifest_rsync_uri.clone(),
publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(),
rrdp_notification_uri: None,
};
let child_discovery =
discover_children_from_fresh_snapshot_with_audit(&ca, &pack, validation_time, None)
.expect("discover children");
let child_entries = build_vcir_child_entries(&child_discovery.children, validation_time)
.expect("build child entries");
let projection = build_vcir_ccr_manifest_projection_from_fresh(&ca, &pack, &child_entries)
.expect("build ccr manifest projection");
let manifest = ManifestObject::decode_der(&pack.manifest_bytes).expect("decode manifest");
let expected_locations = match manifest.signed_object.signed_data.certificates[0]
.resource_cert
.tbs
.extensions
.subject_info_access
.as_ref()
.expect("manifest sia")
{
SubjectInfoAccess::Ee(ee_sia) => ee_sia
.access_descriptions
.iter()
.map(encode_access_description_der_for_vcir_ccr_projection)
.collect::<Result<Vec<_>, _>>()
.expect("encode locations"),
SubjectInfoAccess::Ca(_) => panic!("manifest ee SIA should not be CA variant"),
};
assert_eq!(projection.manifest_rsync_uri, pack.manifest_rsync_uri);
assert_eq!(
projection.manifest_sha256,
sha2::Sha256::digest(&pack.manifest_bytes).to_vec()
);
assert_eq!(projection.manifest_size, pack.manifest_bytes.len() as u64);
assert_eq!(
projection.manifest_ee_aki,
manifest.signed_object.signed_data.certificates[0]
.resource_cert
.tbs
.extensions
.authority_key_identifier
.clone()
.expect("manifest aki")
);
assert_eq!(
projection.manifest_number_be,
manifest.manifest.manifest_number.bytes_be
);
assert_eq!(projection.manifest_this_update, pack.this_update);
assert_eq!(projection.manifest_sia_locations_der, expected_locations);
let expected_subordinate_skis = child_entries
.iter()
.map(|child| hex::decode(&child.child_ski).expect("decode child ski"))
.collect::<Vec<_>>();
assert_eq!(projection.subordinate_skis, expected_subordinate_skis);
}
#[test]
fn build_vcir_related_artifacts_classifies_snapshot_files_and_audit_statuses() {
let manifest_bytes = std::fs::read(
@ -4243,6 +4578,7 @@ authorityKeyIdentifier = keyid:always
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
// For this fixture-driven smoke, we provide the correct issuer CA certificate (the CA for
@ -4404,6 +4740,7 @@ authorityKeyIdentifier = keyid:always
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let first = runner.run_publication_point(&handle).expect("first run ok");
@ -4515,6 +4852,7 @@ authorityKeyIdentifier = keyid:always
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let first = runner.run_publication_point(&handle).expect("first run ok");
@ -4629,6 +4967,7 @@ authorityKeyIdentifier = keyid:always
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let first = runner.run_publication_point(&handle).expect("first run ok");
@ -4715,6 +5054,7 @@ authorityKeyIdentifier = keyid:always
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let first = ok_runner
.run_publication_point(&handle)
@ -4744,6 +5084,7 @@ authorityKeyIdentifier = keyid:always
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let second = bad_runner
.run_publication_point(&handle)
@ -5274,6 +5615,10 @@ authorityKeyIdentifier = keyid:always
projection.discovered_children[0].handle.manifest_rsync_uri,
"rsync://example.test/repo/child/child.mft"
);
assert_eq!(
projection.ccr_manifest_projection.as_ref(),
Some(&vcir.ccr_manifest_projection)
);
assert!(
projection.snapshot.is_some(),
"expected reconstructed snapshot"
@ -5325,6 +5670,7 @@ authorityKeyIdentifier = keyid:always
projection.source,
PublicationPointSource::FailedFetchNoCache
);
assert!(projection.ccr_manifest_projection.is_none());
assert!(projection.objects.vrps.is_empty());
assert!(projection.objects.aspas.is_empty());
assert!(projection.discovered_children.is_empty());
@ -5364,6 +5710,7 @@ authorityKeyIdentifier = keyid:always
PublicationPointSource::FailedFetchNoCache
);
assert!(projection.vcir.is_none());
assert!(projection.ccr_manifest_projection.is_none());
assert!(projection.snapshot.is_none());
assert!(projection.objects.audit.is_empty());
assert!(projection.discovered_children.is_empty());
@ -5414,6 +5761,7 @@ authorityKeyIdentifier = keyid:always
PublicationPointSource::FailedFetchNoCache
);
assert!(projection.vcir.is_some());
assert!(projection.ccr_manifest_projection.is_none());
assert!(projection.snapshot.is_none());
assert!(projection.discovered_children.is_empty());
assert!(projection.warnings.iter().any(|warning| {
@ -5423,6 +5771,144 @@ authorityKeyIdentifier = keyid:always
}));
}
#[test]
fn project_current_instance_vcir_rejects_mismatched_ccr_projection_uri() {
let now = time::OffsetDateTime::now_utc();
let child_cert_hash = sha256_hex(b"child-cert");
let mut vcir = sample_vcir_for_projection(now, &child_cert_hash);
vcir.ccr_manifest_projection.manifest_rsync_uri =
"rsync://example.test/repo/issuer/other.mft".to_string();
let store_dir = tempfile::tempdir().expect("store dir");
let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
store.put_vcir(&vcir).expect("put vcir");
let ca = CaInstanceHandle {
depth: 0,
tal_id: "test-tal".to_string(),
parent_manifest_rsync_uri: None,
ca_certificate_der: Vec::new(),
ca_certificate_rsync_uri: None,
effective_ip_resources: None,
effective_as_resources: None,
rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(),
manifest_rsync_uri: vcir.manifest_rsync_uri.clone(),
publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(),
rrdp_notification_uri: None,
};
let err = project_current_instance_vcir_on_failed_fetch(
&store,
&ca,
&ManifestFreshError::RepoSyncFailed {
detail: "synthetic".to_string(),
},
now,
)
.unwrap_err();
assert!(
err.contains("vcir CCR manifest projection URI mismatch"),
"{err}"
);
}
#[test]
fn fresh_and_reuse_paths_produce_equivalent_ccr_manifest_projection() {
let (pack, issuer_ca_der, validation_time) =
cernet_publication_point_snapshot_for_vcir_tests();
let ca = CaInstanceHandle {
depth: 0,
tal_id: "test-tal".to_string(),
parent_manifest_rsync_uri: None,
ca_certificate_der: issuer_ca_der,
ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()),
effective_ip_resources: None,
effective_as_resources: None,
rsync_base_uri: pack.publication_point_rsync_uri.clone(),
manifest_rsync_uri: pack.manifest_rsync_uri.clone(),
publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(),
rrdp_notification_uri: None,
};
let child_discovery =
discover_children_from_fresh_snapshot_with_audit(&ca, &pack, validation_time, None)
.expect("discover children");
let fresh_vcir = build_vcir_from_fresh_result(
&ca,
&pack,
&empty_objects_output(),
&[],
&child_discovery.audits,
&child_discovery.children,
validation_time,
)
.expect("build fresh vcir");
let reuse_projection = reuse_ccr_manifest_projection_from_vcir(&ca, &fresh_vcir)
.expect("reuse projection from vcir");
assert_eq!(fresh_vcir.ccr_manifest_projection, reuse_projection);
}
#[test]
fn append_ccr_manifest_projection_from_reuse_requires_projection_for_current_instance() {
let store_dir = tempfile::tempdir().expect("store dir");
let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
let policy = Policy::default();
let runner = sample_runner_with_ccr_accumulator(&store, &policy);
let err = runner
.append_ccr_manifest_projection_from_reuse(&VcirReuseProjection {
source: PublicationPointSource::VcirCurrentInstance,
vcir: None,
ccr_manifest_projection: None,
snapshot: None,
objects: empty_objects_output(),
child_audits: Vec::new(),
discovered_children: Vec::new(),
warnings: Vec::new(),
})
.unwrap_err();
assert!(err.contains("missing CCR manifest projection"), "{err}");
assert_eq!(
runner
.ccr_accumulator_snapshot()
.expect("ccr accumulator snapshot")
.manifest_count(),
0
);
}
#[test]
fn append_ccr_manifest_projection_from_reuse_skips_failed_fetch_no_cache() {
let store_dir = tempfile::tempdir().expect("store dir");
let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
let policy = Policy::default();
let runner = sample_runner_with_ccr_accumulator(&store, &policy);
runner
.append_ccr_manifest_projection_from_reuse(&VcirReuseProjection {
source: PublicationPointSource::FailedFetchNoCache,
vcir: None,
ccr_manifest_projection: None,
snapshot: None,
objects: empty_objects_output(),
child_audits: Vec::new(),
discovered_children: Vec::new(),
warnings: Vec::new(),
})
.expect("failed-fetch no-cache should not append");
assert_eq!(
runner
.ccr_accumulator_snapshot()
.expect("ccr accumulator snapshot")
.manifest_count(),
0
);
}
#[test]
fn parse_snapshot_time_value_reports_invalid_timestamp() {
let err = parse_snapshot_time_value(&PackTime {
@ -5919,6 +6405,7 @@ authorityKeyIdentifier = keyid:always
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let first = runner_rrdp
.run_publication_point(&handle)
@ -5951,6 +6438,7 @@ authorityKeyIdentifier = keyid:always
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let third = runner_rsync
.run_publication_point(&handle)

1
tests/test_apnic_stats_live_stage2.rs
View File

@ -184,6 +184,7 @@ fn apnic_tree_full_stats_serial() {
repo_sync_runtime: None,
parallel_phase2_config: None,
parallel_roa_worker_pool: None,
ccr_accumulator: None,
};
let stats = RefCell::new(LiveStats::default());

33
tests/test_ccr_m7.rs
View File

@ -1,7 +1,5 @@
use rpki::ccr::{
CcrContentInfo, CcrDigestAlgorithm, ManifestInstance, ManifestState, RoaPayloadSet,
RoaPayloadState, RouterKey, RouterKeySet, RouterKeyState, TrustAnchorState, compute_state_hash,
decode_content_info, dump_content_info_json_value,
compute_state_hash, decode_content_info, dump_content_info_json_value,
encode::{
encode_aspa_payload_state_payload_der, encode_content_info,
encode_manifest_state_payload_der, encode_roa_payload_state_payload_der,
@ -10,12 +8,14 @@ use rpki::ccr::{
verify::{
verify_against_report_json_path, verify_against_vcir_store, verify_content_info_bytes,
},
CcrContentInfo, CcrDigestAlgorithm, ManifestInstance, ManifestState, RoaPayloadSet,
RoaPayloadState, RouterKey, RouterKeySet, RouterKeyState, TrustAnchorState,
};
use rpki::data_model::common::BigUnsigned;
use rpki::storage::{
PackTime, RocksStore, ValidatedCaInstanceResult, ValidatedManifestMeta, VcirArtifactKind,
VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, VcirChildEntry,
VcirInstanceGate, VcirRelatedArtifact, VcirSummary,
VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, VcirCcrManifestProjection,
VcirChildEntry, VcirInstanceGate, VcirRelatedArtifact, VcirSummary,
};
fn sample_time() -> time::OffsetDateTime {
@ -108,6 +108,25 @@ fn sample_ccr() -> Vec<u8> {
encode_content_info(&CcrContentInfo::new(ccr)).expect("encode ccr")
}
fn sample_ccr_manifest_projection(
manifest_rsync_uri: &str,
manifest_sha256: Vec<u8>,
) -> VcirCcrManifestProjection {
VcirCcrManifestProjection {
manifest_rsync_uri: manifest_rsync_uri.to_string(),
manifest_sha256,
manifest_size: 2048,
manifest_ee_aki: vec![0x55; 20],
manifest_number_be: vec![1],
manifest_this_update: PackTime::from_utc_offset_datetime(sample_time()),
manifest_sia_locations_der: vec![vec![
0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05,
b'r', b's', b'y', b'n', b'c',
]],
subordinate_skis: vec![vec![0x33; 20]],
}
}
#[test]
fn verify_content_info_bytes_succeeds_for_valid_ccr() {
let der = sample_ccr();
@ -195,6 +214,10 @@ fn verify_against_vcir_store_matches_manifest_hashes() {
validated_manifest_this_update: PackTime::from_utc_offset_datetime(sample_time()),
validated_manifest_next_update: PackTime::from_utc_offset_datetime(sample_time()),
},
ccr_manifest_projection: sample_ccr_manifest_projection(
"rsync://example.test/current.mft",
vec![0x10; 32],
),
instance_gate: VcirInstanceGate {
manifest_next_update: PackTime::from_utc_offset_datetime(sample_time()),
current_crl_next_update: PackTime::from_utc_offset_datetime(sample_time()),

20
tests/test_manifest_processor_m4.rs
View File

@ -7,10 +7,10 @@ use rpki::policy::{CaFailedFetchPolicy, Policy};
use rpki::storage::{
PackTime, RawByHashEntry, RepositoryViewEntry, RepositoryViewState, RocksStore,
ValidatedCaInstanceResult, ValidatedManifestMeta, VcirArtifactKind, VcirArtifactRole,
VcirArtifactValidationStatus, VcirAuditSummary, VcirInstanceGate, VcirRelatedArtifact,
VcirSummary,
VcirArtifactValidationStatus, VcirAuditSummary, VcirCcrManifestProjection, VcirInstanceGate,
VcirRelatedArtifact, VcirSummary,
};
use rpki::validation::manifest::{PublicationPointSource, process_manifest_publication_point};
use rpki::validation::manifest::{process_manifest_publication_point, PublicationPointSource};
fn issuer_ca_fixture() -> Vec<u8> {
std::fs::read(
@ -64,6 +64,19 @@ fn store_validated_manifest_baseline(
store
.put_raw_by_hash_entry(&manifest_raw)
.expect("store VCIR manifest raw_by_hash");
let ccr_manifest_projection = VcirCcrManifestProjection {
manifest_rsync_uri: manifest_rsync_uri.to_string(),
manifest_sha256: hex::decode(&manifest_sha256).expect("decode manifest hash"),
manifest_size: manifest_bytes.len() as u64,
manifest_ee_aki: vec![0x11; 20],
manifest_number_be: manifest_number_be.clone(),
manifest_this_update: PackTime::from_utc_offset_datetime(this_update),
manifest_sia_locations_der: vec![vec![
0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05,
b'r', b's', b'y', b'n', b'c',
]],
subordinate_skis: Vec::new(),
};
let vcir = ValidatedCaInstanceResult {
manifest_rsync_uri: manifest_rsync_uri.to_string(),
@ -80,6 +93,7 @@ fn store_validated_manifest_baseline(
validated_manifest_this_update: PackTime::from_utc_offset_datetime(this_update),
validated_manifest_next_update: PackTime::from_utc_offset_datetime(next_update),
},
ccr_manifest_projection,
instance_gate: VcirInstanceGate {
manifest_next_update: PackTime::from_utc_offset_datetime(next_update),
current_crl_next_update: PackTime::from_utc_offset_datetime(next_update),

22
tests/test_manifest_processor_repo_sync_and_cached_snapshot_cov.rs
View File

@ -7,12 +7,12 @@ use rpki::policy::{CaFailedFetchPolicy, Policy};
use rpki::storage::{
PackTime, RawByHashEntry, RepositoryViewEntry, RepositoryViewState, RocksStore,
ValidatedCaInstanceResult, ValidatedManifestMeta, VcirArtifactKind, VcirArtifactRole,
VcirArtifactValidationStatus, VcirAuditSummary, VcirInstanceGate, VcirRelatedArtifact,
VcirSummary,
VcirArtifactValidationStatus, VcirAuditSummary, VcirCcrManifestProjection, VcirInstanceGate,
VcirRelatedArtifact, VcirSummary,
};
use rpki::validation::manifest::{
ManifestProcessError, PublicationPointSource, process_manifest_publication_point,
process_manifest_publication_point_after_repo_sync,
process_manifest_publication_point, process_manifest_publication_point_after_repo_sync,
ManifestProcessError, PublicationPointSource,
};
fn issuer_ca_fixture_der() -> Vec<u8> {
@ -67,6 +67,19 @@ fn store_validated_manifest_baseline(
store
.put_raw_by_hash_entry(&manifest_raw)
.expect("store VCIR manifest raw_by_hash");
let ccr_manifest_projection = VcirCcrManifestProjection {
manifest_rsync_uri: manifest_rsync_uri.to_string(),
manifest_sha256: hex::decode(&manifest_sha256).expect("decode manifest hash"),
manifest_size: manifest_bytes.len() as u64,
manifest_ee_aki: vec![0x11; 20],
manifest_number_be: manifest_number_be.clone(),
manifest_this_update: PackTime::from_utc_offset_datetime(this_update),
manifest_sia_locations_der: vec![vec![
0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05,
b'r', b's', b'y', b'n', b'c',
]],
subordinate_skis: Vec::new(),
};
let vcir = ValidatedCaInstanceResult {
manifest_rsync_uri: manifest_rsync_uri.to_string(),
@ -83,6 +96,7 @@ fn store_validated_manifest_baseline(
validated_manifest_this_update: PackTime::from_utc_offset_datetime(this_update),
validated_manifest_next_update: PackTime::from_utc_offset_datetime(next_update),
},
ccr_manifest_projection,
instance_gate: VcirInstanceGate {
manifest_next_update: PackTime::from_utc_offset_datetime(next_update),
current_crl_next_update: PackTime::from_utc_offset_datetime(next_update),