20260410 完成五个rir 基于cir的三方回放,raw by hash 独立db,发现内存占用大,连续大rir 录制发生oom
This commit is contained in:
17
src/audit.rs
17
src/audit.rs
@@ -75,9 +75,12 @@ pub struct PublicationPointAudit {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub repo_sync_source: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub repo_sync_phase: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub repo_sync_duration_ms: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub repo_sync_error: Option<String>,
|
||||
pub repo_terminal_state: String,
|
||||
pub this_update_rfc3339_utc: String,
|
||||
pub next_update_rfc3339_utc: String,
|
||||
pub verified_at_rfc3339_utc: String,
|
||||
@@ -156,6 +159,19 @@ pub struct AuditDownloadStats {
|
||||
pub by_kind: std::collections::BTreeMap<String, AuditDownloadKindStats>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
|
||||
pub struct AuditRepoSyncStateStat {
|
||||
pub count: u64,
|
||||
pub duration_ms_total: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
|
||||
pub struct AuditRepoSyncStats {
|
||||
pub publication_points_total: u64,
|
||||
pub by_phase: std::collections::BTreeMap<String, AuditRepoSyncStateStat>,
|
||||
pub by_terminal_state: std::collections::BTreeMap<String, AuditRepoSyncStateStat>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub struct AuditReportV1 {
|
||||
pub format_version: u32,
|
||||
@@ -181,6 +197,7 @@ pub struct AuditReportV2 {
|
||||
|
||||
pub downloads: Vec<AuditDownloadEvent>,
|
||||
pub download_stats: AuditDownloadStats,
|
||||
pub repo_sync_stats: AuditRepoSyncStats,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
|
||||
@@ -249,12 +249,12 @@ fn raw_ref_from_entry(sha256_hex: &str, entry: Option<&RawByHashEntry>) -> Audit
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use base64::Engine as _;
|
||||
use crate::audit::sha256_hex;
|
||||
use crate::storage::{
|
||||
PackTime, ValidatedManifestMeta, VcirAuditSummary, VcirChildEntry, VcirInstanceGate,
|
||||
VcirRelatedArtifact, VcirSummary,
|
||||
};
|
||||
use base64::Engine as _;
|
||||
|
||||
fn sample_vcir(
|
||||
manifest_rsync_uri: &str,
|
||||
@@ -495,7 +495,8 @@ mod tests {
|
||||
"as_id": 64496,
|
||||
"ski_hex": "11".repeat(20),
|
||||
"spki_der_base64": base64::engine::general_purpose::STANDARD.encode([0x30u8, 0x00]),
|
||||
}).to_string();
|
||||
})
|
||||
.to_string();
|
||||
let mut vcir = sample_vcir(
|
||||
manifest,
|
||||
None,
|
||||
@@ -504,13 +505,15 @@ mod tests {
|
||||
sample_artifacts(manifest, &sha256_hex(b"router-object")),
|
||||
);
|
||||
vcir.local_outputs[0].output_type = VcirOutputType::RouterKey;
|
||||
vcir.local_outputs[0].source_object_uri = "rsync://example.test/router/router.cer".to_string();
|
||||
vcir.local_outputs[0].source_object_uri =
|
||||
"rsync://example.test/router/router.cer".to_string();
|
||||
vcir.local_outputs[0].source_object_type = "router_key".to_string();
|
||||
vcir.local_outputs[0].payload_json = serde_json::json!({
|
||||
"as_id": 64496,
|
||||
"ski_hex": "11".repeat(20),
|
||||
"spki_der_base64": base64::engine::general_purpose::STANDARD.encode([0x30u8, 0x00]),
|
||||
}).to_string();
|
||||
})
|
||||
.to_string();
|
||||
vcir.summary.local_vrp_count = 0;
|
||||
vcir.summary.local_router_key_count = 1;
|
||||
store.put_vcir(&vcir).expect("put vcir");
|
||||
@@ -523,7 +526,9 @@ mod tests {
|
||||
output_id: vcir.local_outputs[0].output_id.clone(),
|
||||
item_effective_until: vcir.local_outputs[0].item_effective_until.clone(),
|
||||
};
|
||||
store.put_audit_rule_index_entry(&rule_entry).expect("put rule");
|
||||
store
|
||||
.put_audit_rule_index_entry(&rule_entry)
|
||||
.expect("put rule");
|
||||
let trace = trace_rule_to_root(&store, AuditRuleKind::RouterKey, &rule_entry.rule_hash)
|
||||
.expect("trace rule")
|
||||
.expect("trace exists");
|
||||
|
||||
@@ -33,9 +33,13 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
||||
fn main() -> Result<(), String> {
|
||||
let args = parse_args(&std::env::args().collect::<Vec<_>>())?;
|
||||
let ccr_path = args.ccr_path.as_ref().unwrap();
|
||||
let bytes = std::fs::read(ccr_path).map_err(|e| format!("read ccr failed: {}: {e}", ccr_path.display()))?;
|
||||
let bytes = std::fs::read(ccr_path)
|
||||
.map_err(|e| format!("read ccr failed: {}: {e}", ccr_path.display()))?;
|
||||
let json = dump_content_info_json_value(&bytes).map_err(|e| e.to_string())?;
|
||||
println!("{}", serde_json::to_string_pretty(&json).map_err(|e| e.to_string())?);
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&json).map_err(|e| e.to_string())?
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -45,9 +49,16 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn parse_args_accepts_ccr_path() {
|
||||
let argv = vec!["ccr_dump".to_string(), "--ccr".to_string(), "a.ccr".to_string()];
|
||||
let argv = vec![
|
||||
"ccr_dump".to_string(),
|
||||
"--ccr".to_string(),
|
||||
"a.ccr".to_string(),
|
||||
];
|
||||
let args = parse_args(&argv).expect("parse");
|
||||
assert_eq!(args.ccr_path.as_deref(), Some(std::path::Path::new("a.ccr")));
|
||||
assert_eq!(
|
||||
args.ccr_path.as_deref(),
|
||||
Some(std::path::Path::new("a.ccr"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -94,7 +94,10 @@ mod tests {
|
||||
"apnic".to_string(),
|
||||
];
|
||||
let args = parse_args(&argv).expect("parse args");
|
||||
assert_eq!(args.ccr_path.as_deref(), Some(std::path::Path::new("a.ccr")));
|
||||
assert_eq!(
|
||||
args.ccr_path.as_deref(),
|
||||
Some(std::path::Path::new("a.ccr"))
|
||||
);
|
||||
assert_eq!(
|
||||
args.vrps_out_path.as_deref(),
|
||||
Some(std::path::Path::new("vrps.csv"))
|
||||
|
||||
@@ -49,7 +49,9 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
||||
Ok(args)
|
||||
}
|
||||
|
||||
fn collect_vrp_rows(bytes: &[u8]) -> Result<std::collections::BTreeSet<(u32, String, u16)>, String> {
|
||||
fn collect_vrp_rows(
|
||||
bytes: &[u8],
|
||||
) -> Result<std::collections::BTreeSet<(u32, String, u16)>, String> {
|
||||
let content_info = decode_content_info(bytes).map_err(|e| e.to_string())?;
|
||||
extract_vrp_rows(&content_info).map_err(|e| e.to_string())
|
||||
}
|
||||
@@ -95,14 +97,24 @@ mod tests {
|
||||
"apnic".to_string(),
|
||||
];
|
||||
let args = parse_args(&argv).expect("parse args");
|
||||
assert_eq!(args.ccr_path.as_deref(), Some(std::path::Path::new("a.ccr")));
|
||||
assert_eq!(args.out_path.as_deref(), Some(std::path::Path::new("out.csv")));
|
||||
assert_eq!(
|
||||
args.ccr_path.as_deref(),
|
||||
Some(std::path::Path::new("a.ccr"))
|
||||
);
|
||||
assert_eq!(
|
||||
args.out_path.as_deref(),
|
||||
Some(std::path::Path::new("out.csv"))
|
||||
);
|
||||
assert_eq!(args.trust_anchor, "apnic");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_args_rejects_missing_required_flags() {
|
||||
let argv = vec!["ccr_to_routinator_csv".to_string(), "--ccr".to_string(), "a.ccr".to_string()];
|
||||
let argv = vec![
|
||||
"ccr_to_routinator_csv".to_string(),
|
||||
"--ccr".to_string(),
|
||||
"a.ccr".to_string(),
|
||||
];
|
||||
let err = parse_args(&argv).unwrap_err();
|
||||
assert!(err.contains("--out is required"), "{err}");
|
||||
}
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
use rpki::ccr::{decode_content_info, verify::verify_content_info, verify_against_report_json_path, verify_against_vcir_store_path};
|
||||
use rpki::ccr::{
|
||||
decode_content_info, verify::verify_content_info, verify_against_report_json_path,
|
||||
verify_against_vcir_store_path,
|
||||
};
|
||||
|
||||
#[derive(Debug, Default, PartialEq, Eq)]
|
||||
struct Args {
|
||||
@@ -45,7 +48,8 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
||||
fn main() -> Result<(), String> {
|
||||
let args = parse_args(&std::env::args().collect::<Vec<_>>())?;
|
||||
let ccr_path = args.ccr_path.as_ref().unwrap();
|
||||
let bytes = std::fs::read(ccr_path).map_err(|e| format!("read ccr failed: {}: {e}", ccr_path.display()))?;
|
||||
let bytes = std::fs::read(ccr_path)
|
||||
.map_err(|e| format!("read ccr failed: {}: {e}", ccr_path.display()))?;
|
||||
let ci = decode_content_info(&bytes).map_err(|e| e.to_string())?;
|
||||
let summary = verify_content_info(&ci).map_err(|e| e.to_string())?;
|
||||
if let Some(report_json) = args.report_json.as_ref() {
|
||||
@@ -54,7 +58,10 @@ fn main() -> Result<(), String> {
|
||||
if let Some(db_path) = args.db_path.as_ref() {
|
||||
verify_against_vcir_store_path(&ci, db_path).map_err(|e| e.to_string())?;
|
||||
}
|
||||
println!("{}", serde_json::to_string_pretty(&summary).map_err(|e| e.to_string())?);
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&summary).map_err(|e| e.to_string())?
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -74,8 +81,14 @@ mod tests {
|
||||
"db".to_string(),
|
||||
];
|
||||
let args = parse_args(&argv).expect("parse");
|
||||
assert_eq!(args.ccr_path.as_deref(), Some(std::path::Path::new("a.ccr")));
|
||||
assert_eq!(args.report_json.as_deref(), Some(std::path::Path::new("report.json")));
|
||||
assert_eq!(
|
||||
args.ccr_path.as_deref(),
|
||||
Some(std::path::Path::new("a.ccr"))
|
||||
);
|
||||
assert_eq!(
|
||||
args.report_json.as_deref(),
|
||||
Some(std::path::Path::new("report.json"))
|
||||
);
|
||||
assert_eq!(args.db_path.as_deref(), Some(std::path::Path::new("db")));
|
||||
}
|
||||
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use rpki::blob_store::{ExternalRawStoreDb, RawObjectStore};
|
||||
use rpki::bundle::decode_ccr_compare_views;
|
||||
use rpki::ccr::decode_content_info;
|
||||
use rpki::cir::{decode_cir, resolve_static_pool_file};
|
||||
use rpki::data_model::roa::RoaObject;
|
||||
|
||||
const USAGE: &str = "Usage: cir_drop_report --cir <path> --ccr <path> --report-json <path> --static-root <path> --json-out <path> --md-out <path>";
|
||||
const USAGE: &str = "Usage: cir_drop_report --cir <path> --ccr <path> --report-json <path> (--static-root <path> | --raw-store-db <path>) --json-out <path> --md-out <path>";
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct DroppedObjectRecord {
|
||||
@@ -47,11 +48,25 @@ fn classify_reason(detail: Option<&str>, result: &str) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_args(argv: &[String]) -> Result<(PathBuf, PathBuf, PathBuf, PathBuf, PathBuf, PathBuf), String> {
|
||||
fn parse_args(
|
||||
argv: &[String],
|
||||
) -> Result<
|
||||
(
|
||||
PathBuf,
|
||||
PathBuf,
|
||||
PathBuf,
|
||||
Option<PathBuf>,
|
||||
Option<PathBuf>,
|
||||
PathBuf,
|
||||
PathBuf,
|
||||
),
|
||||
String,
|
||||
> {
|
||||
let mut cir = None;
|
||||
let mut ccr = None;
|
||||
let mut report = None;
|
||||
let mut static_root = None;
|
||||
let mut raw_store_db = None;
|
||||
let mut json_out = None;
|
||||
let mut md_out = None;
|
||||
let mut i = 1usize;
|
||||
@@ -77,6 +92,12 @@ fn parse_args(argv: &[String]) -> Result<(PathBuf, PathBuf, PathBuf, PathBuf, Pa
|
||||
argv.get(i).ok_or("--static-root requires a value")?,
|
||||
));
|
||||
}
|
||||
"--raw-store-db" => {
|
||||
i += 1;
|
||||
raw_store_db = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--raw-store-db requires a value")?,
|
||||
));
|
||||
}
|
||||
"--json-out" => {
|
||||
i += 1;
|
||||
json_out = Some(PathBuf::from(
|
||||
@@ -85,7 +106,9 @@ fn parse_args(argv: &[String]) -> Result<(PathBuf, PathBuf, PathBuf, PathBuf, Pa
|
||||
}
|
||||
"--md-out" => {
|
||||
i += 1;
|
||||
md_out = Some(PathBuf::from(argv.get(i).ok_or("--md-out requires a value")?));
|
||||
md_out = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--md-out requires a value")?,
|
||||
));
|
||||
}
|
||||
"-h" | "--help" => return Err(USAGE.to_string()),
|
||||
other => return Err(format!("unknown argument: {other}\n\n{USAGE}")),
|
||||
@@ -96,7 +119,8 @@ fn parse_args(argv: &[String]) -> Result<(PathBuf, PathBuf, PathBuf, PathBuf, Pa
|
||||
cir.ok_or_else(|| format!("--cir is required\n\n{USAGE}"))?,
|
||||
ccr.ok_or_else(|| format!("--ccr is required\n\n{USAGE}"))?,
|
||||
report.ok_or_else(|| format!("--report-json is required\n\n{USAGE}"))?,
|
||||
static_root.ok_or_else(|| format!("--static-root is required\n\n{USAGE}"))?,
|
||||
static_root,
|
||||
raw_store_db,
|
||||
json_out.ok_or_else(|| format!("--json-out is required\n\n{USAGE}"))?,
|
||||
md_out.ok_or_else(|| format!("--md-out is required\n\n{USAGE}"))?,
|
||||
))
|
||||
@@ -104,7 +128,14 @@ fn parse_args(argv: &[String]) -> Result<(PathBuf, PathBuf, PathBuf, PathBuf, Pa
|
||||
|
||||
fn main() -> Result<(), String> {
|
||||
let argv: Vec<String> = std::env::args().collect();
|
||||
let (cir_path, ccr_path, report_path, static_root, json_out, md_out) = parse_args(&argv)?;
|
||||
let (cir_path, ccr_path, report_path, static_root, raw_store_db, json_out, md_out) =
|
||||
parse_args(&argv)?;
|
||||
let backend_count = static_root.is_some() as u8 + raw_store_db.is_some() as u8;
|
||||
if backend_count != 1 {
|
||||
return Err(format!(
|
||||
"must specify exactly one of --static-root or --raw-store-db\n\n{USAGE}"
|
||||
));
|
||||
}
|
||||
|
||||
let cir = decode_cir(&std::fs::read(&cir_path).map_err(|e| format!("read cir failed: {e}"))?)
|
||||
.map_err(|e| format!("decode cir failed: {e}"))?;
|
||||
@@ -112,11 +143,12 @@ fn main() -> Result<(), String> {
|
||||
&std::fs::read(&ccr_path).map_err(|e| format!("read ccr failed: {e}"))?,
|
||||
)
|
||||
.map_err(|e| format!("decode ccr failed: {e}"))?;
|
||||
let (vrps, vaps) =
|
||||
decode_ccr_compare_views(&ccr, "unknown").map_err(|e| format!("decode compare views failed: {e}"))?;
|
||||
let report: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(&report_path).map_err(|e| format!("read report failed: {e}"))?)
|
||||
.map_err(|e| format!("parse report failed: {e}"))?;
|
||||
let (vrps, vaps) = decode_ccr_compare_views(&ccr, "unknown")
|
||||
.map_err(|e| format!("decode compare views failed: {e}"))?;
|
||||
let report: serde_json::Value = serde_json::from_slice(
|
||||
&std::fs::read(&report_path).map_err(|e| format!("read report failed: {e}"))?,
|
||||
)
|
||||
.map_err(|e| format!("parse report failed: {e}"))?;
|
||||
|
||||
let mut object_hash_by_uri = BTreeMap::new();
|
||||
for object in &cir.objects {
|
||||
@@ -134,7 +166,9 @@ fn main() -> Result<(), String> {
|
||||
let mut unknown_roa_objects = 0usize;
|
||||
|
||||
for pp in publication_points {
|
||||
let publication_point = pp["publication_point_rsync_uri"].as_str().map(str::to_string);
|
||||
let publication_point = pp["publication_point_rsync_uri"]
|
||||
.as_str()
|
||||
.map(str::to_string);
|
||||
let manifest_uri = pp["manifest_rsync_uri"].as_str().map(str::to_string);
|
||||
for obj in pp["objects"].as_array().into_iter().flatten() {
|
||||
let result = obj["result"].as_str().unwrap_or("unknown");
|
||||
@@ -155,39 +189,52 @@ fn main() -> Result<(), String> {
|
||||
|
||||
let mut derived_vrp_count = 0usize;
|
||||
if kind == "roa" && !hash.is_empty() {
|
||||
match resolve_static_pool_file(&static_root, &hash) {
|
||||
Ok(path) => {
|
||||
if let Ok(bytes) = std::fs::read(&path) {
|
||||
if let Ok(roa) = RoaObject::decode_der(&bytes) {
|
||||
for family in roa.roa.ip_addr_blocks {
|
||||
for addr in family.addresses {
|
||||
let prefix = match addr.prefix.afi {
|
||||
rpki::data_model::roa::RoaAfi::Ipv4 => format!(
|
||||
"{}.{}.{}.{}/{}",
|
||||
addr.prefix.addr[0],
|
||||
addr.prefix.addr[1],
|
||||
addr.prefix.addr[2],
|
||||
addr.prefix.addr[3],
|
||||
let bytes_opt = if let Some(static_root) = static_root.as_ref() {
|
||||
match resolve_static_pool_file(static_root, &hash) {
|
||||
Ok(path) => std::fs::read(&path).ok(),
|
||||
Err(_) => None,
|
||||
}
|
||||
} else if let Some(raw_store_db) = raw_store_db.as_ref() {
|
||||
ExternalRawStoreDb::open(raw_store_db)
|
||||
.ok()
|
||||
.and_then(|store| store.get_raw_entry(&hash).ok().flatten())
|
||||
.map(|entry| entry.bytes)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
match bytes_opt {
|
||||
Some(bytes) => {
|
||||
if let Ok(roa) = RoaObject::decode_der(&bytes) {
|
||||
for family in roa.roa.ip_addr_blocks {
|
||||
for addr in family.addresses {
|
||||
let prefix = match addr.prefix.afi {
|
||||
rpki::data_model::roa::RoaAfi::Ipv4 => format!(
|
||||
"{}.{}.{}.{}/{}",
|
||||
addr.prefix.addr[0],
|
||||
addr.prefix.addr[1],
|
||||
addr.prefix.addr[2],
|
||||
addr.prefix.addr[3],
|
||||
addr.prefix.prefix_len
|
||||
),
|
||||
rpki::data_model::roa::RoaAfi::Ipv6 => {
|
||||
let bytes: [u8; 16] = addr.prefix.addr;
|
||||
format!(
|
||||
"{}/{}",
|
||||
std::net::Ipv6Addr::from(bytes),
|
||||
addr.prefix.prefix_len
|
||||
),
|
||||
rpki::data_model::roa::RoaAfi::Ipv6 => {
|
||||
let bytes: [u8; 16] = addr.prefix.addr;
|
||||
format!("{}/{}", std::net::Ipv6Addr::from(bytes), addr.prefix.prefix_len)
|
||||
}
|
||||
};
|
||||
let max_len = addr.max_length.unwrap_or(addr.prefix.prefix_len);
|
||||
dropped_vrp_rows.insert((roa.roa.as_id, prefix, max_len));
|
||||
derived_vrp_count += 1;
|
||||
}
|
||||
)
|
||||
}
|
||||
};
|
||||
let max_len = addr.max_length.unwrap_or(addr.prefix.prefix_len);
|
||||
dropped_vrp_rows.insert((roa.roa.as_id, prefix, max_len));
|
||||
derived_vrp_count += 1;
|
||||
}
|
||||
} else {
|
||||
unknown_roa_objects += 1;
|
||||
}
|
||||
} else {
|
||||
unknown_roa_objects += 1;
|
||||
}
|
||||
}
|
||||
Err(_) => unknown_roa_objects += 1,
|
||||
None => unknown_roa_objects += 1,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -227,22 +274,40 @@ fn main() -> Result<(), String> {
|
||||
md.push_str("# CIR Drop Report\n\n");
|
||||
md.push_str(&format!("- `final_vrp_count`: `{}`\n", vrps.len()));
|
||||
md.push_str(&format!("- `final_vap_count`: `{}`\n", vaps.len()));
|
||||
md.push_str(&format!("- `dropped_vrp_count`: `{}`\n", output["summary"]["droppedVrpCount"]));
|
||||
md.push_str(&format!("- `dropped_object_count`: `{}`\n", output["summary"]["droppedObjectCount"]));
|
||||
md.push_str(&format!(
|
||||
"- `dropped_vrp_count`: `{}`\n",
|
||||
output["summary"]["droppedVrpCount"]
|
||||
));
|
||||
md.push_str(&format!(
|
||||
"- `dropped_object_count`: `{}`\n",
|
||||
output["summary"]["droppedObjectCount"]
|
||||
));
|
||||
md.push_str(&format!(
|
||||
"- `unknown_dropped_roa_objects`: `{}`\n\n",
|
||||
output["summary"]["unknownDroppedRoaObjects"]
|
||||
));
|
||||
md.push_str("## Dropped By Kind\n\n");
|
||||
for (kind, count) in output["summary"]["droppedByKind"].as_object().into_iter().flatten() {
|
||||
for (kind, count) in output["summary"]["droppedByKind"]
|
||||
.as_object()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
md.push_str(&format!("- `{kind}`: `{}`\n", count.as_u64().unwrap_or(0)));
|
||||
}
|
||||
md.push_str("\n## Dropped By Reason\n\n");
|
||||
for (reason, count) in output["summary"]["droppedByReason"].as_object().into_iter().flatten() {
|
||||
md.push_str(&format!("- `{reason}`: `{}`\n", count.as_u64().unwrap_or(0)));
|
||||
for (reason, count) in output["summary"]["droppedByReason"]
|
||||
.as_object()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
md.push_str(&format!(
|
||||
"- `{reason}`: `{}`\n",
|
||||
count.as_u64().unwrap_or(0)
|
||||
));
|
||||
}
|
||||
if let Some(parent) = md_out.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| format!("create markdown parent failed: {e}"))?;
|
||||
std::fs::create_dir_all(parent)
|
||||
.map_err(|e| format!("create markdown parent failed: {e}"))?;
|
||||
}
|
||||
std::fs::write(&md_out, md).map_err(|e| format!("write markdown failed: {e}"))?;
|
||||
|
||||
|
||||
@@ -26,11 +26,15 @@ fn run(argv: Vec<String>) -> Result<(), String> {
|
||||
}
|
||||
"--tals-dir" => {
|
||||
i += 1;
|
||||
tals_dir = Some(PathBuf::from(argv.get(i).ok_or("--tals-dir requires a value")?));
|
||||
tals_dir = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--tals-dir requires a value")?,
|
||||
));
|
||||
}
|
||||
"--meta-json" => {
|
||||
i += 1;
|
||||
meta_json = Some(PathBuf::from(argv.get(i).ok_or("--meta-json requires a value")?));
|
||||
meta_json = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--meta-json requires a value")?,
|
||||
));
|
||||
}
|
||||
other => return Err(format!("unknown argument: {other}\n\n{}", usage())),
|
||||
}
|
||||
@@ -76,4 +80,3 @@ fn run(argv: Vec<String>) -> Result<(), String> {
|
||||
.map_err(|e| format!("write meta json failed: {}: {e}", meta_json.display()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn usage() -> &'static str {
|
||||
"Usage: cir_materialize --cir <path> --static-root <path> --mirror-root <path> [--keep-db]"
|
||||
"Usage: cir_materialize --cir <path> (--static-root <path> | --raw-store-db <path>) --mirror-root <path> [--keep-db]"
|
||||
}
|
||||
|
||||
fn main() {
|
||||
@@ -14,6 +14,7 @@ fn main() {
|
||||
fn run(argv: Vec<String>) -> Result<(), String> {
|
||||
let mut cir_path: Option<PathBuf> = None;
|
||||
let mut static_root: Option<PathBuf> = None;
|
||||
let mut raw_store_db: Option<PathBuf> = None;
|
||||
let mut mirror_root: Option<PathBuf> = None;
|
||||
let mut keep_db = false;
|
||||
|
||||
@@ -31,6 +32,12 @@ fn run(argv: Vec<String>) -> Result<(), String> {
|
||||
argv.get(i).ok_or("--static-root requires a value")?,
|
||||
));
|
||||
}
|
||||
"--raw-store-db" => {
|
||||
i += 1;
|
||||
raw_store_db = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--raw-store-db requires a value")?,
|
||||
));
|
||||
}
|
||||
"--mirror-root" => {
|
||||
i += 1;
|
||||
mirror_root = Some(PathBuf::from(
|
||||
@@ -44,16 +51,29 @@ fn run(argv: Vec<String>) -> Result<(), String> {
|
||||
}
|
||||
|
||||
let cir_path = cir_path.ok_or_else(|| format!("--cir is required\n\n{}", usage()))?;
|
||||
let static_root =
|
||||
static_root.ok_or_else(|| format!("--static-root is required\n\n{}", usage()))?;
|
||||
let mirror_root =
|
||||
mirror_root.ok_or_else(|| format!("--mirror-root is required\n\n{}", usage()))?;
|
||||
let backend_count = static_root.is_some() as u8 + raw_store_db.is_some() as u8;
|
||||
if backend_count != 1 {
|
||||
return Err(format!(
|
||||
"must specify exactly one of --static-root or --raw-store-db\n\n{}",
|
||||
usage()
|
||||
));
|
||||
}
|
||||
|
||||
let bytes = std::fs::read(&cir_path)
|
||||
.map_err(|e| format!("read CIR failed: {}: {e}", cir_path.display()))?;
|
||||
let cir = rpki::cir::decode_cir(&bytes).map_err(|e| e.to_string())?;
|
||||
|
||||
let result = rpki::cir::materialize_cir(&cir, &static_root, &mirror_root, true);
|
||||
let result = if let Some(static_root) = static_root {
|
||||
rpki::cir::materialize_cir(&cir, &static_root, &mirror_root, true)
|
||||
.map_err(|e| e.to_string())
|
||||
} else if let Some(raw_store_db) = raw_store_db {
|
||||
rpki::cir::materialize_cir_from_raw_store(&cir, &raw_store_db, &mirror_root, true)
|
||||
.map_err(|e| e.to_string())
|
||||
} else {
|
||||
unreachable!("validated backend count")
|
||||
};
|
||||
match result {
|
||||
Ok(summary) => {
|
||||
eprintln!(
|
||||
@@ -74,4 +94,3 @@ fn run(argv: Vec<String>) -> Result<(), String> {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,26 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use rpki::cir::{encode_cir, write_bytes_to_static_pool, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal, CIR_VERSION_V1};
|
||||
use rpki::cir::{
|
||||
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal, encode_cir,
|
||||
write_bytes_to_static_pool,
|
||||
};
|
||||
use sha2::Digest;
|
||||
|
||||
const USAGE: &str = "Usage: cir_ta_only_fixture --tal-path <path> --ta-path <path> --tal-uri <url> --validation-time <rfc3339> --cir-out <path> --static-root <path>";
|
||||
|
||||
fn parse_args(argv: &[String]) -> Result<(PathBuf, PathBuf, String, time::OffsetDateTime, PathBuf, PathBuf), String> {
|
||||
fn parse_args(
|
||||
argv: &[String],
|
||||
) -> Result<
|
||||
(
|
||||
PathBuf,
|
||||
PathBuf,
|
||||
String,
|
||||
time::OffsetDateTime,
|
||||
PathBuf,
|
||||
PathBuf,
|
||||
),
|
||||
String,
|
||||
> {
|
||||
let mut tal_path = None;
|
||||
let mut ta_path = None;
|
||||
let mut tal_uri = None;
|
||||
@@ -17,11 +32,15 @@ fn parse_args(argv: &[String]) -> Result<(PathBuf, PathBuf, String, time::Offset
|
||||
match argv[i].as_str() {
|
||||
"--tal-path" => {
|
||||
i += 1;
|
||||
tal_path = Some(PathBuf::from(argv.get(i).ok_or("--tal-path requires a value")?));
|
||||
tal_path = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--tal-path requires a value")?,
|
||||
));
|
||||
}
|
||||
"--ta-path" => {
|
||||
i += 1;
|
||||
ta_path = Some(PathBuf::from(argv.get(i).ok_or("--ta-path requires a value")?));
|
||||
ta_path = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--ta-path requires a value")?,
|
||||
));
|
||||
}
|
||||
"--tal-uri" => {
|
||||
i += 1;
|
||||
@@ -40,12 +59,15 @@ fn parse_args(argv: &[String]) -> Result<(PathBuf, PathBuf, String, time::Offset
|
||||
}
|
||||
"--cir-out" => {
|
||||
i += 1;
|
||||
cir_out = Some(PathBuf::from(argv.get(i).ok_or("--cir-out requires a value")?));
|
||||
cir_out = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--cir-out requires a value")?,
|
||||
));
|
||||
}
|
||||
"--static-root" => {
|
||||
i += 1;
|
||||
static_root =
|
||||
Some(PathBuf::from(argv.get(i).ok_or("--static-root requires a value")?));
|
||||
static_root = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--static-root requires a value")?,
|
||||
));
|
||||
}
|
||||
"-h" | "--help" => return Err(USAGE.to_string()),
|
||||
other => return Err(format!("unknown argument: {other}\n\n{USAGE}")),
|
||||
|
||||
@@ -4,8 +4,7 @@ use std::path::PathBuf;
|
||||
use rocksdb::{DB, IteratorMode, Options};
|
||||
use rpki::storage::{
|
||||
ALL_COLUMN_FAMILY_NAMES, CF_AUDIT_RULE_INDEX, CF_RAW_BY_HASH, CF_REPOSITORY_VIEW,
|
||||
CF_RRDP_SOURCE, CF_RRDP_SOURCE_MEMBER, CF_RRDP_URI_OWNER, CF_VCIR,
|
||||
column_family_descriptors,
|
||||
CF_RRDP_SOURCE, CF_RRDP_SOURCE_MEMBER, CF_RRDP_URI_OWNER, CF_VCIR, column_family_descriptors,
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -77,9 +76,7 @@ fn cf_group(cf_name: &str) -> CfGroup {
|
||||
match cf_name {
|
||||
CF_REPOSITORY_VIEW | CF_RAW_BY_HASH => CfGroup::CurrentRepositoryView,
|
||||
CF_VCIR | CF_AUDIT_RULE_INDEX => CfGroup::CurrentValidationState,
|
||||
CF_RRDP_SOURCE | CF_RRDP_SOURCE_MEMBER | CF_RRDP_URI_OWNER => {
|
||||
CfGroup::CurrentRrdpState
|
||||
}
|
||||
CF_RRDP_SOURCE | CF_RRDP_SOURCE_MEMBER | CF_RRDP_URI_OWNER => CfGroup::CurrentRrdpState,
|
||||
_ => CfGroup::LegacyCompatibility,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,10 +56,7 @@ fn parse_args() -> Result<Args, String> {
|
||||
i += 1;
|
||||
}
|
||||
if out.bundle_root.is_none() || out.out.is_none() {
|
||||
return Err(format!(
|
||||
"--bundle-root and --out are required\n{}",
|
||||
usage()
|
||||
));
|
||||
return Err(format!("--bundle-root and --out are required\n{}", usage()));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
@@ -188,10 +185,7 @@ fn real_main() -> Result<(), String> {
|
||||
.as_array()
|
||||
.ok_or("bundle missing deltaSequence.steps")?
|
||||
{
|
||||
let step_id = step["id"]
|
||||
.as_str()
|
||||
.ok_or("step missing id")?
|
||||
.to_string();
|
||||
let step_id = step["id"].as_str().ok_or("step missing id")?.to_string();
|
||||
let step_dir = path_join(
|
||||
&rir_dir,
|
||||
step["relativePath"]
|
||||
@@ -210,8 +204,9 @@ fn real_main() -> Result<(), String> {
|
||||
.as_str()
|
||||
.ok_or("step missing relativeTransitionLocksPath")?,
|
||||
);
|
||||
let validation_time = load_validation_time(&delta_locks)
|
||||
.map_err(|e| format!("load step validation time failed for {rir}/{step_id}: {e}"))?;
|
||||
let validation_time = load_validation_time(&delta_locks).map_err(|e| {
|
||||
format!("load step validation time failed for {rir}/{step_id}: {e}")
|
||||
})?;
|
||||
let start = Instant::now();
|
||||
let step_out = run_tree_from_tal_and_ta_der_payload_delta_replay_step_serial_audit(
|
||||
&store,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use rpki::bundle::{
|
||||
RirBundleMetadata, RecordingHttpFetcher, RecordingRsyncFetcher,
|
||||
build_single_rir_bundle_manifest, build_vap_compare_rows, build_vrp_compare_rows,
|
||||
sha256_hex, write_json, write_live_base_replay_bundle_inputs, write_live_bundle_rir_readme,
|
||||
RecordingHttpFetcher, RecordingRsyncFetcher, RirBundleMetadata,
|
||||
build_single_rir_bundle_manifest, build_vap_compare_rows, build_vrp_compare_rows, sha256_hex,
|
||||
write_json, write_live_base_replay_bundle_inputs, write_live_bundle_rir_readme,
|
||||
write_live_bundle_top_readme, write_timing_json, write_vap_csv, write_vrp_csv,
|
||||
};
|
||||
use rpki::ccr::{build_ccr_from_run, verify_content_info, write_ccr_file};
|
||||
@@ -54,15 +54,21 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
||||
}
|
||||
"--out-dir" => {
|
||||
i += 1;
|
||||
args.out_dir = Some(PathBuf::from(argv.get(i).ok_or("--out-dir requires a value")?));
|
||||
args.out_dir = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--out-dir requires a value")?,
|
||||
));
|
||||
}
|
||||
"--tal-path" => {
|
||||
i += 1;
|
||||
args.tal_path = Some(PathBuf::from(argv.get(i).ok_or("--tal-path requires a value")?));
|
||||
args.tal_path = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--tal-path requires a value")?,
|
||||
));
|
||||
}
|
||||
"--ta-path" => {
|
||||
i += 1;
|
||||
args.ta_path = Some(PathBuf::from(argv.get(i).ok_or("--ta-path requires a value")?));
|
||||
args.ta_path = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--ta-path requires a value")?,
|
||||
));
|
||||
}
|
||||
"--validation-time" => {
|
||||
i += 1;
|
||||
@@ -90,8 +96,9 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
||||
}
|
||||
"--rsync-mirror-root" => {
|
||||
i += 1;
|
||||
args.rsync_mirror_root =
|
||||
Some(PathBuf::from(argv.get(i).ok_or("--rsync-mirror-root requires a value")?));
|
||||
args.rsync_mirror_root = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--rsync-mirror-root requires a value")?,
|
||||
));
|
||||
}
|
||||
"--max-depth" => {
|
||||
i += 1;
|
||||
@@ -113,7 +120,11 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
||||
}
|
||||
"--trust-anchor" => {
|
||||
i += 1;
|
||||
args.trust_anchor = Some(argv.get(i).ok_or("--trust-anchor requires a value")?.clone());
|
||||
args.trust_anchor = Some(
|
||||
argv.get(i)
|
||||
.ok_or("--trust-anchor requires a value")?
|
||||
.clone(),
|
||||
);
|
||||
}
|
||||
other => return Err(format!("unknown argument: {other}\n{}", usage())),
|
||||
}
|
||||
@@ -147,11 +158,13 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
fs::create_dir_all(&rir_dir)
|
||||
.map_err(|e| format!("create rir dir failed: {}: {e}", rir_dir.display()))?;
|
||||
|
||||
let tal_bytes = fs::read(args.tal_path.as_ref().unwrap())
|
||||
.map_err(|e| format!("read tal failed: {e}"))?;
|
||||
let ta_bytes = fs::read(args.ta_path.as_ref().unwrap())
|
||||
.map_err(|e| format!("read ta failed: {e}"))?;
|
||||
let validation_time = args.validation_time.unwrap_or_else(time::OffsetDateTime::now_utc);
|
||||
let tal_bytes =
|
||||
fs::read(args.tal_path.as_ref().unwrap()).map_err(|e| format!("read tal failed: {e}"))?;
|
||||
let ta_bytes =
|
||||
fs::read(args.ta_path.as_ref().unwrap()).map_err(|e| format!("read ta failed: {e}"))?;
|
||||
let validation_time = args
|
||||
.validation_time
|
||||
.unwrap_or_else(time::OffsetDateTime::now_utc);
|
||||
|
||||
let db_dir = out_root.join(".tmp").join(format!("{rir}-live-base-db"));
|
||||
let replay_db_dir = out_root.join(".tmp").join(format!("{rir}-self-replay-db"));
|
||||
@@ -205,8 +218,8 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
.map_err(|e| format!("build ccr failed: {e}"))?;
|
||||
let base_ccr_path = rir_dir.join("base.ccr");
|
||||
write_ccr_file(&base_ccr_path, &ccr).map_err(|e| format!("write ccr failed: {e}"))?;
|
||||
let ccr_bytes =
|
||||
fs::read(&base_ccr_path).map_err(|e| format!("read written ccr failed: {}: {e}", base_ccr_path.display()))?;
|
||||
let ccr_bytes = fs::read(&base_ccr_path)
|
||||
.map_err(|e| format!("read written ccr failed: {}: {e}", base_ccr_path.display()))?;
|
||||
let decoded = rpki::ccr::decode_content_info(&ccr_bytes)
|
||||
.map_err(|e| format!("decode written ccr failed: {e}"))?;
|
||||
let verify = verify_content_info(&decoded).map_err(|e| format!("verify ccr failed: {e}"))?;
|
||||
@@ -235,8 +248,8 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
&rsync.snapshot_fetches(),
|
||||
)?;
|
||||
|
||||
let replay_store =
|
||||
RocksStore::open(&replay_db_dir).map_err(|e| format!("open self replay rocksdb failed: {e}"))?;
|
||||
let replay_store = RocksStore::open(&replay_db_dir)
|
||||
.map_err(|e| format!("open self replay rocksdb failed: {e}"))?;
|
||||
let replay_out = run_tree_from_tal_and_ta_der_payload_replay_serial_audit(
|
||||
&replay_store,
|
||||
&Policy::default(),
|
||||
@@ -320,7 +333,11 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
}),
|
||||
)?;
|
||||
write_live_bundle_top_readme(&out_root.join("README.md"), &rir_normalized)?;
|
||||
write_live_bundle_rir_readme(&rir_dir.join("README.md"), &rir_normalized, &metadata.base_validation_time)?;
|
||||
write_live_bundle_rir_readme(
|
||||
&rir_dir.join("README.md"),
|
||||
&rir_normalized,
|
||||
&metadata.base_validation_time,
|
||||
)?;
|
||||
write_json(
|
||||
&out_root.join("bundle-manifest.json"),
|
||||
&build_single_rir_bundle_manifest(
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use rpki::bundle::{
|
||||
RecordingHttpFetcher, RecordingRsyncFetcher, build_single_rir_bundle_manifest,
|
||||
build_vap_compare_rows, build_vrp_compare_rows, copy_dir_all, load_validation_time,
|
||||
sha256_hex, write_json, write_live_delta_replay_bundle_inputs, write_vap_csv, write_vrp_csv,
|
||||
build_vap_compare_rows, build_vrp_compare_rows, copy_dir_all, load_validation_time, sha256_hex,
|
||||
write_json, write_live_delta_replay_bundle_inputs, write_vap_csv, write_vrp_csv,
|
||||
};
|
||||
use rpki::ccr::{build_ccr_from_run, decode_content_info, verify_content_info, write_ccr_file};
|
||||
use rpki::fetch::http::{BlockingHttpFetcher, HttpFetcherConfig};
|
||||
@@ -54,12 +54,15 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
||||
}
|
||||
"--base-bundle-dir" => {
|
||||
i += 1;
|
||||
args.base_bundle_dir =
|
||||
Some(PathBuf::from(argv.get(i).ok_or("--base-bundle-dir requires a value")?));
|
||||
args.base_bundle_dir = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--base-bundle-dir requires a value")?,
|
||||
));
|
||||
}
|
||||
"--out-dir" => {
|
||||
i += 1;
|
||||
args.out_dir = Some(PathBuf::from(argv.get(i).ok_or("--out-dir requires a value")?));
|
||||
args.out_dir = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--out-dir requires a value")?,
|
||||
));
|
||||
}
|
||||
"--validation-time" => {
|
||||
i += 1;
|
||||
@@ -87,8 +90,9 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
||||
}
|
||||
"--rsync-mirror-root" => {
|
||||
i += 1;
|
||||
args.rsync_mirror_root =
|
||||
Some(PathBuf::from(argv.get(i).ok_or("--rsync-mirror-root requires a value")?));
|
||||
args.rsync_mirror_root = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--rsync-mirror-root requires a value")?,
|
||||
));
|
||||
}
|
||||
"--max-depth" => {
|
||||
i += 1;
|
||||
@@ -110,7 +114,11 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
||||
}
|
||||
"--trust-anchor" => {
|
||||
i += 1;
|
||||
args.trust_anchor = Some(argv.get(i).ok_or("--trust-anchor requires a value")?.clone());
|
||||
args.trust_anchor = Some(
|
||||
argv.get(i)
|
||||
.ok_or("--trust-anchor requires a value")?
|
||||
.clone(),
|
||||
);
|
||||
}
|
||||
other => return Err(format!("unknown argument: {other}\n{}", usage())),
|
||||
}
|
||||
@@ -193,7 +201,10 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
let base_root = args.base_bundle_dir.as_ref().unwrap();
|
||||
let base_rir_dir = base_root.join(&rir_normalized);
|
||||
if !base_rir_dir.is_dir() {
|
||||
return Err(format!("base bundle rir dir not found: {}", base_rir_dir.display()));
|
||||
return Err(format!(
|
||||
"base bundle rir dir not found: {}",
|
||||
base_rir_dir.display()
|
||||
));
|
||||
}
|
||||
if out_root.exists() {
|
||||
fs::remove_dir_all(out_root)
|
||||
@@ -206,12 +217,14 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
.trust_anchor
|
||||
.clone()
|
||||
.unwrap_or_else(|| rir_normalized.clone());
|
||||
let tal_bytes =
|
||||
fs::read(rir_dir.join("tal.tal")).map_err(|e| format!("read tal from base bundle failed: {e}"))?;
|
||||
let ta_bytes =
|
||||
fs::read(rir_dir.join("ta.cer")).map_err(|e| format!("read ta from base bundle failed: {e}"))?;
|
||||
let tal_bytes = fs::read(rir_dir.join("tal.tal"))
|
||||
.map_err(|e| format!("read tal from base bundle failed: {e}"))?;
|
||||
let ta_bytes = fs::read(rir_dir.join("ta.cer"))
|
||||
.map_err(|e| format!("read ta from base bundle failed: {e}"))?;
|
||||
let base_validation_time = load_validation_time(&rir_dir.join("base-locks.json"))?;
|
||||
let target_validation_time = args.validation_time.unwrap_or_else(time::OffsetDateTime::now_utc);
|
||||
let target_validation_time = args
|
||||
.validation_time
|
||||
.unwrap_or_else(time::OffsetDateTime::now_utc);
|
||||
|
||||
let target_store_dir = out_root.join(".tmp").join(format!("{rir}-live-target-db"));
|
||||
let self_replay_dir = out_root.join(".tmp").join(format!("{rir}-self-delta-db"));
|
||||
@@ -221,8 +234,8 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
fs::create_dir_all(parent)
|
||||
.map_err(|e| format!("create tmp dir failed: {}: {e}", parent.display()))?;
|
||||
}
|
||||
let target_store =
|
||||
RocksStore::open(&target_store_dir).map_err(|e| format!("open target rocksdb failed: {e}"))?;
|
||||
let target_store = RocksStore::open(&target_store_dir)
|
||||
.map_err(|e| format!("open target rocksdb failed: {e}"))?;
|
||||
|
||||
let _base = run_tree_from_tal_and_ta_der_payload_replay_serial_audit(
|
||||
&target_store,
|
||||
@@ -282,17 +295,19 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
)
|
||||
.map_err(|e| format!("build delta ccr failed: {e}"))?;
|
||||
let delta_ccr_path = rir_dir.join("delta.ccr");
|
||||
write_ccr_file(&delta_ccr_path, &delta_ccr).map_err(|e| format!("write delta ccr failed: {e}"))?;
|
||||
let delta_ccr_bytes =
|
||||
fs::read(&delta_ccr_path).map_err(|e| format!("read delta ccr failed: {}: {e}", delta_ccr_path.display()))?;
|
||||
let delta_decoded =
|
||||
decode_content_info(&delta_ccr_bytes).map_err(|e| format!("decode delta ccr failed: {e}"))?;
|
||||
write_ccr_file(&delta_ccr_path, &delta_ccr)
|
||||
.map_err(|e| format!("write delta ccr failed: {e}"))?;
|
||||
let delta_ccr_bytes = fs::read(&delta_ccr_path)
|
||||
.map_err(|e| format!("read delta ccr failed: {}: {e}", delta_ccr_path.display()))?;
|
||||
let delta_decoded = decode_content_info(&delta_ccr_bytes)
|
||||
.map_err(|e| format!("decode delta ccr failed: {e}"))?;
|
||||
let delta_verify =
|
||||
verify_content_info(&delta_decoded).map_err(|e| format!("verify delta ccr failed: {e}"))?;
|
||||
|
||||
let delta_vrp_rows = build_vrp_compare_rows(&target_out.tree.vrps, &trust_anchor);
|
||||
let delta_vap_rows = build_vap_compare_rows(&target_out.tree.aspas, &trust_anchor);
|
||||
let (ccr_vrps, ccr_vaps) = rpki::bundle::decode_ccr_compare_views(&delta_decoded, &trust_anchor)?;
|
||||
let (ccr_vrps, ccr_vaps) =
|
||||
rpki::bundle::decode_ccr_compare_views(&delta_decoded, &trust_anchor)?;
|
||||
if delta_vrp_rows != ccr_vrps {
|
||||
return Err("record-delta.csv compare view does not match delta.ccr".to_string());
|
||||
}
|
||||
@@ -312,8 +327,8 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
&rsync.snapshot_fetches(),
|
||||
)?;
|
||||
|
||||
let self_store =
|
||||
RocksStore::open(&self_replay_dir).map_err(|e| format!("open self replay db failed: {e}"))?;
|
||||
let self_store = RocksStore::open(&self_replay_dir)
|
||||
.map_err(|e| format!("open self replay db failed: {e}"))?;
|
||||
let replay_out = run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit(
|
||||
&self_store,
|
||||
&Policy::default(),
|
||||
@@ -355,7 +370,8 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
)?;
|
||||
|
||||
let mut bundle_json: serde_json::Value = serde_json::from_slice(
|
||||
&fs::read(rir_dir.join("bundle.json")).map_err(|e| format!("read base bundle.json failed: {e}"))?,
|
||||
&fs::read(rir_dir.join("bundle.json"))
|
||||
.map_err(|e| format!("read base bundle.json failed: {e}"))?,
|
||||
)
|
||||
.map_err(|e| format!("parse base bundle.json failed: {e}"))?;
|
||||
bundle_json["deltaValidationTime"] = serde_json::Value::String(
|
||||
|
||||
@@ -65,19 +65,27 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
||||
}
|
||||
"--out-dir" => {
|
||||
i += 1;
|
||||
args.out_dir = Some(PathBuf::from(argv.get(i).ok_or("--out-dir requires a value")?));
|
||||
args.out_dir = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--out-dir requires a value")?,
|
||||
));
|
||||
}
|
||||
"--tal-path" => {
|
||||
i += 1;
|
||||
args.tal_path = Some(PathBuf::from(argv.get(i).ok_or("--tal-path requires a value")?));
|
||||
args.tal_path = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--tal-path requires a value")?,
|
||||
));
|
||||
}
|
||||
"--ta-path" => {
|
||||
i += 1;
|
||||
args.ta_path = Some(PathBuf::from(argv.get(i).ok_or("--ta-path requires a value")?));
|
||||
args.ta_path = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--ta-path requires a value")?,
|
||||
));
|
||||
}
|
||||
"--base-validation-time" => {
|
||||
i += 1;
|
||||
let value = argv.get(i).ok_or("--base-validation-time requires a value")?;
|
||||
let value = argv
|
||||
.get(i)
|
||||
.ok_or("--base-validation-time requires a value")?;
|
||||
args.base_validation_time = Some(
|
||||
time::OffsetDateTime::parse(value, &Rfc3339)
|
||||
.map_err(|e| format!("invalid --base-validation-time: {e}"))?,
|
||||
@@ -117,8 +125,9 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
||||
}
|
||||
"--rsync-mirror-root" => {
|
||||
i += 1;
|
||||
args.rsync_mirror_root =
|
||||
Some(PathBuf::from(argv.get(i).ok_or("--rsync-mirror-root requires a value")?));
|
||||
args.rsync_mirror_root = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--rsync-mirror-root requires a value")?,
|
||||
));
|
||||
}
|
||||
"--max-depth" => {
|
||||
i += 1;
|
||||
@@ -140,7 +149,11 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
||||
}
|
||||
"--trust-anchor" => {
|
||||
i += 1;
|
||||
args.trust_anchor = Some(argv.get(i).ok_or("--trust-anchor requires a value")?.clone());
|
||||
args.trust_anchor = Some(
|
||||
argv.get(i)
|
||||
.ok_or("--trust-anchor requires a value")?
|
||||
.clone(),
|
||||
);
|
||||
}
|
||||
"--keep-db" => args.keep_db = true,
|
||||
"--capture-inputs-only" => args.capture_inputs_only = true,
|
||||
@@ -164,7 +177,12 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
||||
Ok(args)
|
||||
}
|
||||
|
||||
fn write_v2_top_readme(path: &Path, rir: &str, delta_count: usize, delta_interval_secs: u64) -> Result<(), String> {
|
||||
fn write_v2_top_readme(
|
||||
path: &Path,
|
||||
rir: &str,
|
||||
delta_count: usize,
|
||||
delta_interval_secs: u64,
|
||||
) -> Result<(), String> {
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.map_err(|e| format!("create parent failed: {}: {e}", parent.display()))?;
|
||||
@@ -203,11 +221,19 @@ fn ensure_recorded_target_snapshots_for_locks(
|
||||
previous_locks_path: &Path,
|
||||
http: &RecordingHttpFetcher<BlockingHttpFetcher>,
|
||||
) -> Result<(), String> {
|
||||
let previous_locks: serde_json::Value = serde_json::from_slice(
|
||||
&fs::read(previous_locks_path)
|
||||
.map_err(|e| format!("read previous locks failed: {}: {e}", previous_locks_path.display()))?,
|
||||
)
|
||||
.map_err(|e| format!("parse previous locks failed: {}: {e}", previous_locks_path.display()))?;
|
||||
let previous_locks: serde_json::Value =
|
||||
serde_json::from_slice(&fs::read(previous_locks_path).map_err(|e| {
|
||||
format!(
|
||||
"read previous locks failed: {}: {e}",
|
||||
previous_locks_path.display()
|
||||
)
|
||||
})?)
|
||||
.map_err(|e| {
|
||||
format!(
|
||||
"parse previous locks failed: {}: {e}",
|
||||
previous_locks_path.display()
|
||||
)
|
||||
})?;
|
||||
let previous_rrdp = previous_locks
|
||||
.get("rrdp")
|
||||
.and_then(|v| v.as_object())
|
||||
@@ -251,9 +277,7 @@ fn ensure_recorded_target_snapshots_for_locks(
|
||||
if let Err(err) = http.fetch(snapshot_uri) {
|
||||
eprintln!(
|
||||
"[sequence] warning: fetch target snapshot failed notify_uri={} snapshot_uri={} err={}",
|
||||
notify_uri,
|
||||
snapshot_uri,
|
||||
err
|
||||
notify_uri, snapshot_uri, err
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -281,20 +305,30 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
}
|
||||
let rir_dir = out_root.join(&rir_normalized);
|
||||
let delta_steps_root = rir_dir.join("delta-steps");
|
||||
fs::create_dir_all(&delta_steps_root)
|
||||
.map_err(|e| format!("create delta steps dir failed: {}: {e}", delta_steps_root.display()))?;
|
||||
fs::create_dir_all(&delta_steps_root).map_err(|e| {
|
||||
format!(
|
||||
"create delta steps dir failed: {}: {e}",
|
||||
delta_steps_root.display()
|
||||
)
|
||||
})?;
|
||||
|
||||
let tal_bytes = fs::read(args.tal_path.as_ref().unwrap())
|
||||
.map_err(|e| format!("read tal failed: {e}"))?;
|
||||
let ta_bytes = fs::read(args.ta_path.as_ref().unwrap())
|
||||
.map_err(|e| format!("read ta failed: {e}"))?;
|
||||
let tal_bytes =
|
||||
fs::read(args.tal_path.as_ref().unwrap()).map_err(|e| format!("read tal failed: {e}"))?;
|
||||
let ta_bytes =
|
||||
fs::read(args.ta_path.as_ref().unwrap()).map_err(|e| format!("read ta failed: {e}"))?;
|
||||
fs::write(rir_dir.join("tal.tal"), &tal_bytes).map_err(|e| format!("write tal failed: {e}"))?;
|
||||
fs::write(rir_dir.join("ta.cer"), &ta_bytes).map_err(|e| format!("write ta failed: {e}"))?;
|
||||
|
||||
let base_validation_time = args.base_validation_time.unwrap_or_else(time::OffsetDateTime::now_utc);
|
||||
let base_validation_time = args
|
||||
.base_validation_time
|
||||
.unwrap_or_else(time::OffsetDateTime::now_utc);
|
||||
|
||||
let work_db_dir = out_root.join(".tmp").join(format!("{rir}-sequence-work-db"));
|
||||
let base_self_replay_dir = out_root.join(".tmp").join(format!("{rir}-sequence-base-self-replay-db"));
|
||||
let work_db_dir = out_root
|
||||
.join(".tmp")
|
||||
.join(format!("{rir}-sequence-work-db"));
|
||||
let base_self_replay_dir = out_root
|
||||
.join(".tmp")
|
||||
.join(format!("{rir}-sequence-base-self-replay-db"));
|
||||
let _ = fs::remove_dir_all(&work_db_dir);
|
||||
let _ = fs::remove_dir_all(&base_self_replay_dir);
|
||||
if let Some(parent) = work_db_dir.parent() {
|
||||
@@ -356,9 +390,7 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
)?;
|
||||
eprintln!(
|
||||
"[sequence] base input materialization done rir={} rrdp_repos={} rsync_modules={}",
|
||||
rir_normalized,
|
||||
base_capture.rrdp_repo_count,
|
||||
base_capture.rsync_module_count
|
||||
rir_normalized, base_capture.rrdp_repo_count, base_capture.rsync_module_count
|
||||
);
|
||||
let base_ccr_path = rir_dir.join("base.ccr");
|
||||
let base_vrps_path = rir_dir.join("base-vrps.csv");
|
||||
@@ -402,8 +434,8 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
.map_err(|e| format!("read base ccr failed: {}: {e}", base_ccr_path.display()))?;
|
||||
let base_decoded = decode_content_info(&base_ccr_bytes)
|
||||
.map_err(|e| format!("decode base ccr failed: {e}"))?;
|
||||
let base_verify =
|
||||
verify_content_info(&base_decoded).map_err(|e| format!("verify base ccr failed: {e}"))?;
|
||||
let base_verify = verify_content_info(&base_decoded)
|
||||
.map_err(|e| format!("verify base ccr failed: {e}"))?;
|
||||
let base_vrp_rows = build_vrp_compare_rows(&base_out.tree.vrps, &trust_anchor);
|
||||
let base_vap_rows = build_vap_compare_rows(&base_out.tree.aspas, &trust_anchor);
|
||||
let (base_ccr_vrps, base_ccr_vaps) =
|
||||
@@ -464,7 +496,9 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
let mut delta_steps = Vec::new();
|
||||
let mut previous_locks_path = rir_dir.join("base-locks.json");
|
||||
let mut previous_ref = "base".to_string();
|
||||
let sequence_self_replay_dir = out_root.join(".tmp").join(format!("{rir}-sequence-self-replay-db"));
|
||||
let sequence_self_replay_dir = out_root
|
||||
.join(".tmp")
|
||||
.join(format!("{rir}-sequence-self-replay-db"));
|
||||
let _ = fs::remove_dir_all(&sequence_self_replay_dir);
|
||||
let sequence_replay_store = if args.capture_inputs_only {
|
||||
None
|
||||
@@ -499,7 +533,10 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
fs::create_dir_all(&step_dir)
|
||||
.map_err(|e| format!("create step dir failed: {}: {e}", step_dir.display()))?;
|
||||
let step_validation_time = time::OffsetDateTime::now_utc();
|
||||
eprintln!("[sequence] step live run start rir={} step={}", rir_normalized, step_id);
|
||||
eprintln!(
|
||||
"[sequence] step live run start rir={} step={}",
|
||||
rir_normalized, step_id
|
||||
);
|
||||
let step_http = RecordingHttpFetcher::new(
|
||||
BlockingHttpFetcher::new(HttpFetcherConfig {
|
||||
timeout: Duration::from_secs(args.http_timeout_secs),
|
||||
@@ -542,7 +579,10 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
);
|
||||
ensure_recorded_target_snapshots_for_locks(&store, &previous_locks_path, &step_http)?;
|
||||
|
||||
eprintln!("[sequence] step output generation phase start rir={} step={}", rir_normalized, step_id);
|
||||
eprintln!(
|
||||
"[sequence] step output generation phase start rir={} step={}",
|
||||
rir_normalized, step_id
|
||||
);
|
||||
let delta_ccr_path = step_dir.join("delta.ccr");
|
||||
let delta_vrps_path = step_dir.join("record-delta.csv");
|
||||
let delta_vaps_path = step_dir.join("record-delta-vaps.csv");
|
||||
@@ -550,8 +590,7 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
if args.capture_inputs_only {
|
||||
eprintln!(
|
||||
"[sequence] step CCR/self-replay skipped rir={} step={}",
|
||||
rir_normalized,
|
||||
step_id
|
||||
rir_normalized, step_id
|
||||
);
|
||||
(
|
||||
String::new(),
|
||||
@@ -573,7 +612,10 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
false,
|
||||
)
|
||||
} else {
|
||||
eprintln!("[sequence] step CCR/self-replay start rir={} step={}", rir_normalized, step_id);
|
||||
eprintln!(
|
||||
"[sequence] step CCR/self-replay start rir={} step={}",
|
||||
rir_normalized, step_id
|
||||
);
|
||||
let delta_ccr = build_ccr_from_run(
|
||||
&store,
|
||||
&[step_out.discovery.trust_anchor.clone()],
|
||||
@@ -597,31 +639,39 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
let (ccr_vrps, ccr_vaps) =
|
||||
rpki::bundle::decode_ccr_compare_views(&delta_decoded, &trust_anchor)?;
|
||||
if delta_vrp_rows != ccr_vrps {
|
||||
return Err(format!("{step_id} VRP compare view does not match delta.ccr"));
|
||||
return Err(format!(
|
||||
"{step_id} VRP compare view does not match delta.ccr"
|
||||
));
|
||||
}
|
||||
if delta_vap_rows != ccr_vaps {
|
||||
return Err(format!("{step_id} VAP compare view does not match delta.ccr"));
|
||||
return Err(format!(
|
||||
"{step_id} VAP compare view does not match delta.ccr"
|
||||
));
|
||||
}
|
||||
write_vrp_csv(&delta_vrps_path, &delta_vrp_rows)?;
|
||||
write_vap_csv(&delta_vaps_path, &delta_vap_rows)?;
|
||||
let step_replay_out = run_tree_from_tal_and_ta_der_payload_delta_replay_step_serial_audit(
|
||||
sequence_replay_store.as_ref().expect("sequence replay store"),
|
||||
&Policy::default(),
|
||||
&tal_bytes,
|
||||
&ta_bytes,
|
||||
None,
|
||||
&step_dir.join("payload-delta-archive"),
|
||||
&previous_locks_path,
|
||||
&step_dir.join("locks-delta.json"),
|
||||
step_validation_time,
|
||||
&TreeRunConfig {
|
||||
max_depth: args.max_depth,
|
||||
max_instances: args.max_instances,
|
||||
},
|
||||
)
|
||||
.map_err(|e| format!("sequence self replay failed for {step_id}: {e}"))?;
|
||||
let step_replay_out =
|
||||
run_tree_from_tal_and_ta_der_payload_delta_replay_step_serial_audit(
|
||||
sequence_replay_store
|
||||
.as_ref()
|
||||
.expect("sequence replay store"),
|
||||
&Policy::default(),
|
||||
&tal_bytes,
|
||||
&ta_bytes,
|
||||
None,
|
||||
&step_dir.join("payload-delta-archive"),
|
||||
&previous_locks_path,
|
||||
&step_dir.join("locks-delta.json"),
|
||||
step_validation_time,
|
||||
&TreeRunConfig {
|
||||
max_depth: args.max_depth,
|
||||
max_instances: args.max_instances,
|
||||
},
|
||||
)
|
||||
.map_err(|e| format!("sequence self replay failed for {step_id}: {e}"))?;
|
||||
let step_self_replay_ok =
|
||||
build_vrp_compare_rows(&step_replay_out.tree.vrps, &trust_anchor) == delta_vrp_rows
|
||||
build_vrp_compare_rows(&step_replay_out.tree.vrps, &trust_anchor)
|
||||
== delta_vrp_rows
|
||||
&& build_vap_compare_rows(&step_replay_out.tree.aspas, &trust_anchor)
|
||||
== delta_vap_rows;
|
||||
let output = (
|
||||
@@ -631,11 +681,17 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
delta_verify,
|
||||
step_self_replay_ok,
|
||||
);
|
||||
eprintln!("[sequence] step CCR/self-replay done rir={} step={}", rir_normalized, step_id);
|
||||
eprintln!(
|
||||
"[sequence] step CCR/self-replay done rir={} step={}",
|
||||
rir_normalized, step_id
|
||||
);
|
||||
output
|
||||
};
|
||||
|
||||
eprintln!("[sequence] step input materialization start rir={} step={}", rir_normalized, step_id);
|
||||
eprintln!(
|
||||
"[sequence] step input materialization start rir={} step={}",
|
||||
rir_normalized, step_id
|
||||
);
|
||||
let delta_capture = write_live_delta_replay_step_inputs(
|
||||
&step_dir,
|
||||
&rir_normalized,
|
||||
@@ -816,7 +872,12 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
}
|
||||
}),
|
||||
)?;
|
||||
write_v2_top_readme(&out_root.join("README.md"), &rir_normalized, args.delta_count, args.delta_interval_secs)?;
|
||||
write_v2_top_readme(
|
||||
&out_root.join("README.md"),
|
||||
&rir_normalized,
|
||||
args.delta_count,
|
||||
args.delta_interval_secs,
|
||||
)?;
|
||||
write_v2_rir_readme(
|
||||
&rir_dir.join("README.md"),
|
||||
&rir_normalized,
|
||||
|
||||
@@ -48,35 +48,49 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
||||
}
|
||||
"--out-dir" => {
|
||||
i += 1;
|
||||
args.out_dir = Some(PathBuf::from(argv.get(i).ok_or("--out-dir requires a value")?));
|
||||
args.out_dir = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--out-dir requires a value")?,
|
||||
));
|
||||
}
|
||||
"--tal-path" => {
|
||||
i += 1;
|
||||
args.tal_path = Some(PathBuf::from(argv.get(i).ok_or("--tal-path requires a value")?));
|
||||
args.tal_path = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--tal-path requires a value")?,
|
||||
));
|
||||
}
|
||||
"--ta-path" => {
|
||||
i += 1;
|
||||
args.ta_path = Some(PathBuf::from(argv.get(i).ok_or("--ta-path requires a value")?));
|
||||
args.ta_path = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--ta-path requires a value")?,
|
||||
));
|
||||
}
|
||||
"--payload-replay-archive" => {
|
||||
i += 1;
|
||||
args.payload_replay_archive =
|
||||
Some(PathBuf::from(argv.get(i).ok_or("--payload-replay-archive requires a value")?));
|
||||
args.payload_replay_archive = Some(PathBuf::from(
|
||||
argv.get(i)
|
||||
.ok_or("--payload-replay-archive requires a value")?,
|
||||
));
|
||||
}
|
||||
"--payload-replay-locks" => {
|
||||
i += 1;
|
||||
args.payload_replay_locks =
|
||||
Some(PathBuf::from(argv.get(i).ok_or("--payload-replay-locks requires a value")?));
|
||||
args.payload_replay_locks = Some(PathBuf::from(
|
||||
argv.get(i)
|
||||
.ok_or("--payload-replay-locks requires a value")?,
|
||||
));
|
||||
}
|
||||
"--payload-delta-archive" => {
|
||||
i += 1;
|
||||
args.payload_delta_archive =
|
||||
Some(PathBuf::from(argv.get(i).ok_or("--payload-delta-archive requires a value")?));
|
||||
args.payload_delta_archive = Some(PathBuf::from(
|
||||
argv.get(i)
|
||||
.ok_or("--payload-delta-archive requires a value")?,
|
||||
));
|
||||
}
|
||||
"--payload-delta-locks" => {
|
||||
i += 1;
|
||||
args.payload_delta_locks =
|
||||
Some(PathBuf::from(argv.get(i).ok_or("--payload-delta-locks requires a value")?));
|
||||
args.payload_delta_locks = Some(PathBuf::from(
|
||||
argv.get(i)
|
||||
.ok_or("--payload-delta-locks requires a value")?,
|
||||
));
|
||||
}
|
||||
"--validation-time" => {
|
||||
i += 1;
|
||||
@@ -106,7 +120,11 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
||||
}
|
||||
"--trust-anchor" => {
|
||||
i += 1;
|
||||
args.trust_anchor = Some(argv.get(i).ok_or("--trust-anchor requires a value")?.clone());
|
||||
args.trust_anchor = Some(
|
||||
argv.get(i)
|
||||
.ok_or("--trust-anchor requires a value")?
|
||||
.clone(),
|
||||
);
|
||||
}
|
||||
other => return Err(format!("unknown argument: {other}\n{}", usage())),
|
||||
}
|
||||
@@ -155,7 +173,9 @@ fn sha256_hex(bytes: &[u8]) -> String {
|
||||
fn copy_dir_all(src: &Path, dst: &Path) -> Result<(), String> {
|
||||
fs::create_dir_all(dst)
|
||||
.map_err(|e| format!("create directory failed: {}: {e}", dst.display()))?;
|
||||
for entry in fs::read_dir(src).map_err(|e| format!("read_dir failed: {}: {e}", src.display()))? {
|
||||
for entry in
|
||||
fs::read_dir(src).map_err(|e| format!("read_dir failed: {}: {e}", src.display()))?
|
||||
{
|
||||
let entry = entry.map_err(|e| format!("read_dir entry failed: {}: {e}", src.display()))?;
|
||||
let ty = entry
|
||||
.file_type()
|
||||
@@ -168,8 +188,13 @@ fn copy_dir_all(src: &Path, dst: &Path) -> Result<(), String> {
|
||||
fs::create_dir_all(parent)
|
||||
.map_err(|e| format!("create parent failed: {}: {e}", parent.display()))?;
|
||||
}
|
||||
fs::copy(entry.path(), &to)
|
||||
.map_err(|e| format!("copy failed: {} -> {}: {e}", entry.path().display(), to.display()))?;
|
||||
fs::copy(entry.path(), &to).map_err(|e| {
|
||||
format!(
|
||||
"copy failed: {} -> {}: {e}",
|
||||
entry.path().display(),
|
||||
to.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -222,7 +247,10 @@ fn write_timing_json(
|
||||
)
|
||||
}
|
||||
|
||||
fn rewrite_delta_base_locks_sha(delta_root: &Path, emitted_base_locks_sha256: &str) -> Result<(), String> {
|
||||
fn rewrite_delta_base_locks_sha(
|
||||
delta_root: &Path,
|
||||
emitted_base_locks_sha256: &str,
|
||||
) -> Result<(), String> {
|
||||
let delta_locks = delta_root.join("locks-delta.json");
|
||||
if delta_locks.is_file() {
|
||||
let mut json: serde_json::Value = serde_json::from_slice(
|
||||
@@ -243,7 +271,8 @@ fn rewrite_delta_base_locks_sha(delta_root: &Path, emitted_base_locks_sha256: &s
|
||||
if archive_root.is_dir() {
|
||||
for path in walk_json_files_named(&archive_root, "base.json")? {
|
||||
let mut json: serde_json::Value = serde_json::from_slice(
|
||||
&fs::read(&path).map_err(|e| format!("read base.json failed: {}: {e}", path.display()))?,
|
||||
&fs::read(&path)
|
||||
.map_err(|e| format!("read base.json failed: {}: {e}", path.display()))?,
|
||||
)
|
||||
.map_err(|e| format!("parse base.json failed: {}: {e}", path.display()))?;
|
||||
json.as_object_mut()
|
||||
@@ -265,8 +294,11 @@ fn walk_json_files_named(root: &Path, name: &str) -> Result<Vec<PathBuf>, String
|
||||
}
|
||||
let mut stack = vec![root.to_path_buf()];
|
||||
while let Some(dir) = stack.pop() {
|
||||
for entry in fs::read_dir(&dir).map_err(|e| format!("read_dir failed: {}: {e}", dir.display()))? {
|
||||
let entry = entry.map_err(|e| format!("read_dir entry failed: {}: {e}", dir.display()))?;
|
||||
for entry in
|
||||
fs::read_dir(&dir).map_err(|e| format!("read_dir failed: {}: {e}", dir.display()))?
|
||||
{
|
||||
let entry =
|
||||
entry.map_err(|e| format!("read_dir entry failed: {}: {e}", dir.display()))?;
|
||||
let path = entry.path();
|
||||
let ty = entry
|
||||
.file_type()
|
||||
@@ -308,8 +340,10 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
fs::create_dir_all(&rir_dir)
|
||||
.map_err(|e| format!("create rir dir failed: {}: {e}", rir_dir.display()))?;
|
||||
|
||||
let tal_bytes = fs::read(tal_path).map_err(|e| format!("read tal failed: {}: {e}", tal_path.display()))?;
|
||||
let ta_bytes = fs::read(ta_path).map_err(|e| format!("read ta failed: {}: {e}", ta_path.display()))?;
|
||||
let tal_bytes =
|
||||
fs::read(tal_path).map_err(|e| format!("read tal failed: {}: {e}", tal_path.display()))?;
|
||||
let ta_bytes =
|
||||
fs::read(ta_path).map_err(|e| format!("read ta failed: {}: {e}", ta_path.display()))?;
|
||||
|
||||
let db_dir = run_root.join(".tmp").join(format!("{rir}-base-db"));
|
||||
if db_dir.exists() {
|
||||
@@ -354,7 +388,8 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
write_ccr_file(&base_ccr_path, &ccr).map_err(|e| format!("write ccr failed: {e}"))?;
|
||||
let ccr_bytes = fs::read(&base_ccr_path)
|
||||
.map_err(|e| format!("read written ccr failed: {}: {e}", base_ccr_path.display()))?;
|
||||
let decoded = decode_content_info(&ccr_bytes).map_err(|e| format!("decode written ccr failed: {e}"))?;
|
||||
let decoded =
|
||||
decode_content_info(&ccr_bytes).map_err(|e| format!("decode written ccr failed: {e}"))?;
|
||||
let verify = verify_content_info(&decoded).map_err(|e| format!("verify ccr failed: {e}"))?;
|
||||
|
||||
let vrp_rows = build_vrp_compare_rows(&out.tree.vrps, &trust_anchor);
|
||||
@@ -385,10 +420,13 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
);
|
||||
let emitted_base_locks_path = rir_dir.join("base-locks.json");
|
||||
write_json(&emitted_base_locks_path, &base_locks_json)?;
|
||||
let emitted_base_locks_sha256 = sha256_hex(
|
||||
&fs::read(&emitted_base_locks_path)
|
||||
.map_err(|e| format!("read emitted base locks failed: {}: {e}", emitted_base_locks_path.display()))?,
|
||||
);
|
||||
let emitted_base_locks_sha256 =
|
||||
sha256_hex(&fs::read(&emitted_base_locks_path).map_err(|e| {
|
||||
format!(
|
||||
"read emitted base locks failed: {}: {e}",
|
||||
emitted_base_locks_path.display()
|
||||
)
|
||||
})?);
|
||||
|
||||
if let Some(delta_archive) = args.payload_delta_archive.as_ref() {
|
||||
copy_dir_all(delta_archive, &rir_dir.join("payload-delta-archive"))?;
|
||||
@@ -418,10 +456,8 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
rewrite_delta_base_locks_sha(&rir_dir, &emitted_base_locks_sha256)?;
|
||||
}
|
||||
|
||||
fs::write(rir_dir.join("tal.tal"), &tal_bytes)
|
||||
.map_err(|e| format!("write tal failed: {e}"))?;
|
||||
fs::write(rir_dir.join("ta.cer"), &ta_bytes)
|
||||
.map_err(|e| format!("write ta failed: {e}"))?;
|
||||
fs::write(rir_dir.join("tal.tal"), &tal_bytes).map_err(|e| format!("write tal failed: {e}"))?;
|
||||
fs::write(rir_dir.join("ta.cer"), &ta_bytes).map_err(|e| format!("write ta failed: {e}"))?;
|
||||
|
||||
let mut metadata = RirBundleMetadata {
|
||||
schema_version: "20260330-v1".to_string(),
|
||||
@@ -484,15 +520,19 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
) {
|
||||
let delta_db_dir = run_root.join(".tmp").join(format!("{rir}-delta-db"));
|
||||
if delta_db_dir.exists() {
|
||||
fs::remove_dir_all(&delta_db_dir)
|
||||
.map_err(|e| format!("remove old delta db failed: {}: {e}", delta_db_dir.display()))?;
|
||||
fs::remove_dir_all(&delta_db_dir).map_err(|e| {
|
||||
format!(
|
||||
"remove old delta db failed: {}: {e}",
|
||||
delta_db_dir.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
if let Some(parent) = delta_db_dir.parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.map_err(|e| format!("create delta db parent failed: {}: {e}", parent.display()))?;
|
||||
}
|
||||
let delta_store =
|
||||
RocksStore::open(&delta_db_dir).map_err(|e| format!("open delta rocksdb failed: {e}"))?;
|
||||
let delta_store = RocksStore::open(&delta_db_dir)
|
||||
.map_err(|e| format!("open delta rocksdb failed: {e}"))?;
|
||||
let delta_started = Instant::now();
|
||||
let delta_out = run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit(
|
||||
&delta_store,
|
||||
@@ -527,16 +567,21 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
let delta_ccr_path = rir_dir.join("delta.ccr");
|
||||
write_ccr_file(&delta_ccr_path, &delta_ccr)
|
||||
.map_err(|e| format!("write delta ccr failed: {e}"))?;
|
||||
let delta_ccr_bytes = fs::read(&delta_ccr_path)
|
||||
.map_err(|e| format!("read written delta ccr failed: {}: {e}", delta_ccr_path.display()))?;
|
||||
let delta_ccr_bytes = fs::read(&delta_ccr_path).map_err(|e| {
|
||||
format!(
|
||||
"read written delta ccr failed: {}: {e}",
|
||||
delta_ccr_path.display()
|
||||
)
|
||||
})?;
|
||||
let delta_decoded = decode_content_info(&delta_ccr_bytes)
|
||||
.map_err(|e| format!("decode written delta ccr failed: {e}"))?;
|
||||
let delta_verify =
|
||||
verify_content_info(&delta_decoded).map_err(|e| format!("verify delta ccr failed: {e}"))?;
|
||||
let delta_verify = verify_content_info(&delta_decoded)
|
||||
.map_err(|e| format!("verify delta ccr failed: {e}"))?;
|
||||
|
||||
let delta_vrp_rows = build_vrp_compare_rows(&delta_out.tree.vrps, &trust_anchor);
|
||||
let delta_vap_rows = build_vap_compare_rows(&delta_out.tree.aspas, &trust_anchor);
|
||||
let (delta_ccr_vrps, delta_ccr_vaps) = decode_ccr_compare_views(&delta_decoded, &trust_anchor)?;
|
||||
let (delta_ccr_vrps, delta_ccr_vaps) =
|
||||
decode_ccr_compare_views(&delta_decoded, &trust_anchor)?;
|
||||
if delta_vrp_rows != delta_ccr_vrps {
|
||||
return Err("record-delta.csv compare view does not match delta.ccr".to_string());
|
||||
}
|
||||
@@ -585,7 +630,11 @@ fn run(args: Args) -> Result<PathBuf, String> {
|
||||
write_json(&rir_dir.join("bundle.json"), &metadata)?;
|
||||
write_json(&rir_dir.join("verification.json"), &verification)?;
|
||||
write_top_readme(&run_root.join("README.md"), rir)?;
|
||||
write_rir_readme(&rir_dir.join("README.md"), rir, &metadata.base_validation_time)?;
|
||||
write_rir_readme(
|
||||
&rir_dir.join("README.md"),
|
||||
rir,
|
||||
&metadata.base_validation_time,
|
||||
)?;
|
||||
|
||||
let bundle_manifest = BundleManifest {
|
||||
schema_version: "20260330-v1".to_string(),
|
||||
@@ -654,11 +703,8 @@ mod tests {
|
||||
fn load_validation_time_reads_top_level_validation_time() {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
let path = dir.path().join("locks.json");
|
||||
std::fs::write(
|
||||
&path,
|
||||
r#"{"validationTime":"2026-03-16T11:49:15+08:00"}"#,
|
||||
)
|
||||
.expect("write locks");
|
||||
std::fs::write(&path, r#"{"validationTime":"2026-03-16T11:49:15+08:00"}"#)
|
||||
.expect("write locks");
|
||||
let got = load_validation_time(&path).expect("load validation time");
|
||||
assert_eq!(
|
||||
got.format(&Rfc3339).expect("format"),
|
||||
@@ -722,7 +768,12 @@ mod tests {
|
||||
assert!(out_dir.join("apnic").join("base-vaps.csv").is_file());
|
||||
assert!(out_dir.join("apnic").join("delta.ccr").is_file());
|
||||
assert!(out_dir.join("apnic").join("record-delta.csv").is_file());
|
||||
assert!(out_dir.join("apnic").join("record-delta-vaps.csv").is_file());
|
||||
assert!(
|
||||
out_dir
|
||||
.join("apnic")
|
||||
.join("record-delta-vaps.csv")
|
||||
.is_file()
|
||||
);
|
||||
assert!(out_dir.join("apnic").join("verification.json").is_file());
|
||||
let bundle_json: serde_json::Value = serde_json::from_slice(
|
||||
&std::fs::read(out_dir.join("apnic").join("bundle.json")).expect("read bundle.json"),
|
||||
@@ -734,11 +785,12 @@ mod tests {
|
||||
assert!(bundle_json.get("baseCcrSha256").is_some());
|
||||
assert!(bundle_json.get("deltaVrpCount").is_some());
|
||||
assert!(bundle_json.get("deltaCcrSha256").is_some());
|
||||
let base_locks_bytes =
|
||||
std::fs::read(out_dir.join("apnic").join("base-locks.json")).expect("read emitted base locks");
|
||||
let base_locks_bytes = std::fs::read(out_dir.join("apnic").join("base-locks.json"))
|
||||
.expect("read emitted base locks");
|
||||
let expected_base_locks_sha = sha256_hex(&base_locks_bytes);
|
||||
let delta_locks_json: serde_json::Value = serde_json::from_slice(
|
||||
&std::fs::read(out_dir.join("apnic").join("locks-delta.json")).expect("read delta locks"),
|
||||
&std::fs::read(out_dir.join("apnic").join("locks-delta.json"))
|
||||
.expect("read delta locks"),
|
||||
)
|
||||
.expect("parse delta locks");
|
||||
assert_eq!(delta_locks_json["baseLocksSha256"], expected_base_locks_sha);
|
||||
|
||||
@@ -177,16 +177,24 @@ fn materialize_rsync_module_from_store(
|
||||
.strip_prefix("rsync://")
|
||||
.ok_or_else(|| format!("invalid rsync module uri: {module_uri}"))?
|
||||
.trim_end_matches('/');
|
||||
fs::create_dir_all(tree_root.join(relative_root))
|
||||
.map_err(|e| format!("create rsync tree root failed: {}: {e}", tree_root.join(relative_root).display()))?;
|
||||
fs::create_dir_all(tree_root.join(relative_root)).map_err(|e| {
|
||||
format!(
|
||||
"create rsync tree root failed: {}: {e}",
|
||||
tree_root.join(relative_root).display()
|
||||
)
|
||||
})?;
|
||||
for (uri, bytes) in objects {
|
||||
let rel = uri
|
||||
.strip_prefix(module_uri)
|
||||
.ok_or_else(|| format!("object uri {uri} does not belong to module {module_uri}"))?;
|
||||
let path = tree_root.join(relative_root).join(rel);
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.map_err(|e| format!("create rsync object parent failed: {}: {e}", parent.display()))?;
|
||||
fs::create_dir_all(parent).map_err(|e| {
|
||||
format!(
|
||||
"create rsync object parent failed: {}: {e}",
|
||||
parent.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
fs::write(&path, bytes)
|
||||
.map_err(|e| format!("write rsync object failed: {}: {e}", path.display()))?;
|
||||
@@ -197,8 +205,8 @@ fn materialize_rsync_module_from_store(
|
||||
fn copy_dir_all(src: &Path, dst: &Path) -> Result<(), String> {
|
||||
fs::create_dir_all(dst)
|
||||
.map_err(|e| format!("create directory failed: {}: {e}", dst.display()))?;
|
||||
for entry in fs::read_dir(src)
|
||||
.map_err(|e| format!("read directory failed: {}: {e}", src.display()))?
|
||||
for entry in
|
||||
fs::read_dir(src).map_err(|e| format!("read directory failed: {}: {e}", src.display()))?
|
||||
{
|
||||
let entry = entry.map_err(|e| format!("read entry failed: {}: {e}", src.display()))?;
|
||||
let file_type = entry
|
||||
@@ -252,8 +260,12 @@ fn keep_rsync_module(pp: &rpki::audit::PublicationPointAudit) -> Result<Option<S
|
||||
if is_failed_fetch_source(&pp.source) {
|
||||
return Ok(None);
|
||||
}
|
||||
let module_uri = canonical_rsync_module(&pp.rsync_base_uri)
|
||||
.map_err(|e| format!("canonicalize rsync module failed for {}: {e}", pp.rsync_base_uri))?;
|
||||
let module_uri = canonical_rsync_module(&pp.rsync_base_uri).map_err(|e| {
|
||||
format!(
|
||||
"canonicalize rsync module failed for {}: {e}",
|
||||
pp.rsync_base_uri
|
||||
)
|
||||
})?;
|
||||
if pp.rrdp_notification_uri.is_none() || pp.repo_sync_source.as_deref() == Some("rsync") {
|
||||
return Ok(Some(module_uri));
|
||||
}
|
||||
@@ -317,8 +329,9 @@ fn repair_base_inputs(
|
||||
for entry in fs::read_dir(&repos_dir)
|
||||
.map_err(|e| format!("scan rrdp repo dir failed: {}: {e}", repos_dir.display()))?
|
||||
{
|
||||
let entry = entry
|
||||
.map_err(|e| format!("read rrdp repo entry failed: {}: {e}", repos_dir.display()))?;
|
||||
let entry = entry.map_err(|e| {
|
||||
format!("read rrdp repo entry failed: {}: {e}", repos_dir.display())
|
||||
})?;
|
||||
let meta = entry.path().join("meta.json");
|
||||
if !meta.exists() {
|
||||
continue;
|
||||
@@ -385,13 +398,13 @@ fn repair_base_inputs(
|
||||
}
|
||||
write_json_value(locks_path, &locks)?;
|
||||
verification.base["capture"]["rrdpRepoCount"] = serde_json::Value::from(
|
||||
locks.get("rrdp")
|
||||
locks
|
||||
.get("rrdp")
|
||||
.and_then(|v| v.as_object())
|
||||
.map(|m| m.len())
|
||||
.unwrap_or(0),
|
||||
);
|
||||
verification.base["capture"]["rsyncModuleCount"] =
|
||||
serde_json::Value::from(final_modules.len());
|
||||
verification.base["capture"]["rsyncModuleCount"] = serde_json::Value::from(final_modules.len());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -502,8 +515,9 @@ fn repair_delta_step_inputs(
|
||||
for entry in fs::read_dir(&repos_dir)
|
||||
.map_err(|e| format!("scan rrdp repo dir failed: {}: {e}", repos_dir.display()))?
|
||||
{
|
||||
let entry = entry
|
||||
.map_err(|e| format!("read rrdp repo entry failed: {}: {e}", repos_dir.display()))?;
|
||||
let entry = entry.map_err(|e| {
|
||||
format!("read rrdp repo entry failed: {}: {e}", repos_dir.display())
|
||||
})?;
|
||||
let meta = entry.path().join("meta.json");
|
||||
if !meta.exists() {
|
||||
continue;
|
||||
@@ -634,7 +648,8 @@ fn repair_delta_step_inputs(
|
||||
.and_then(|v| v.get("session"))
|
||||
.and_then(|v| v.as_str())
|
||||
.or_else(|| {
|
||||
entry.get("base")
|
||||
entry
|
||||
.get("base")
|
||||
.and_then(|v| v.get("session"))
|
||||
.and_then(|v| v.as_str())
|
||||
});
|
||||
@@ -645,7 +660,10 @@ fn repair_delta_step_inputs(
|
||||
if session_dir.exists() {
|
||||
continue;
|
||||
}
|
||||
let base_bucket_dir = base_capture_root.join("rrdp").join("repos").join(&bucket_hash);
|
||||
let base_bucket_dir = base_capture_root
|
||||
.join("rrdp")
|
||||
.join("repos")
|
||||
.join(&bucket_hash);
|
||||
let base_session_dir = base_bucket_dir.join(session);
|
||||
if !base_session_dir.exists() {
|
||||
continue;
|
||||
@@ -673,13 +691,13 @@ fn repair_delta_step_inputs(
|
||||
write_json_value(&locks_path, &locks)?;
|
||||
|
||||
step_verification["capture"]["rrdpRepoCount"] = serde_json::Value::from(
|
||||
locks.get("rrdp")
|
||||
locks
|
||||
.get("rrdp")
|
||||
.and_then(|v| v.as_object())
|
||||
.map(|m| m.len())
|
||||
.unwrap_or(0),
|
||||
);
|
||||
step_verification["capture"]["rsyncModuleCount"] =
|
||||
serde_json::Value::from(final_modules.len());
|
||||
step_verification["capture"]["rsyncModuleCount"] = serde_json::Value::from(final_modules.len());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -694,7 +712,12 @@ fn rewrite_delta_base_hash(step_dir: &Path, previous_locks_path: &Path) -> Resul
|
||||
let locks_path = step_dir.join("locks-delta.json");
|
||||
let mut locks = load_json(&locks_path)?;
|
||||
let previous_locks = serde_json::from_slice::<serde_json::Value>(&previous_locks_bytes)
|
||||
.map_err(|e| format!("parse previous locks failed: {}: {e}", previous_locks_path.display()))?;
|
||||
.map_err(|e| {
|
||||
format!(
|
||||
"parse previous locks failed: {}: {e}",
|
||||
previous_locks_path.display()
|
||||
)
|
||||
})?;
|
||||
locks["baseLocksSha256"] = serde_json::Value::String(previous_locks_sha256.clone());
|
||||
let capture = locks
|
||||
.get("capture")
|
||||
@@ -760,17 +783,33 @@ fn real_main() -> Result<(), String> {
|
||||
let bundle_json_path = rir_dir.join("bundle.json");
|
||||
let verification_path = rir_dir.join("verification.json");
|
||||
|
||||
let mut bundle: RirBundleMetadataV2Serde = serde_json::from_slice(
|
||||
&fs::read(&bundle_json_path)
|
||||
.map_err(|e| format!("read bundle.json failed: {}: {e}", bundle_json_path.display()))?,
|
||||
)
|
||||
.map_err(|e| format!("parse bundle.json failed: {}: {e}", bundle_json_path.display()))?;
|
||||
let mut bundle: RirBundleMetadataV2Serde =
|
||||
serde_json::from_slice(&fs::read(&bundle_json_path).map_err(|e| {
|
||||
format!(
|
||||
"read bundle.json failed: {}: {e}",
|
||||
bundle_json_path.display()
|
||||
)
|
||||
})?)
|
||||
.map_err(|e| {
|
||||
format!(
|
||||
"parse bundle.json failed: {}: {e}",
|
||||
bundle_json_path.display()
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut verification: VerificationV2 = serde_json::from_slice(
|
||||
&fs::read(&verification_path)
|
||||
.map_err(|e| format!("read verification.json failed: {}: {e}", verification_path.display()))?,
|
||||
)
|
||||
.map_err(|e| format!("parse verification.json failed: {}: {e}", verification_path.display()))?;
|
||||
let mut verification: VerificationV2 =
|
||||
serde_json::from_slice(&fs::read(&verification_path).map_err(|e| {
|
||||
format!(
|
||||
"read verification.json failed: {}: {e}",
|
||||
verification_path.display()
|
||||
)
|
||||
})?)
|
||||
.map_err(|e| {
|
||||
format!(
|
||||
"parse verification.json failed: {}: {e}",
|
||||
verification_path.display()
|
||||
)
|
||||
})?;
|
||||
|
||||
let tal_bytes = fs::read(rir_dir.join("tal.tal"))
|
||||
.map_err(|e| format!("read tal.tal failed: {}: {e}", rir_dir.display()))?;
|
||||
@@ -787,7 +826,8 @@ fn real_main() -> Result<(), String> {
|
||||
fs::create_dir_all(parent)
|
||||
.map_err(|e| format!("create refresh db parent failed: {}: {e}", parent.display()))?;
|
||||
}
|
||||
let store = RocksStore::open(&work_db).map_err(|e| format!("open refresh rocksdb failed: {e}"))?;
|
||||
let store =
|
||||
RocksStore::open(&work_db).map_err(|e| format!("open refresh rocksdb failed: {e}"))?;
|
||||
|
||||
let base_archive = path_join(&rir_dir, &bundle.base.relative_archive_path);
|
||||
let base_locks = path_join(&rir_dir, &bundle.base.relative_locks_path);
|
||||
@@ -823,8 +863,8 @@ fn real_main() -> Result<(), String> {
|
||||
.map_err(|e| format!("build base ccr failed: {e}"))?;
|
||||
write_ccr_file(&base_ccr, &base_ccr_content)
|
||||
.map_err(|e| format!("write base ccr failed: {}: {e}", base_ccr.display()))?;
|
||||
let base_ccr_bytes =
|
||||
fs::read(&base_ccr).map_err(|e| format!("read base ccr failed: {}: {e}", base_ccr.display()))?;
|
||||
let base_ccr_bytes = fs::read(&base_ccr)
|
||||
.map_err(|e| format!("read base ccr failed: {}: {e}", base_ccr.display()))?;
|
||||
let base_decoded =
|
||||
decode_content_info(&base_ccr_bytes).map_err(|e| format!("decode base ccr failed: {e}"))?;
|
||||
let base_verify =
|
||||
@@ -838,15 +878,18 @@ fn real_main() -> Result<(), String> {
|
||||
bundle.base.vap_count = base_vap_rows.len();
|
||||
|
||||
verification.base["ccr"]["sha256"] = serde_json::Value::String(bundle.base.ccr_sha256.clone());
|
||||
verification.base["ccr"]["stateHashesOk"] = serde_json::Value::Bool(base_verify.state_hashes_ok);
|
||||
verification.base["ccr"]["stateHashesOk"] =
|
||||
serde_json::Value::Bool(base_verify.state_hashes_ok);
|
||||
verification.base["ccr"]["manifestInstances"] =
|
||||
serde_json::Value::from(base_verify.manifest_instances);
|
||||
verification.base["ccr"]["roaVrpCount"] = serde_json::Value::from(base_vrp_rows.len());
|
||||
verification.base["ccr"]["aspaPayloadSets"] = serde_json::Value::from(base_vap_rows.len());
|
||||
verification.base["ccr"]["routerKeyCount"] =
|
||||
serde_json::Value::from(base_verify.router_key_count);
|
||||
verification.base["compareViews"]["baseVrpCount"] = serde_json::Value::from(base_vrp_rows.len());
|
||||
verification.base["compareViews"]["baseVapCount"] = serde_json::Value::from(base_vap_rows.len());
|
||||
verification.base["compareViews"]["baseVrpCount"] =
|
||||
serde_json::Value::from(base_vrp_rows.len());
|
||||
verification.base["compareViews"]["baseVapCount"] =
|
||||
serde_json::Value::from(base_vap_rows.len());
|
||||
verification.base["capture"]["selfReplayOk"] = serde_json::Value::Bool(true);
|
||||
repair_base_inputs(
|
||||
&base_archive,
|
||||
@@ -922,7 +965,8 @@ fn real_main() -> Result<(), String> {
|
||||
step_verification["ccr"]["manifestInstances"] =
|
||||
serde_json::Value::from(delta_verify.manifest_instances);
|
||||
step_verification["ccr"]["roaVrpCount"] = serde_json::Value::from(delta_vrp_rows.len());
|
||||
step_verification["ccr"]["aspaPayloadSets"] = serde_json::Value::from(delta_vap_rows.len());
|
||||
step_verification["ccr"]["aspaPayloadSets"] =
|
||||
serde_json::Value::from(delta_vap_rows.len());
|
||||
step_verification["ccr"]["routerKeyCount"] =
|
||||
serde_json::Value::from(delta_verify.router_key_count);
|
||||
step_verification["compareViews"]["vrpCount"] =
|
||||
@@ -931,28 +975,29 @@ fn real_main() -> Result<(), String> {
|
||||
serde_json::Value::from(delta_vap_rows.len());
|
||||
step_verification["selfReplayOk"] = serde_json::Value::Bool(true);
|
||||
}
|
||||
let step_verification_path = path_join(&rir_dir, &step.relative_path).join("verification.json");
|
||||
let mut step_verification_json: serde_json::Value = serde_json::from_slice(
|
||||
&fs::read(&step_verification_path).map_err(|e| {
|
||||
let step_verification_path =
|
||||
path_join(&rir_dir, &step.relative_path).join("verification.json");
|
||||
let mut step_verification_json: serde_json::Value =
|
||||
serde_json::from_slice(&fs::read(&step_verification_path).map_err(|e| {
|
||||
format!(
|
||||
"read step verification failed: {}: {e}",
|
||||
step_verification_path.display()
|
||||
)
|
||||
})?,
|
||||
)
|
||||
.map_err(|e| {
|
||||
format!(
|
||||
"parse step verification failed: {}: {e}",
|
||||
step_verification_path.display()
|
||||
)
|
||||
})?;
|
||||
})?)
|
||||
.map_err(|e| {
|
||||
format!(
|
||||
"parse step verification failed: {}: {e}",
|
||||
step_verification_path.display()
|
||||
)
|
||||
})?;
|
||||
step_verification_json["ccr"]["sha256"] =
|
||||
serde_json::Value::String(step.delta_ccr_sha256.clone());
|
||||
step_verification_json["ccr"]["stateHashesOk"] =
|
||||
serde_json::Value::Bool(delta_verify.state_hashes_ok);
|
||||
step_verification_json["ccr"]["manifestInstances"] =
|
||||
serde_json::Value::from(delta_verify.manifest_instances);
|
||||
step_verification_json["ccr"]["roaVrpCount"] = serde_json::Value::from(delta_vrp_rows.len());
|
||||
step_verification_json["ccr"]["roaVrpCount"] =
|
||||
serde_json::Value::from(delta_vrp_rows.len());
|
||||
step_verification_json["ccr"]["aspaPayloadSets"] =
|
||||
serde_json::Value::from(delta_vap_rows.len());
|
||||
step_verification_json["ccr"]["routerKeyCount"] =
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use rocksdb::{DB, IteratorMode, Options};
|
||||
use rpki::storage::{column_family_descriptors, CF_REPOSITORY_VIEW};
|
||||
use rpki::storage::{CF_REPOSITORY_VIEW, column_family_descriptors};
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
|
||||
@@ -64,12 +64,9 @@ fn main() {
|
||||
println!("rule_hash={}", output.rule_hash);
|
||||
println!("validation_path_hint={:?}", output.validation_path_hint);
|
||||
|
||||
if let Some(trace) = trace_rule_to_root(
|
||||
&store,
|
||||
AuditRuleKind::Roa,
|
||||
&output.rule_hash,
|
||||
)
|
||||
.expect("trace rule")
|
||||
if let Some(trace) =
|
||||
trace_rule_to_root(&store, AuditRuleKind::Roa, &output.rule_hash)
|
||||
.expect("trace rule")
|
||||
{
|
||||
println!(
|
||||
"trace_leaf_manifest={}",
|
||||
@@ -79,11 +76,17 @@ fn main() {
|
||||
.map(|node| node.manifest_rsync_uri.as_str())
|
||||
.unwrap_or("")
|
||||
);
|
||||
println!("trace_source_object_uri={}", trace.resolved_output.source_object_uri);
|
||||
println!(
|
||||
"trace_source_object_uri={}",
|
||||
trace.resolved_output.source_object_uri
|
||||
);
|
||||
println!("trace_chain_len={}", trace.chain_leaf_to_root.len());
|
||||
for (idx, node) in trace.chain_leaf_to_root.iter().enumerate() {
|
||||
println!("chain[{idx}].manifest={}", node.manifest_rsync_uri);
|
||||
println!("chain[{idx}].current_manifest={}", node.current_manifest_rsync_uri);
|
||||
println!(
|
||||
"chain[{idx}].current_manifest={}",
|
||||
node.current_manifest_rsync_uri
|
||||
);
|
||||
println!("chain[{idx}].current_crl={}", node.current_crl_rsync_uri);
|
||||
}
|
||||
}
|
||||
|
||||
323
src/blob_store.rs
Normal file
323
src/blob_store.rs
Normal file
@@ -0,0 +1,323 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use rocksdb::{DB, Options, WriteBatch};
|
||||
|
||||
use crate::storage::{RawByHashEntry, RocksStore, StorageError, StorageResult};
|
||||
|
||||
const RAW_BY_HASH_KEY_PREFIX: &str = "rawbyhash:";
|
||||
|
||||
fn raw_by_hash_key(sha256_hex: &str) -> String {
|
||||
format!("{RAW_BY_HASH_KEY_PREFIX}{sha256_hex}")
|
||||
}
|
||||
|
||||
pub trait RawObjectStore {
|
||||
fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>>;
|
||||
|
||||
fn get_raw_entries_batch(
|
||||
&self,
|
||||
sha256_hexes: &[String],
|
||||
) -> StorageResult<Vec<Option<RawByHashEntry>>>;
|
||||
|
||||
fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult<Option<Vec<u8>>> {
|
||||
self.get_raw_entry(sha256_hex)
|
||||
.map(|entry| entry.map(|entry| entry.bytes))
|
||||
}
|
||||
|
||||
fn get_blob_bytes_batch(
|
||||
&self,
|
||||
sha256_hexes: &[String],
|
||||
) -> StorageResult<Vec<Option<Vec<u8>>>> {
|
||||
self.get_raw_entries_batch(sha256_hexes).map(|entries| {
|
||||
entries
|
||||
.into_iter()
|
||||
.map(|entry| entry.map(|entry| entry.bytes))
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ExternalRawStoreDb {
|
||||
path: PathBuf,
|
||||
db: Arc<DB>,
|
||||
}
|
||||
|
||||
impl ExternalRawStoreDb {
|
||||
pub fn open(path: impl Into<PathBuf>) -> StorageResult<Self> {
|
||||
let path = path.into();
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
}
|
||||
let mut opts = Options::default();
|
||||
opts.create_if_missing(true);
|
||||
opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
|
||||
let db = DB::open(&opts, &path).map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(Self {
|
||||
path,
|
||||
db: Arc::new(db),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn put_raw_entry(&self, entry: &RawByHashEntry) -> StorageResult<()> {
|
||||
entry.validate_internal()?;
|
||||
let key = raw_by_hash_key(&entry.sha256_hex);
|
||||
let value =
|
||||
serde_cbor::to_vec(entry).map_err(|e| StorageError::Codec { entity: "raw_by_hash", detail: e.to_string() })?;
|
||||
self.db
|
||||
.put(key.as_bytes(), value)
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn put_raw_entries_batch(&self, entries: &[RawByHashEntry]) -> StorageResult<()> {
|
||||
if entries.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut batch = WriteBatch::default();
|
||||
for entry in entries {
|
||||
entry.validate_internal()?;
|
||||
let key = raw_by_hash_key(&entry.sha256_hex);
|
||||
let value = serde_cbor::to_vec(entry).map_err(|e| StorageError::Codec {
|
||||
entity: "raw_by_hash",
|
||||
detail: e.to_string(),
|
||||
})?;
|
||||
batch.put(key.as_bytes(), value);
|
||||
}
|
||||
self.db
|
||||
.write(batch)
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn delete_raw_entry(&self, sha256_hex: &str) -> StorageResult<()> {
|
||||
let key = raw_by_hash_key(sha256_hex);
|
||||
self.db
|
||||
.delete(key.as_bytes())
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &PathBuf {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
|
||||
impl RawObjectStore for RocksStore {
|
||||
fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>> {
|
||||
self.get_raw_by_hash_entry(sha256_hex)
|
||||
}
|
||||
|
||||
fn get_raw_entries_batch(
|
||||
&self,
|
||||
sha256_hexes: &[String],
|
||||
) -> StorageResult<Vec<Option<RawByHashEntry>>> {
|
||||
self.get_raw_by_hash_entries_batch(sha256_hexes)
|
||||
}
|
||||
}
|
||||
|
||||
impl RawObjectStore for ExternalRawStoreDb {
|
||||
fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>> {
|
||||
let key = raw_by_hash_key(sha256_hex);
|
||||
let Some(bytes) = self
|
||||
.db
|
||||
.get(key.as_bytes())
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let entry = serde_cbor::from_slice::<RawByHashEntry>(&bytes).map_err(|e| StorageError::Codec {
|
||||
entity: "raw_by_hash",
|
||||
detail: e.to_string(),
|
||||
})?;
|
||||
entry.validate_internal()?;
|
||||
Ok(Some(entry))
|
||||
}
|
||||
|
||||
fn get_raw_entries_batch(
|
||||
&self,
|
||||
sha256_hexes: &[String],
|
||||
) -> StorageResult<Vec<Option<RawByHashEntry>>> {
|
||||
if sha256_hexes.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let keys: Vec<String> = sha256_hexes.iter().map(|hash| raw_by_hash_key(hash)).collect();
|
||||
self.db
|
||||
.multi_get(keys.iter().map(|key| key.as_bytes()))
|
||||
.into_iter()
|
||||
.map(|res| {
|
||||
let maybe = res.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
match maybe {
|
||||
Some(bytes) => {
|
||||
let entry = serde_cbor::from_slice::<RawByHashEntry>(&bytes).map_err(|e| {
|
||||
StorageError::Codec {
|
||||
entity: "raw_by_hash",
|
||||
detail: e.to_string(),
|
||||
}
|
||||
})?;
|
||||
entry.validate_internal()?;
|
||||
Ok(Some(entry))
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{ExternalRawStoreDb, RawObjectStore};
|
||||
use crate::storage::{RawByHashEntry, RocksStore, StorageError};
|
||||
|
||||
fn sha256_hex(bytes: &[u8]) -> String {
|
||||
use sha2::{Digest, Sha256};
|
||||
hex::encode(Sha256::digest(bytes))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rocks_store_raw_object_store_reads_single_and_batch_entries() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
|
||||
let a = b"object-a".to_vec();
|
||||
let b = b"object-b".to_vec();
|
||||
let a_hash = sha256_hex(&a);
|
||||
let b_hash = sha256_hex(&b);
|
||||
|
||||
store
|
||||
.put_raw_by_hash_entry(&RawByHashEntry::from_bytes(a_hash.clone(), a.clone()))
|
||||
.expect("put a");
|
||||
store
|
||||
.put_raw_by_hash_entry(&RawByHashEntry::from_bytes(b_hash.clone(), b.clone()))
|
||||
.expect("put b");
|
||||
|
||||
let single = store
|
||||
.get_raw_entry(&a_hash)
|
||||
.expect("get single")
|
||||
.expect("present");
|
||||
assert_eq!(single.bytes, a);
|
||||
|
||||
let batch = store
|
||||
.get_raw_entries_batch(&[a_hash.clone(), "00".repeat(32), b_hash.clone()])
|
||||
.expect("get batch");
|
||||
assert_eq!(batch.len(), 3);
|
||||
assert_eq!(batch[0].as_ref().map(|entry| entry.bytes.as_slice()), Some(a.as_slice()));
|
||||
assert!(batch[1].is_none());
|
||||
assert_eq!(batch[2].as_ref().map(|entry| entry.bytes.as_slice()), Some(b.as_slice()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_raw_store_db_roundtrips_entries() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||
|
||||
let mut entry = RawByHashEntry::from_bytes(sha256_hex(b"blob"), b"blob".to_vec());
|
||||
entry.origin_uris.push("rsync://example.test/repo/a.cer".to_string());
|
||||
entry.object_type = Some("cer".to_string());
|
||||
raw_store.put_raw_entry(&entry).expect("put raw entry");
|
||||
|
||||
let got = raw_store
|
||||
.get_raw_entry(&entry.sha256_hex)
|
||||
.expect("read raw entry")
|
||||
.expect("entry exists");
|
||||
assert_eq!(got, entry);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_raw_store_db_batch_writes_and_reads() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||
|
||||
let a = RawByHashEntry::from_bytes(sha256_hex(b"a"), b"a".to_vec());
|
||||
let b = RawByHashEntry::from_bytes(sha256_hex(b"b"), b"b".to_vec());
|
||||
raw_store
|
||||
.put_raw_entries_batch(&[a.clone(), b.clone()])
|
||||
.expect("batch put");
|
||||
|
||||
let batch = raw_store
|
||||
.get_raw_entries_batch(&[a.sha256_hex.clone(), b.sha256_hex.clone()])
|
||||
.expect("batch get");
|
||||
assert_eq!(batch.len(), 2);
|
||||
assert_eq!(batch[0], Some(a));
|
||||
assert_eq!(batch[1], Some(b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_object_store_default_blob_helpers_return_bytes_only() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store = ExternalRawStoreDb::open(td.path().join("nested/raw-store.db"))
|
||||
.expect("open raw store");
|
||||
|
||||
let mut entry = RawByHashEntry::from_bytes(sha256_hex(b"blob"), b"blob".to_vec());
|
||||
entry.origin_uris.push("rsync://example.test/repo/blob.roa".to_string());
|
||||
raw_store.put_raw_entry(&entry).expect("put raw entry");
|
||||
|
||||
let single = raw_store
|
||||
.get_blob_bytes(&entry.sha256_hex)
|
||||
.expect("get blob bytes")
|
||||
.expect("entry exists");
|
||||
assert_eq!(single, b"blob".to_vec());
|
||||
|
||||
let batch = raw_store
|
||||
.get_blob_bytes_batch(&[entry.sha256_hex.clone(), "00".repeat(32)])
|
||||
.expect("get blob bytes batch");
|
||||
assert_eq!(batch, vec![Some(b"blob".to_vec()), None]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_raw_store_db_delete_removes_entry() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||
|
||||
let entry = RawByHashEntry::from_bytes(sha256_hex(b"gone"), b"gone".to_vec());
|
||||
raw_store.put_raw_entry(&entry).expect("put");
|
||||
assert!(raw_store.get_raw_entry(&entry.sha256_hex).unwrap().is_some());
|
||||
|
||||
raw_store
|
||||
.delete_raw_entry(&entry.sha256_hex)
|
||||
.expect("delete entry");
|
||||
assert!(raw_store.get_raw_entry(&entry.sha256_hex).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_raw_store_db_rejects_invalid_entry_on_put() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||
|
||||
let bad = RawByHashEntry {
|
||||
sha256_hex: "11".repeat(32),
|
||||
bytes: b"blob".to_vec(),
|
||||
origin_uris: Vec::new(),
|
||||
object_type: None,
|
||||
encoding: None,
|
||||
};
|
||||
let err = raw_store.put_raw_entry(&bad).expect_err("invalid hash should fail");
|
||||
assert!(matches!(err, StorageError::InvalidData { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_raw_store_db_reports_codec_error_for_corrupt_value() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||
raw_store
|
||||
.db
|
||||
.put(b"rawbyhash:deadbeef", b"not-cbor")
|
||||
.expect("inject corrupt bytes");
|
||||
|
||||
let err = raw_store
|
||||
.get_raw_entry("deadbeef")
|
||||
.expect_err("corrupt value should fail");
|
||||
assert!(matches!(err, StorageError::Codec { entity: "raw_by_hash", .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_raw_store_db_batch_returns_empty_for_empty_request() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||
let entries = raw_store
|
||||
.get_raw_entries_batch(&[])
|
||||
.expect("empty batch succeeds");
|
||||
assert!(entries.is_empty());
|
||||
raw_store.put_raw_entries_batch(&[]).expect("empty put succeeds");
|
||||
}
|
||||
}
|
||||
@@ -65,13 +65,10 @@ pub fn build_vap_compare_rows(
|
||||
aspas: &[AspaAttestation],
|
||||
trust_anchor: &str,
|
||||
) -> BTreeSet<VapCompareRow> {
|
||||
aspas.iter()
|
||||
aspas
|
||||
.iter()
|
||||
.map(|aspa| {
|
||||
let mut providers = aspa
|
||||
.provider_as_ids
|
||||
.iter()
|
||||
.copied()
|
||||
.collect::<Vec<_>>();
|
||||
let mut providers = aspa.provider_as_ids.iter().copied().collect::<Vec<_>>();
|
||||
providers.sort_unstable();
|
||||
providers.dedup();
|
||||
VapCompareRow {
|
||||
@@ -173,7 +170,10 @@ pub fn write_vap_csv(path: &Path, rows: &BTreeSet<VapCompareRow>) -> Result<(),
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::ccr::{CcrContentInfo, CcrDigestAlgorithm, RpkiCanonicalCacheRepresentation, build_aspa_payload_state, build_roa_payload_state};
|
||||
use crate::ccr::{
|
||||
CcrContentInfo, CcrDigestAlgorithm, RpkiCanonicalCacheRepresentation,
|
||||
build_aspa_payload_state, build_roa_payload_state,
|
||||
};
|
||||
use crate::data_model::roa::{IpPrefix, RoaAfi};
|
||||
|
||||
#[test]
|
||||
@@ -218,7 +218,8 @@ mod tests {
|
||||
tas: None,
|
||||
rks: None,
|
||||
});
|
||||
let (vrp_rows, vap_rows) = decode_ccr_compare_views(&content, "apnic").expect("decode compare views");
|
||||
let (vrp_rows, vap_rows) =
|
||||
decode_ccr_compare_views(&content, "apnic").expect("decode compare views");
|
||||
assert_eq!(vrp_rows.len(), 1);
|
||||
assert_eq!(vap_rows.len(), 1);
|
||||
assert_eq!(vap_rows.iter().next().unwrap().providers, "AS64497");
|
||||
|
||||
@@ -8,10 +8,12 @@ use time::format_description::well_known::Rfc3339;
|
||||
|
||||
use crate::audit::PublicationPointAudit;
|
||||
use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher};
|
||||
use crate::replay::archive::{ReplayArchiveIndex, ReplayRrdpLock, ReplayTransport, canonical_rsync_module, sha256_hex};
|
||||
use crate::sync::rrdp::{NotificationDeltaRef, parse_notification};
|
||||
use crate::replay::archive::{
|
||||
ReplayArchiveIndex, ReplayRrdpLock, ReplayTransport, canonical_rsync_module, sha256_hex,
|
||||
};
|
||||
use crate::storage::{RocksStore, RrdpSourceRecord};
|
||||
use crate::sync::rrdp::Fetcher;
|
||||
use crate::sync::rrdp::{NotificationDeltaRef, parse_notification};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RecordedHttpResponse {
|
||||
@@ -124,10 +126,12 @@ impl<F: RsyncFetcher> RsyncFetcher for RecordingRsyncFetcher<F> {
|
||||
|
||||
if self.capture_objects {
|
||||
let mut recorded = Vec::new();
|
||||
let result = self.inner.visit_objects(rsync_base_uri, &mut |uri, bytes| {
|
||||
recorded.push((uri.clone(), bytes.clone()));
|
||||
visitor(uri, bytes)
|
||||
})?;
|
||||
let result = self
|
||||
.inner
|
||||
.visit_objects(rsync_base_uri, &mut |uri, bytes| {
|
||||
recorded.push((uri.clone(), bytes.clone()));
|
||||
visitor(uri, bytes)
|
||||
})?;
|
||||
self.fetches.lock().expect("rsync recorder lock").insert(
|
||||
rsync_base_uri.to_string(),
|
||||
RecordedRsyncFetch {
|
||||
@@ -347,7 +351,13 @@ fn rrdp_repo_is_replayable(record: &RrdpSourceRecord) -> bool {
|
||||
fn collect_current_state_locks(
|
||||
publication_points: &[PublicationPointAudit],
|
||||
store: &RocksStore,
|
||||
) -> Result<(BTreeMap<String, RrdpLockJson>, BTreeMap<String, RsyncLockJson>), String> {
|
||||
) -> Result<
|
||||
(
|
||||
BTreeMap<String, RrdpLockJson>,
|
||||
BTreeMap<String, RsyncLockJson>,
|
||||
),
|
||||
String,
|
||||
> {
|
||||
let mut rrdp_locks = BTreeMap::new();
|
||||
let mut rsync_locks = BTreeMap::new();
|
||||
let mut seen_modules = BTreeSet::new();
|
||||
@@ -357,8 +367,12 @@ fn collect_current_state_locks(
|
||||
if pp.source == "failed_fetch_no_cache" {
|
||||
continue;
|
||||
}
|
||||
let module_uri = canonical_rsync_module(&pp.rsync_base_uri)
|
||||
.map_err(|e| format!("canonicalize rsync module failed for {}: {e}", pp.rsync_base_uri))?;
|
||||
let module_uri = canonical_rsync_module(&pp.rsync_base_uri).map_err(|e| {
|
||||
format!(
|
||||
"canonicalize rsync module failed for {}: {e}",
|
||||
pp.rsync_base_uri
|
||||
)
|
||||
})?;
|
||||
if let Some(notify_uri) = pp.rrdp_notification_uri.as_deref() {
|
||||
if !seen_rrdp.insert(notify_uri.to_string()) {
|
||||
continue;
|
||||
@@ -476,7 +490,10 @@ fn materialize_rsync_module(
|
||||
last_seen_at: &str,
|
||||
) -> Result<(), String> {
|
||||
let bucket_hash = sha256_hex(module_uri.as_bytes());
|
||||
let bucket_dir = capture_root.join("rsync").join("modules").join(&bucket_hash);
|
||||
let bucket_dir = capture_root
|
||||
.join("rsync")
|
||||
.join("modules")
|
||||
.join(&bucket_hash);
|
||||
write_json(
|
||||
&bucket_dir.join("meta.json"),
|
||||
&ModuleMetaJson {
|
||||
@@ -490,8 +507,12 @@ fn materialize_rsync_module(
|
||||
.strip_prefix("rsync://")
|
||||
.ok_or_else(|| format!("invalid rsync module uri: {module_uri}"))?;
|
||||
let relative_root = without_scheme.trim_end_matches('/');
|
||||
fs::create_dir_all(bucket_dir.join("tree").join(relative_root))
|
||||
.map_err(|e| format!("create rsync tree root failed: {}: {e}", bucket_dir.join("tree").join(relative_root).display()))?;
|
||||
fs::create_dir_all(bucket_dir.join("tree").join(relative_root)).map_err(|e| {
|
||||
format!(
|
||||
"create rsync tree root failed: {}: {e}",
|
||||
bucket_dir.join("tree").join(relative_root).display()
|
||||
)
|
||||
})?;
|
||||
for (uri, bytes) in objects {
|
||||
let rel = uri
|
||||
.strip_prefix(module_uri)
|
||||
@@ -557,8 +578,12 @@ pub fn write_live_base_replay_bundle_inputs(
|
||||
if pp.source == "failed_fetch_no_cache" {
|
||||
continue;
|
||||
}
|
||||
let module_uri = canonical_rsync_module(&pp.rsync_base_uri)
|
||||
.map_err(|e| format!("canonicalize rsync module failed for {}: {e}", pp.rsync_base_uri))?;
|
||||
let module_uri = canonical_rsync_module(&pp.rsync_base_uri).map_err(|e| {
|
||||
format!(
|
||||
"canonicalize rsync module failed for {}: {e}",
|
||||
pp.rsync_base_uri
|
||||
)
|
||||
})?;
|
||||
if let Some(notify_uri) = pp.rrdp_notification_uri.as_deref() {
|
||||
if !seen_rrdp.insert(notify_uri.to_string()) {
|
||||
continue;
|
||||
@@ -567,7 +592,9 @@ pub fn write_live_base_replay_bundle_inputs(
|
||||
if rrdp_repo_is_replayable(&source_record) {
|
||||
let notification_bytes = &http_records
|
||||
.get(notify_uri)
|
||||
.ok_or_else(|| format!("missing recorded notification body for {notify_uri}"))?
|
||||
.ok_or_else(|| {
|
||||
format!("missing recorded notification body for {notify_uri}")
|
||||
})?
|
||||
.bytes;
|
||||
let snapshot_uri = source_record
|
||||
.last_snapshot_uri
|
||||
@@ -575,9 +602,16 @@ pub fn write_live_base_replay_bundle_inputs(
|
||||
.ok_or_else(|| format!("missing last_snapshot_uri for {notify_uri}"))?;
|
||||
let snapshot_bytes = &http_records
|
||||
.get(snapshot_uri)
|
||||
.ok_or_else(|| format!("missing recorded snapshot body for {snapshot_uri}"))?
|
||||
.ok_or_else(|| {
|
||||
format!("missing recorded snapshot body for {snapshot_uri}")
|
||||
})?
|
||||
.bytes;
|
||||
materialize_rrdp_repo(&capture_root, &source_record, notification_bytes, snapshot_bytes)?;
|
||||
materialize_rrdp_repo(
|
||||
&capture_root,
|
||||
&source_record,
|
||||
notification_bytes,
|
||||
snapshot_bytes,
|
||||
)?;
|
||||
rrdp_locks.insert(
|
||||
notify_uri.to_string(),
|
||||
RrdpLockJson {
|
||||
@@ -621,10 +655,15 @@ pub fn write_live_base_replay_bundle_inputs(
|
||||
for fetch in rsync_records.values() {
|
||||
let module_uri = canonical_rsync_module(&fetch.requested_base_uri)
|
||||
.map_err(|e| format!("canonicalize requested rsync module failed: {e}"))?;
|
||||
let objects = rsync_objects_by_module.entry(module_uri.clone()).or_default();
|
||||
let times = rsync_times_by_module
|
||||
.entry(module_uri)
|
||||
.or_insert_with(|| (fetch.fetched_at_rfc3339_utc.clone(), fetch.fetched_at_rfc3339_utc.clone()));
|
||||
let objects = rsync_objects_by_module
|
||||
.entry(module_uri.clone())
|
||||
.or_default();
|
||||
let times = rsync_times_by_module.entry(module_uri).or_insert_with(|| {
|
||||
(
|
||||
fetch.fetched_at_rfc3339_utc.clone(),
|
||||
fetch.fetched_at_rfc3339_utc.clone(),
|
||||
)
|
||||
});
|
||||
if fetch.fetched_at_rfc3339_utc < times.0 {
|
||||
times.0 = fetch.fetched_at_rfc3339_utc.clone();
|
||||
}
|
||||
@@ -653,7 +692,13 @@ pub fn write_live_base_replay_bundle_inputs(
|
||||
.unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_string());
|
||||
(now.clone(), now)
|
||||
});
|
||||
materialize_rsync_module(&capture_root, module_uri, objects, &created_at, &last_seen_at)?;
|
||||
materialize_rsync_module(
|
||||
&capture_root,
|
||||
module_uri,
|
||||
objects,
|
||||
&created_at,
|
||||
&last_seen_at,
|
||||
)?;
|
||||
}
|
||||
|
||||
let locks = ReplayLocksJson {
|
||||
@@ -761,13 +806,25 @@ pub fn write_live_delta_replay_step_inputs(
|
||||
http_records: &BTreeMap<String, RecordedHttpResponse>,
|
||||
rsync_records: &BTreeMap<String, RecordedRsyncFetch>,
|
||||
) -> Result<LiveDeltaCaptureSummary, String> {
|
||||
let previous_locks: crate::replay::archive::ReplayLocks = serde_json::from_slice(
|
||||
&fs::read(previous_locks_path)
|
||||
.map_err(|e| format!("read previous locks failed: {}: {e}", previous_locks_path.display()))?,
|
||||
)
|
||||
.map_err(|e| format!("parse previous locks failed: {}: {e}", previous_locks_path.display()))?;
|
||||
let previous_locks_bytes = fs::read(previous_locks_path)
|
||||
.map_err(|e| format!("read previous locks bytes failed: {}: {e}", previous_locks_path.display()))?;
|
||||
let previous_locks: crate::replay::archive::ReplayLocks =
|
||||
serde_json::from_slice(&fs::read(previous_locks_path).map_err(|e| {
|
||||
format!(
|
||||
"read previous locks failed: {}: {e}",
|
||||
previous_locks_path.display()
|
||||
)
|
||||
})?)
|
||||
.map_err(|e| {
|
||||
format!(
|
||||
"parse previous locks failed: {}: {e}",
|
||||
previous_locks_path.display()
|
||||
)
|
||||
})?;
|
||||
let previous_locks_bytes = fs::read(previous_locks_path).map_err(|e| {
|
||||
format!(
|
||||
"read previous locks bytes failed: {}: {e}",
|
||||
previous_locks_path.display()
|
||||
)
|
||||
})?;
|
||||
let previous_locks_sha256 = sha256_hex(&previous_locks_bytes);
|
||||
|
||||
let recorded_at = time::OffsetDateTime::now_utc();
|
||||
@@ -810,10 +867,15 @@ pub fn write_live_delta_replay_step_inputs(
|
||||
for fetch in rsync_records.values() {
|
||||
let module_uri = canonical_rsync_module(&fetch.requested_base_uri)
|
||||
.map_err(|e| format!("canonicalize requested rsync module failed: {e}"))?;
|
||||
let objects = rsync_objects_by_module.entry(module_uri.clone()).or_default();
|
||||
let times = rsync_times_by_module
|
||||
.entry(module_uri)
|
||||
.or_insert_with(|| (fetch.fetched_at_rfc3339_utc.clone(), fetch.fetched_at_rfc3339_utc.clone()));
|
||||
let objects = rsync_objects_by_module
|
||||
.entry(module_uri.clone())
|
||||
.or_default();
|
||||
let times = rsync_times_by_module.entry(module_uri).or_insert_with(|| {
|
||||
(
|
||||
fetch.fetched_at_rfc3339_utc.clone(),
|
||||
fetch.fetched_at_rfc3339_utc.clone(),
|
||||
)
|
||||
});
|
||||
if fetch.fetched_at_rfc3339_utc < times.0 {
|
||||
times.0 = fetch.fetched_at_rfc3339_utc.clone();
|
||||
}
|
||||
@@ -834,22 +896,31 @@ pub fn write_live_delta_replay_step_inputs(
|
||||
if pp.source == "failed_fetch_no_cache" {
|
||||
continue;
|
||||
}
|
||||
let module_uri = canonical_rsync_module(&pp.rsync_base_uri)
|
||||
.map_err(|e| format!("canonicalize rsync module failed for {}: {e}", pp.rsync_base_uri))?;
|
||||
let module_uri = canonical_rsync_module(&pp.rsync_base_uri).map_err(|e| {
|
||||
format!(
|
||||
"canonicalize rsync module failed for {}: {e}",
|
||||
pp.rsync_base_uri
|
||||
)
|
||||
})?;
|
||||
if let Some(notify_uri) = pp.rrdp_notification_uri.as_deref() {
|
||||
if !seen_notifications.insert(notify_uri.to_string()) {
|
||||
continue;
|
||||
}
|
||||
let base_lock = previous_locks.rrdp.get(notify_uri);
|
||||
let target_record = store
|
||||
.get_rrdp_source_record(notify_uri)
|
||||
.map_err(|e| format!("read target rrdp source record failed for {notify_uri}: {e}"))?;
|
||||
let target_record = store.get_rrdp_source_record(notify_uri).map_err(|e| {
|
||||
format!("read target rrdp source record failed for {notify_uri}: {e}")
|
||||
})?;
|
||||
|
||||
let bucket_hash = sha256_hex(notify_uri.as_bytes());
|
||||
let bucket_dir = capture_root.join("rrdp").join("repos").join(&bucket_hash);
|
||||
let (created_at, last_seen_at) = target_record
|
||||
.as_ref()
|
||||
.map(|record| (record.first_seen_at.rfc3339_utc.clone(), record.last_seen_at.rfc3339_utc.clone()))
|
||||
.map(|record| {
|
||||
(
|
||||
record.first_seen_at.rfc3339_utc.clone(),
|
||||
record.last_seen_at.rfc3339_utc.clone(),
|
||||
)
|
||||
})
|
||||
.unwrap_or_else(|| {
|
||||
let now = recorded_at
|
||||
.format(&Rfc3339)
|
||||
@@ -869,7 +940,9 @@ pub fn write_live_delta_replay_step_inputs(
|
||||
let entry = if let (Some(base_lock), Some(target_record), Some(target_state)) = (
|
||||
base_lock,
|
||||
target_record.as_ref(),
|
||||
target_record.as_ref().and_then(target_rrdp_state_from_record),
|
||||
target_record
|
||||
.as_ref()
|
||||
.and_then(target_rrdp_state_from_record),
|
||||
) {
|
||||
if base_lock.transport == ReplayTransport::Rrdp
|
||||
&& base_lock.session.as_deref() == target_record.last_session_id.as_deref()
|
||||
@@ -900,10 +973,16 @@ pub fn write_live_delta_replay_step_inputs(
|
||||
let notification_bytes = http_records
|
||||
.get(notify_uri)
|
||||
.map(|record| record.bytes.as_slice())
|
||||
.ok_or_else(|| format!("missing recorded target notification body for {notify_uri}"))?;
|
||||
.ok_or_else(|| {
|
||||
format!("missing recorded target notification body for {notify_uri}")
|
||||
})?;
|
||||
let base_serial = base_lock.serial.expect("checked above");
|
||||
let target_serial = target_record.last_serial.expect("checked above");
|
||||
let deltas = notification_deltas_after_serial(notification_bytes, base_serial, target_serial)?;
|
||||
let deltas = notification_deltas_after_serial(
|
||||
notification_bytes,
|
||||
base_serial,
|
||||
target_serial,
|
||||
)?;
|
||||
let mut all_present = true;
|
||||
let session = target_record
|
||||
.last_session_id
|
||||
@@ -913,8 +992,10 @@ pub fn write_live_delta_replay_step_inputs(
|
||||
let notification_path =
|
||||
session_dir.join(format!("notification-target-{target_serial}.xml"));
|
||||
write_bytes(¬ification_path, notification_bytes)?;
|
||||
let target_notification = parse_notification(notification_bytes)
|
||||
.map_err(|e| format!("parse target notification failed for {notify_uri}: {e}"))?;
|
||||
let target_notification =
|
||||
parse_notification(notification_bytes).map_err(|e| {
|
||||
format!("parse target notification failed for {notify_uri}: {e}")
|
||||
})?;
|
||||
let snapshot_hash_hex = hex::encode(target_notification.snapshot_hash_sha256);
|
||||
if let Some(snapshot_bytes) = http_records
|
||||
.get(&target_notification.snapshot_uri)
|
||||
@@ -928,9 +1009,13 @@ pub fn write_live_delta_replay_step_inputs(
|
||||
let deltas_dir = session_dir.join("deltas");
|
||||
let mut delta_serials = Vec::new();
|
||||
for dref in &deltas {
|
||||
if let Some(delta_bytes) = http_records.get(&dref.uri).map(|record| record.bytes.as_slice()) {
|
||||
if let Some(delta_bytes) = http_records
|
||||
.get(&dref.uri)
|
||||
.map(|record| record.bytes.as_slice())
|
||||
{
|
||||
let hash = hex::encode(dref.hash_sha256);
|
||||
let path = deltas_dir.join(format!("delta-{}-{}.xml", dref.serial, hash));
|
||||
let path =
|
||||
deltas_dir.join(format!("delta-{}-{}.xml", dref.serial, hash));
|
||||
write_bytes(&path, delta_bytes)?;
|
||||
delta_serials.push(dref.serial);
|
||||
} else {
|
||||
@@ -1014,8 +1099,17 @@ pub fn write_live_delta_replay_step_inputs(
|
||||
(now.clone(), now)
|
||||
});
|
||||
let bucket_hash = sha256_hex(module_uri.as_bytes());
|
||||
let bucket_dir = capture_root.join("rsync").join("modules").join(&bucket_hash);
|
||||
materialize_rsync_module(&capture_root, &module_uri, objects, &created_at, &last_seen_at)?;
|
||||
let bucket_dir = capture_root
|
||||
.join("rsync")
|
||||
.join("modules")
|
||||
.join(&bucket_hash);
|
||||
materialize_rsync_module(
|
||||
&capture_root,
|
||||
&module_uri,
|
||||
objects,
|
||||
&created_at,
|
||||
&last_seen_at,
|
||||
)?;
|
||||
let files = objects.keys().cloned().collect::<Vec<_>>();
|
||||
write_json(
|
||||
&bucket_dir.join("files.json"),
|
||||
@@ -1072,8 +1166,9 @@ pub fn write_live_delta_replay_bundle_inputs(
|
||||
) -> Result<LiveDeltaCaptureSummary, String> {
|
||||
let base_archive_root = rir_dir.join("base-payload-archive");
|
||||
let base_locks_path = rir_dir.join("base-locks.json");
|
||||
let base_index = ReplayArchiveIndex::load_allow_missing_rsync_modules(&base_archive_root, &base_locks_path)
|
||||
.map_err(|e| format!("load base replay index failed: {e}"))?;
|
||||
let base_index =
|
||||
ReplayArchiveIndex::load_allow_missing_rsync_modules(&base_archive_root, &base_locks_path)
|
||||
.map_err(|e| format!("load base replay index failed: {e}"))?;
|
||||
let base_locks_bytes = fs::read(&base_locks_path)
|
||||
.map_err(|e| format!("read base locks failed: {}: {e}", base_locks_path.display()))?;
|
||||
let base_locks_sha256 = sha256_hex(&base_locks_bytes);
|
||||
@@ -1118,10 +1213,15 @@ pub fn write_live_delta_replay_bundle_inputs(
|
||||
for fetch in rsync_records.values() {
|
||||
let module_uri = canonical_rsync_module(&fetch.requested_base_uri)
|
||||
.map_err(|e| format!("canonicalize requested rsync module failed: {e}"))?;
|
||||
let objects = rsync_objects_by_module.entry(module_uri.clone()).or_default();
|
||||
let times = rsync_times_by_module
|
||||
.entry(module_uri)
|
||||
.or_insert_with(|| (fetch.fetched_at_rfc3339_utc.clone(), fetch.fetched_at_rfc3339_utc.clone()));
|
||||
let objects = rsync_objects_by_module
|
||||
.entry(module_uri.clone())
|
||||
.or_default();
|
||||
let times = rsync_times_by_module.entry(module_uri).or_insert_with(|| {
|
||||
(
|
||||
fetch.fetched_at_rfc3339_utc.clone(),
|
||||
fetch.fetched_at_rfc3339_utc.clone(),
|
||||
)
|
||||
});
|
||||
if fetch.fetched_at_rfc3339_utc < times.0 {
|
||||
times.0 = fetch.fetched_at_rfc3339_utc.clone();
|
||||
}
|
||||
@@ -1139,22 +1239,31 @@ pub fn write_live_delta_replay_bundle_inputs(
|
||||
let mut needed_modules = BTreeSet::new();
|
||||
|
||||
for pp in publication_points {
|
||||
let module_uri = canonical_rsync_module(&pp.rsync_base_uri)
|
||||
.map_err(|e| format!("canonicalize rsync module failed for {}: {e}", pp.rsync_base_uri))?;
|
||||
let module_uri = canonical_rsync_module(&pp.rsync_base_uri).map_err(|e| {
|
||||
format!(
|
||||
"canonicalize rsync module failed for {}: {e}",
|
||||
pp.rsync_base_uri
|
||||
)
|
||||
})?;
|
||||
if let Some(notify_uri) = pp.rrdp_notification_uri.as_deref() {
|
||||
if !seen_notifications.insert(notify_uri.to_string()) {
|
||||
continue;
|
||||
}
|
||||
let base_lock = base_index.rrdp_lock(notify_uri);
|
||||
let target_record = store
|
||||
.get_rrdp_source_record(notify_uri)
|
||||
.map_err(|e| format!("read target rrdp source record failed for {notify_uri}: {e}"))?;
|
||||
let target_record = store.get_rrdp_source_record(notify_uri).map_err(|e| {
|
||||
format!("read target rrdp source record failed for {notify_uri}: {e}")
|
||||
})?;
|
||||
|
||||
let bucket_hash = sha256_hex(notify_uri.as_bytes());
|
||||
let bucket_dir = capture_root.join("rrdp").join("repos").join(&bucket_hash);
|
||||
let (created_at, last_seen_at) = target_record
|
||||
.as_ref()
|
||||
.map(|record| (record.first_seen_at.rfc3339_utc.clone(), record.last_seen_at.rfc3339_utc.clone()))
|
||||
.map(|record| {
|
||||
(
|
||||
record.first_seen_at.rfc3339_utc.clone(),
|
||||
record.last_seen_at.rfc3339_utc.clone(),
|
||||
)
|
||||
})
|
||||
.unwrap_or_else(|| {
|
||||
let now = recorded_at
|
||||
.format(&Rfc3339)
|
||||
@@ -1207,20 +1316,29 @@ pub fn write_live_delta_replay_bundle_inputs(
|
||||
let notification_bytes = http_records
|
||||
.get(notify_uri)
|
||||
.map(|record| record.bytes.as_slice())
|
||||
.ok_or_else(|| format!("missing recorded target notification body for {notify_uri}"))?;
|
||||
.ok_or_else(|| {
|
||||
format!("missing recorded target notification body for {notify_uri}")
|
||||
})?;
|
||||
let base_serial = base_lock.serial.expect("checked above");
|
||||
let target_serial = target_record.last_serial.expect("checked above");
|
||||
let deltas = notification_deltas_after_serial(notification_bytes, base_serial, target_serial)?;
|
||||
let deltas = notification_deltas_after_serial(
|
||||
notification_bytes,
|
||||
base_serial,
|
||||
target_serial,
|
||||
)?;
|
||||
let mut all_present = true;
|
||||
let session = target_record
|
||||
.last_session_id
|
||||
.as_deref()
|
||||
.ok_or_else(|| format!("missing target session for {notify_uri}"))?;
|
||||
let session_dir = bucket_dir.join(session);
|
||||
let notification_path = session_dir.join(format!("notification-target-{target_serial}.xml"));
|
||||
let notification_path =
|
||||
session_dir.join(format!("notification-target-{target_serial}.xml"));
|
||||
write_bytes(¬ification_path, notification_bytes)?;
|
||||
let target_notification = parse_notification(notification_bytes)
|
||||
.map_err(|e| format!("parse target notification failed for {notify_uri}: {e}"))?;
|
||||
let target_notification =
|
||||
parse_notification(notification_bytes).map_err(|e| {
|
||||
format!("parse target notification failed for {notify_uri}: {e}")
|
||||
})?;
|
||||
let snapshot_hash_hex = hex::encode(target_notification.snapshot_hash_sha256);
|
||||
if let Some(snapshot_bytes) = http_records
|
||||
.get(&target_notification.snapshot_uri)
|
||||
@@ -1234,9 +1352,13 @@ pub fn write_live_delta_replay_bundle_inputs(
|
||||
let deltas_dir = session_dir.join("deltas");
|
||||
let mut delta_serials = Vec::new();
|
||||
for dref in &deltas {
|
||||
if let Some(delta_bytes) = http_records.get(&dref.uri).map(|record| record.bytes.as_slice()) {
|
||||
if let Some(delta_bytes) = http_records
|
||||
.get(&dref.uri)
|
||||
.map(|record| record.bytes.as_slice())
|
||||
{
|
||||
let hash = hex::encode(dref.hash_sha256);
|
||||
let path = deltas_dir.join(format!("delta-{}-{}.xml", dref.serial, hash));
|
||||
let path =
|
||||
deltas_dir.join(format!("delta-{}-{}.xml", dref.serial, hash));
|
||||
write_bytes(&path, delta_bytes)?;
|
||||
delta_serials.push(dref.serial);
|
||||
} else {
|
||||
@@ -1320,8 +1442,17 @@ pub fn write_live_delta_replay_bundle_inputs(
|
||||
(now.clone(), now)
|
||||
});
|
||||
let bucket_hash = sha256_hex(module_uri.as_bytes());
|
||||
let bucket_dir = capture_root.join("rsync").join("modules").join(&bucket_hash);
|
||||
materialize_rsync_module(&capture_root, &module_uri, objects, &created_at, &last_seen_at)?;
|
||||
let bucket_dir = capture_root
|
||||
.join("rsync")
|
||||
.join("modules")
|
||||
.join(&bucket_hash);
|
||||
materialize_rsync_module(
|
||||
&capture_root,
|
||||
&module_uri,
|
||||
objects,
|
||||
&created_at,
|
||||
&last_seen_at,
|
||||
)?;
|
||||
let files = objects.keys().cloned().collect::<Vec<_>>();
|
||||
write_json(
|
||||
&bucket_dir.join("files.json"),
|
||||
@@ -1402,7 +1533,12 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn minimal_notification(notify_uri: &str, snapshot_uri: &str, session: &str, serial: u64) -> Vec<u8> {
|
||||
fn minimal_notification(
|
||||
notify_uri: &str,
|
||||
snapshot_uri: &str,
|
||||
session: &str,
|
||||
serial: u64,
|
||||
) -> Vec<u8> {
|
||||
format!(
|
||||
r#"<notification xmlns="http://www.ripe.net/rpki/rrdp" version="1" session_id="{session}" serial="{serial}"><snapshot uri="{snapshot_uri}" hash="{}"/></notification>"#,
|
||||
sha256_hex(b"<snapshot/>")
|
||||
@@ -1426,7 +1562,10 @@ mod tests {
|
||||
#[test]
|
||||
fn recording_rsync_fetcher_records_object_sets() {
|
||||
let fetcher = RecordingRsyncFetcher::new(DummyRsyncFetcher {
|
||||
objects: vec![("rsync://example.test/repo/a.roa".to_string(), b"roa".to_vec())],
|
||||
objects: vec![(
|
||||
"rsync://example.test/repo/a.roa".to_string(),
|
||||
b"roa".to_vec(),
|
||||
)],
|
||||
});
|
||||
let got = fetcher
|
||||
.fetch_objects("rsync://example.test/repo/")
|
||||
@@ -1480,8 +1619,10 @@ mod tests {
|
||||
rrdp_notification_uri: Some(notify_uri.to_string()),
|
||||
source: "fresh".to_string(),
|
||||
repo_sync_source: None,
|
||||
repo_sync_phase: None,
|
||||
repo_sync_duration_ms: None,
|
||||
repo_sync_error: None,
|
||||
repo_terminal_state: "fresh".to_string(),
|
||||
this_update_rfc3339_utc: "2026-03-30T00:00:00Z".to_string(),
|
||||
next_update_rfc3339_utc: "2026-03-30T01:00:00Z".to_string(),
|
||||
verified_at_rfc3339_utc: "2026-03-30T00:00:02Z".to_string(),
|
||||
@@ -1498,8 +1639,10 @@ mod tests {
|
||||
rrdp_notification_uri: None,
|
||||
source: "fresh".to_string(),
|
||||
repo_sync_source: None,
|
||||
repo_sync_phase: None,
|
||||
repo_sync_duration_ms: None,
|
||||
repo_sync_error: None,
|
||||
repo_terminal_state: "fresh".to_string(),
|
||||
this_update_rfc3339_utc: "2026-03-30T00:00:00Z".to_string(),
|
||||
next_update_rfc3339_utc: "2026-03-30T01:00:00Z".to_string(),
|
||||
verified_at_rfc3339_utc: "2026-03-30T00:00:02Z".to_string(),
|
||||
|
||||
@@ -9,17 +9,16 @@ pub use compare_view::{
|
||||
};
|
||||
pub use live_capture::{
|
||||
LiveBaseCaptureSummary, LiveDeltaCaptureSummary, RecordedHttpResponse, RecordedRsyncFetch,
|
||||
RecordingHttpFetcher, RecordingRsyncFetcher, write_live_base_replay_bundle_inputs,
|
||||
write_live_delta_replay_bundle_inputs, write_live_delta_replay_step_inputs,
|
||||
write_current_replay_state_locks,
|
||||
RecordingHttpFetcher, RecordingRsyncFetcher, write_current_replay_state_locks,
|
||||
write_live_base_replay_bundle_inputs, write_live_delta_replay_bundle_inputs,
|
||||
write_live_delta_replay_step_inputs,
|
||||
};
|
||||
pub use record_io::{
|
||||
build_single_rir_bundle_manifest, copy_dir_all, load_validation_time, sha256_hex,
|
||||
write_bytes, write_json, write_live_bundle_rir_readme, write_live_bundle_top_readme,
|
||||
write_timing_json,
|
||||
build_single_rir_bundle_manifest, copy_dir_all, load_validation_time, sha256_hex, write_bytes,
|
||||
write_json, write_live_bundle_rir_readme, write_live_bundle_top_readme, write_timing_json,
|
||||
};
|
||||
pub use spec::{BundleManifest, BundleManifestEntry, RirBundleMetadata};
|
||||
pub use spec::{
|
||||
BaseBundleStateMetadataV2, BundleManifestEntryV2, BundleManifestV2, DeltaSequenceMetadataV2,
|
||||
DeltaStepMetadataV2, RirBundleMetadataV2,
|
||||
};
|
||||
pub use spec::{BundleManifest, BundleManifestEntry, RirBundleMetadata};
|
||||
|
||||
@@ -81,7 +81,9 @@ pub fn write_bytes(path: &Path, bytes: &[u8]) -> Result<(), String> {
|
||||
pub fn copy_dir_all(src: &Path, dst: &Path) -> Result<(), String> {
|
||||
fs::create_dir_all(dst)
|
||||
.map_err(|e| format!("create directory failed: {}: {e}", dst.display()))?;
|
||||
for entry in fs::read_dir(src).map_err(|e| format!("read_dir failed: {}: {e}", src.display()))? {
|
||||
for entry in
|
||||
fs::read_dir(src).map_err(|e| format!("read_dir failed: {}: {e}", src.display()))?
|
||||
{
|
||||
let entry = entry.map_err(|e| format!("read_dir entry failed: {}: {e}", src.display()))?;
|
||||
let ty = entry
|
||||
.file_type()
|
||||
@@ -94,8 +96,13 @@ pub fn copy_dir_all(src: &Path, dst: &Path) -> Result<(), String> {
|
||||
fs::create_dir_all(parent)
|
||||
.map_err(|e| format!("create parent failed: {}: {e}", parent.display()))?;
|
||||
}
|
||||
fs::copy(entry.path(), &to)
|
||||
.map_err(|e| format!("copy failed: {} -> {}: {e}", entry.path().display(), to.display()))?;
|
||||
fs::copy(entry.path(), &to).map_err(|e| {
|
||||
format!(
|
||||
"copy failed: {} -> {}: {e}",
|
||||
entry.path().display(),
|
||||
to.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -197,7 +204,10 @@ mod tests {
|
||||
.expect("manifest");
|
||||
assert_eq!(manifest.schema_version, "20260330-v1");
|
||||
assert_eq!(manifest.rirs, vec!["apnic".to_string()]);
|
||||
assert_eq!(manifest.per_rir_bundles[0].base_validation_time, "2026-04-01T00:00:00Z");
|
||||
assert_eq!(
|
||||
manifest.per_rir_bundles[0].base_validation_time,
|
||||
"2026-04-01T00:00:00Z"
|
||||
);
|
||||
assert_eq!(
|
||||
manifest.per_rir_bundles[0].delta_validation_time.as_deref(),
|
||||
Some("2026-04-01T00:10:00Z")
|
||||
@@ -255,15 +265,9 @@ mod tests {
|
||||
#[test]
|
||||
fn build_single_rir_bundle_manifest_supports_none_delta_time() {
|
||||
let base = time::OffsetDateTime::parse("2026-04-01T00:00:00Z", &Rfc3339).expect("base");
|
||||
let manifest = build_single_rir_bundle_manifest(
|
||||
"20260330-v1",
|
||||
"ours",
|
||||
"afrinic",
|
||||
&base,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.expect("manifest");
|
||||
let manifest =
|
||||
build_single_rir_bundle_manifest("20260330-v1", "ours", "afrinic", &base, None, false)
|
||||
.expect("manifest");
|
||||
assert_eq!(manifest.per_rir_bundles[0].delta_validation_time, None);
|
||||
assert!(!manifest.per_rir_bundles[0].has_aspa);
|
||||
}
|
||||
|
||||
@@ -19,7 +19,10 @@ pub struct BundleManifestEntry {
|
||||
pub relative_path: String,
|
||||
#[serde(rename = "baseValidationTime")]
|
||||
pub base_validation_time: String,
|
||||
#[serde(rename = "deltaValidationTime", skip_serializing_if = "Option::is_none")]
|
||||
#[serde(
|
||||
rename = "deltaValidationTime",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub delta_validation_time: Option<String>,
|
||||
#[serde(rename = "hasAspa")]
|
||||
pub has_aspa: bool,
|
||||
@@ -34,7 +37,10 @@ pub struct RirBundleMetadata {
|
||||
pub rir: String,
|
||||
#[serde(rename = "baseValidationTime")]
|
||||
pub base_validation_time: String,
|
||||
#[serde(rename = "deltaValidationTime", skip_serializing_if = "Option::is_none")]
|
||||
#[serde(
|
||||
rename = "deltaValidationTime",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub delta_validation_time: Option<String>,
|
||||
#[serde(rename = "talSha256")]
|
||||
pub tal_sha256: String,
|
||||
@@ -79,9 +85,15 @@ pub struct BundleManifestEntryV2 {
|
||||
pub base_validation_time: String,
|
||||
#[serde(rename = "stepCount")]
|
||||
pub step_count: usize,
|
||||
#[serde(rename = "firstDeltaValidationTime", skip_serializing_if = "Option::is_none")]
|
||||
#[serde(
|
||||
rename = "firstDeltaValidationTime",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub first_delta_validation_time: Option<String>,
|
||||
#[serde(rename = "lastDeltaValidationTime", skip_serializing_if = "Option::is_none")]
|
||||
#[serde(
|
||||
rename = "lastDeltaValidationTime",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub last_delta_validation_time: Option<String>,
|
||||
#[serde(rename = "hasAspa")]
|
||||
pub has_aspa: bool,
|
||||
|
||||
174
src/ccr/build.rs
174
src/ccr/build.rs
@@ -12,12 +12,13 @@ use crate::ccr::model::{
|
||||
AspaPayloadSet, AspaPayloadState, ManifestInstance, ManifestState, RoaPayloadSet,
|
||||
RoaPayloadState, RouterKey, RouterKeySet, RouterKeyState, TrustAnchorState,
|
||||
};
|
||||
use crate::blob_store::RawObjectStore;
|
||||
use crate::data_model::manifest::ManifestObject;
|
||||
use crate::data_model::rc::{AccessDescription, SubjectInfoAccess};
|
||||
use crate::data_model::roa::RoaAfi;
|
||||
use crate::data_model::router_cert::BgpsecRouterCertificate;
|
||||
use crate::data_model::ta::TrustAnchor;
|
||||
use crate::storage::{RocksStore, VcirArtifactRole, ValidatedCaInstanceResult};
|
||||
use crate::storage::{RocksStore, ValidatedCaInstanceResult, VcirArtifactRole};
|
||||
use crate::validation::objects::{AspaAttestation, Vrp};
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
@@ -41,13 +42,22 @@ pub enum CcrBuildError {
|
||||
MissingManifestArtifact(String),
|
||||
|
||||
#[error("manifest raw bytes missing in store for {manifest_rsync_uri}: {sha256_hex}")]
|
||||
MissingManifestRawBytes { manifest_rsync_uri: String, sha256_hex: String },
|
||||
MissingManifestRawBytes {
|
||||
manifest_rsync_uri: String,
|
||||
sha256_hex: String,
|
||||
},
|
||||
|
||||
#[error("manifest raw bytes load failed for {manifest_rsync_uri}: {detail}")]
|
||||
LoadManifestRawBytes { manifest_rsync_uri: String, detail: String },
|
||||
LoadManifestRawBytes {
|
||||
manifest_rsync_uri: String,
|
||||
detail: String,
|
||||
},
|
||||
|
||||
#[error("manifest decode failed for {manifest_rsync_uri}: {detail}")]
|
||||
ManifestDecode { manifest_rsync_uri: String, detail: String },
|
||||
ManifestDecode {
|
||||
manifest_rsync_uri: String,
|
||||
detail: String,
|
||||
},
|
||||
|
||||
#[error("manifest EE certificate missing AuthorityKeyIdentifier: {0}")]
|
||||
ManifestEeMissingAki(String),
|
||||
@@ -99,8 +109,8 @@ pub fn build_roa_payload_state(vrps: &[Vrp]) -> Result<RoaPayloadState, CcrBuild
|
||||
});
|
||||
}
|
||||
|
||||
let payload_der =
|
||||
encode_roa_payload_state_payload_der(&rps).map_err(|e| CcrBuildError::RoaEncode(e.to_string()))?;
|
||||
let payload_der = encode_roa_payload_state_payload_der(&rps)
|
||||
.map_err(|e| CcrBuildError::RoaEncode(e.to_string()))?;
|
||||
Ok(RoaPayloadState {
|
||||
rps,
|
||||
hash: compute_state_hash(&payload_der),
|
||||
@@ -176,10 +186,12 @@ pub fn build_manifest_state_from_vcirs(
|
||||
artifact.artifact_role == VcirArtifactRole::Manifest
|
||||
&& artifact.uri.as_deref() == Some(vcir.current_manifest_rsync_uri.as_str())
|
||||
})
|
||||
.ok_or_else(|| CcrBuildError::MissingManifestArtifact(vcir.current_manifest_rsync_uri.clone()))?;
|
||||
.ok_or_else(|| {
|
||||
CcrBuildError::MissingManifestArtifact(vcir.current_manifest_rsync_uri.clone())
|
||||
})?;
|
||||
|
||||
let raw_entry = store
|
||||
.get_raw_by_hash_entry(&manifest_artifact.sha256)
|
||||
let raw_bytes = store
|
||||
.get_blob_bytes(&manifest_artifact.sha256)
|
||||
.map_err(|e| CcrBuildError::LoadManifestRawBytes {
|
||||
manifest_rsync_uri: vcir.current_manifest_rsync_uri.clone(),
|
||||
detail: e.to_string(),
|
||||
@@ -189,9 +201,11 @@ pub fn build_manifest_state_from_vcirs(
|
||||
sha256_hex: manifest_artifact.sha256.clone(),
|
||||
})?;
|
||||
|
||||
let manifest = ManifestObject::decode_der(&raw_entry.bytes).map_err(|e| CcrBuildError::ManifestDecode {
|
||||
manifest_rsync_uri: vcir.current_manifest_rsync_uri.clone(),
|
||||
detail: e.to_string(),
|
||||
let manifest = ManifestObject::decode_der(&raw_bytes).map_err(|e| {
|
||||
CcrBuildError::ManifestDecode {
|
||||
manifest_rsync_uri: vcir.current_manifest_rsync_uri.clone(),
|
||||
detail: e.to_string(),
|
||||
}
|
||||
})?;
|
||||
|
||||
let ee = &manifest.signed_object.signed_data.certificates[0].resource_cert;
|
||||
@@ -200,13 +214,17 @@ pub fn build_manifest_state_from_vcirs(
|
||||
.extensions
|
||||
.authority_key_identifier
|
||||
.clone()
|
||||
.ok_or_else(|| CcrBuildError::ManifestEeMissingAki(vcir.current_manifest_rsync_uri.clone()))?;
|
||||
.ok_or_else(|| {
|
||||
CcrBuildError::ManifestEeMissingAki(vcir.current_manifest_rsync_uri.clone())
|
||||
})?;
|
||||
let sia = ee
|
||||
.tbs
|
||||
.extensions
|
||||
.subject_info_access
|
||||
.as_ref()
|
||||
.ok_or_else(|| CcrBuildError::ManifestEeMissingSia(vcir.current_manifest_rsync_uri.clone()))?;
|
||||
.ok_or_else(|| {
|
||||
CcrBuildError::ManifestEeMissingSia(vcir.current_manifest_rsync_uri.clone())
|
||||
})?;
|
||||
let locations = match sia {
|
||||
SubjectInfoAccess::Ee(ee_sia) => ee_sia
|
||||
.access_descriptions
|
||||
@@ -216,7 +234,7 @@ pub fn build_manifest_state_from_vcirs(
|
||||
SubjectInfoAccess::Ca(_) => {
|
||||
return Err(CcrBuildError::ManifestEeSiaWrongVariant(
|
||||
vcir.current_manifest_rsync_uri.clone(),
|
||||
))
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -234,11 +252,14 @@ pub fn build_manifest_state_from_vcirs(
|
||||
}
|
||||
|
||||
let instance = ManifestInstance {
|
||||
hash: sha2::Sha256::digest(&raw_entry.bytes).to_vec(),
|
||||
size: raw_entry.bytes.len() as u64,
|
||||
hash: sha2::Sha256::digest(&raw_bytes).to_vec(),
|
||||
size: raw_bytes.len() as u64,
|
||||
aki,
|
||||
manifest_number: crate::data_model::common::BigUnsigned {
|
||||
bytes_be: vcir.validated_manifest_meta.validated_manifest_number.clone(),
|
||||
bytes_be: vcir
|
||||
.validated_manifest_meta
|
||||
.validated_manifest_number
|
||||
.clone(),
|
||||
},
|
||||
this_update,
|
||||
locations,
|
||||
@@ -292,7 +313,10 @@ fn encode_access_description_der(ad: &AccessDescription) -> Result<Vec<u8>, CcrB
|
||||
fn encode_oid_from_string(oid: &str) -> Result<Vec<u8>, CcrBuildError> {
|
||||
let arcs = oid
|
||||
.split('.')
|
||||
.map(|part| part.parse::<u64>().map_err(|_| CcrBuildError::UnsupportedAccessMethodOid(oid.to_string())))
|
||||
.map(|part| {
|
||||
part.parse::<u64>()
|
||||
.map_err(|_| CcrBuildError::UnsupportedAccessMethodOid(oid.to_string()))
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
if arcs.len() < 2 {
|
||||
return Err(CcrBuildError::UnsupportedAccessMethodOid(oid.to_string()));
|
||||
@@ -471,7 +495,9 @@ fn encode_length(len: usize, out: &mut Vec<u8>) {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::ccr::decode::decode_content_info;
|
||||
use crate::ccr::encode::{encode_aspa_payload_state, encode_content_info, encode_trust_anchor_state};
|
||||
use crate::ccr::encode::{
|
||||
encode_aspa_payload_state, encode_content_info, encode_trust_anchor_state,
|
||||
};
|
||||
use crate::ccr::model::{CcrContentInfo, CcrDigestAlgorithm, RpkiCanonicalCacheRepresentation};
|
||||
use crate::data_model::roa::{IpPrefix, RoaAfi};
|
||||
use crate::data_model::ta::TrustAnchor;
|
||||
@@ -561,13 +587,19 @@ mod tests {
|
||||
ca_subject_name: "CN=test".to_string(),
|
||||
ca_ski: "11".repeat(20),
|
||||
issuer_ski: "22".repeat(20),
|
||||
last_successful_validation_time: crate::storage::PackTime::from_utc_offset_datetime(now),
|
||||
last_successful_validation_time: crate::storage::PackTime::from_utc_offset_datetime(
|
||||
now,
|
||||
),
|
||||
current_manifest_rsync_uri: manifest_uri.to_string(),
|
||||
current_crl_rsync_uri: format!("{manifest_uri}.crl"),
|
||||
validated_manifest_meta: crate::storage::ValidatedManifestMeta {
|
||||
validated_manifest_number: manifest.manifest.manifest_number.bytes_be.clone(),
|
||||
validated_manifest_this_update: crate::storage::PackTime::from_utc_offset_datetime(now),
|
||||
validated_manifest_next_update: crate::storage::PackTime::from_utc_offset_datetime(next),
|
||||
validated_manifest_this_update: crate::storage::PackTime::from_utc_offset_datetime(
|
||||
now,
|
||||
),
|
||||
validated_manifest_next_update: crate::storage::PackTime::from_utc_offset_datetime(
|
||||
next,
|
||||
),
|
||||
},
|
||||
instance_gate: crate::storage::VcirInstanceGate {
|
||||
manifest_next_update: crate::storage::PackTime::from_utc_offset_datetime(next),
|
||||
@@ -585,7 +617,9 @@ mod tests {
|
||||
child_rrdp_notification_uri: None,
|
||||
child_effective_ip_resources: None,
|
||||
child_effective_as_resources: None,
|
||||
accepted_at_validation_time: crate::storage::PackTime::from_utc_offset_datetime(now),
|
||||
accepted_at_validation_time: crate::storage::PackTime::from_utc_offset_datetime(
|
||||
now,
|
||||
),
|
||||
}],
|
||||
local_outputs: Vec::new(),
|
||||
related_artifacts: vec![crate::storage::VcirRelatedArtifact {
|
||||
@@ -617,21 +651,29 @@ mod tests {
|
||||
fn build_manifest_state_from_vcirs_collects_current_manifests_and_hashes_payload() {
|
||||
let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
|
||||
let manifest_a = std::fs::read(base.join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft")).expect("read manifest a");
|
||||
let manifest_b = std::fs::read(base.join("tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.mft")).expect("read manifest b");
|
||||
let vcir_a = sample_manifest_vcir("rsync://example.test/a.mft", &manifest_a, &"33".repeat(20));
|
||||
let vcir_b = sample_manifest_vcir("rsync://example.test/b.mft", &manifest_b, &"44".repeat(20));
|
||||
let manifest_b = std::fs::read(base.join(
|
||||
"tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.mft",
|
||||
))
|
||||
.expect("read manifest b");
|
||||
let vcir_a =
|
||||
sample_manifest_vcir("rsync://example.test/a.mft", &manifest_a, &"33".repeat(20));
|
||||
let vcir_b =
|
||||
sample_manifest_vcir("rsync://example.test/b.mft", &manifest_b, &"44".repeat(20));
|
||||
let store_dir = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
|
||||
for (vcir, bytes) in [(&vcir_a, &manifest_a), (&vcir_b, &manifest_b)] {
|
||||
let artifact = &vcir.related_artifacts[0];
|
||||
let mut raw = crate::storage::RawByHashEntry::from_bytes(artifact.sha256.clone(), bytes.to_vec());
|
||||
raw.origin_uris.push(vcir.current_manifest_rsync_uri.clone());
|
||||
let mut raw =
|
||||
crate::storage::RawByHashEntry::from_bytes(artifact.sha256.clone(), bytes.to_vec());
|
||||
raw.origin_uris
|
||||
.push(vcir.current_manifest_rsync_uri.clone());
|
||||
raw.object_type = Some("mft".to_string());
|
||||
raw.encoding = Some("der".to_string());
|
||||
store.put_raw_by_hash_entry(&raw).expect("put raw manifest");
|
||||
}
|
||||
|
||||
let state = build_manifest_state_from_vcirs(&store, &[vcir_a.clone(), vcir_b.clone()]).expect("build manifest state");
|
||||
let state = build_manifest_state_from_vcirs(&store, &[vcir_a.clone(), vcir_b.clone()])
|
||||
.expect("build manifest state");
|
||||
assert_eq!(state.mis.len(), 2);
|
||||
assert!(state.mis[0].hash < state.mis[1].hash);
|
||||
let expected_subordinates = [
|
||||
@@ -647,13 +689,24 @@ mod tests {
|
||||
mi.subordinates[0].clone()
|
||||
})
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
let expected_subordinates = expected_subordinates.into_iter().collect::<std::collections::BTreeSet<_>>();
|
||||
let expected_subordinates = expected_subordinates
|
||||
.into_iter()
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
assert_eq!(actual_subordinates, expected_subordinates);
|
||||
let payload_der = encode_manifest_state_payload_der(&state.mis).expect("encode mis payload");
|
||||
let payload_der =
|
||||
encode_manifest_state_payload_der(&state.mis).expect("encode mis payload");
|
||||
assert!(crate::ccr::verify_state_hash(&state.hash, &payload_der));
|
||||
let max_time = [
|
||||
vcir_a.validated_manifest_meta.validated_manifest_this_update.parse().unwrap(),
|
||||
vcir_b.validated_manifest_meta.validated_manifest_this_update.parse().unwrap(),
|
||||
vcir_a
|
||||
.validated_manifest_meta
|
||||
.validated_manifest_this_update
|
||||
.parse()
|
||||
.unwrap(),
|
||||
vcir_b
|
||||
.validated_manifest_meta
|
||||
.validated_manifest_this_update
|
||||
.parse()
|
||||
.unwrap(),
|
||||
]
|
||||
.into_iter()
|
||||
.max()
|
||||
@@ -668,7 +721,8 @@ mod tests {
|
||||
let state = build_manifest_state_from_vcirs(&store, &[]).expect("empty manifest state");
|
||||
assert!(state.mis.is_empty());
|
||||
assert_eq!(state.most_recent_update, time::OffsetDateTime::UNIX_EPOCH);
|
||||
let payload_der = encode_manifest_state_payload_der(&state.mis).expect("encode mis payload");
|
||||
let payload_der =
|
||||
encode_manifest_state_payload_der(&state.mis).expect("encode mis payload");
|
||||
assert!(crate::ccr::verify_state_hash(&state.hash, &payload_der));
|
||||
}
|
||||
|
||||
@@ -687,7 +741,8 @@ mod tests {
|
||||
assert_eq!(state.rksets[0].router_keys.len(), 2);
|
||||
assert_eq!(state.rksets[1].router_keys.len(), 2);
|
||||
assert!(state.rksets[0].router_keys[0].ski <= state.rksets[0].router_keys[1].ski);
|
||||
let payload_der = encode_router_key_state_payload_der(&state.rksets).expect("encode rk payload");
|
||||
let payload_der =
|
||||
encode_router_key_state_payload_der(&state.rksets).expect("encode rk payload");
|
||||
assert!(crate::ccr::verify_state_hash(&state.hash, &payload_der));
|
||||
}
|
||||
|
||||
@@ -695,7 +750,8 @@ mod tests {
|
||||
fn build_router_key_state_empty_is_valid_and_hashes_empty_sequence() {
|
||||
let state = build_router_key_state(&[]).expect("empty router key state");
|
||||
assert!(state.rksets.is_empty());
|
||||
let payload_der = encode_router_key_state_payload_der(&state.rksets).expect("encode rk payload");
|
||||
let payload_der =
|
||||
encode_router_key_state_payload_der(&state.rksets).expect("encode rk payload");
|
||||
assert!(crate::ccr::verify_state_hash(&state.hash, &payload_der));
|
||||
}
|
||||
|
||||
@@ -706,7 +762,12 @@ mod tests {
|
||||
sample_vrp_v4(64496, [10, 0, 0, 0], 8, 8),
|
||||
sample_vrp_v4(64496, [10, 0, 0, 0], 8, 8),
|
||||
sample_vrp_v4(64496, [10, 1, 0, 0], 16, 24),
|
||||
sample_vrp_v6(64496, [0x20,0x01,0x0d,0xb8,0,0,0,0,0,0,0,0,0,0,0,0], 32, 48),
|
||||
sample_vrp_v6(
|
||||
64496,
|
||||
[0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
|
||||
32,
|
||||
48,
|
||||
),
|
||||
];
|
||||
let state = build_roa_payload_state(&vrps).expect("build roa state");
|
||||
assert_eq!(state.rps.len(), 2);
|
||||
@@ -721,7 +782,7 @@ mod tests {
|
||||
assert_eq!(entries4[0], (8, vec![10], None));
|
||||
assert_eq!(entries4[1], (16, vec![10, 1], Some(24)));
|
||||
assert_eq!(entries6.len(), 1);
|
||||
assert_eq!(entries6[0], (32, vec![0x20,0x01,0x0d,0xb8], Some(48)));
|
||||
assert_eq!(entries6[0], (32, vec![0x20, 0x01, 0x0d, 0xb8], Some(48)));
|
||||
let payload_der = encode_roa_payload_state_payload_der(&state.rps).expect("encode payload");
|
||||
assert!(crate::ccr::verify_state_hash(&state.hash, &payload_der));
|
||||
}
|
||||
@@ -737,9 +798,18 @@ mod tests {
|
||||
#[test]
|
||||
fn build_aspa_payload_state_merges_and_sorts() {
|
||||
let aspas = vec![
|
||||
AspaAttestation { customer_as_id: 64497, provider_as_ids: vec![65002] },
|
||||
AspaAttestation { customer_as_id: 64496, provider_as_ids: vec![65003, 65001] },
|
||||
AspaAttestation { customer_as_id: 64496, provider_as_ids: vec![65002, 65001] },
|
||||
AspaAttestation {
|
||||
customer_as_id: 64497,
|
||||
provider_as_ids: vec![65002],
|
||||
},
|
||||
AspaAttestation {
|
||||
customer_as_id: 64496,
|
||||
provider_as_ids: vec![65003, 65001],
|
||||
},
|
||||
AspaAttestation {
|
||||
customer_as_id: 64496,
|
||||
provider_as_ids: vec![65002, 65001],
|
||||
},
|
||||
];
|
||||
let state = build_aspa_payload_state(&aspas).expect("build aspa state");
|
||||
assert_eq!(state.aps.len(), 2);
|
||||
@@ -747,8 +817,8 @@ mod tests {
|
||||
assert_eq!(state.aps[0].providers, vec![65001, 65002, 65003]);
|
||||
assert_eq!(state.aps[1].customer_as_id, 64497);
|
||||
let encoded = encode_aspa_payload_state(&state).expect("encode aspa state");
|
||||
let decoded = decode_content_info(&encode_content_info(&CcrContentInfo::new(
|
||||
RpkiCanonicalCacheRepresentation {
|
||||
let decoded = decode_content_info(
|
||||
&encode_content_info(&CcrContentInfo::new(RpkiCanonicalCacheRepresentation {
|
||||
version: 0,
|
||||
hash_alg: CcrDigestAlgorithm::Sha256,
|
||||
produced_at: time::OffsetDateTime::now_utc(),
|
||||
@@ -757,9 +827,10 @@ mod tests {
|
||||
vaps: Some(state.clone()),
|
||||
tas: None,
|
||||
rks: None,
|
||||
},
|
||||
))
|
||||
.expect("encode ccr")).expect("decode ccr");
|
||||
}))
|
||||
.expect("encode ccr"),
|
||||
)
|
||||
.expect("decode ccr");
|
||||
assert_eq!(decoded.content.vaps, Some(state));
|
||||
assert!(!encoded.is_empty());
|
||||
}
|
||||
@@ -778,7 +849,8 @@ mod tests {
|
||||
.expect("build ta state");
|
||||
assert_eq!(state.skis.len(), 2);
|
||||
assert!(state.skis[0] < state.skis[1]);
|
||||
let payload_der = encode_trust_anchor_state_payload_der(&state.skis).expect("encode ta payload");
|
||||
let payload_der =
|
||||
encode_trust_anchor_state_payload_der(&state.skis).expect("encode ta payload");
|
||||
assert!(crate::ccr::verify_state_hash(&state.hash, &payload_der));
|
||||
let encoded = encode_trust_anchor_state(&state).expect("encode ta state");
|
||||
assert!(!encoded.is_empty());
|
||||
@@ -793,7 +865,11 @@ mod tests {
|
||||
"tests/fixtures/tal/apnic-rfc7730-https.tal",
|
||||
"tests/fixtures/ta/apnic-ta.cer",
|
||||
);
|
||||
ta.ta_certificate.rc_ca.tbs.extensions.subject_key_identifier = None;
|
||||
ta.ta_certificate
|
||||
.rc_ca
|
||||
.tbs
|
||||
.extensions
|
||||
.subject_key_identifier = None;
|
||||
let err = build_trust_anchor_state(&[ta]).expect_err("missing ski must fail");
|
||||
assert!(err.to_string().contains("SubjectKeyIdentifier"), "{err}");
|
||||
}
|
||||
|
||||
@@ -13,10 +13,16 @@ pub enum CcrDecodeError {
|
||||
Parse(String),
|
||||
|
||||
#[error("unexpected contentType OID: expected {expected}, got {actual}")]
|
||||
UnexpectedContentType { expected: &'static str, actual: String },
|
||||
UnexpectedContentType {
|
||||
expected: &'static str,
|
||||
actual: String,
|
||||
},
|
||||
|
||||
#[error("unexpected digest algorithm OID: expected {expected}, got {actual}")]
|
||||
UnexpectedDigestAlgorithm { expected: &'static str, actual: String },
|
||||
UnexpectedDigestAlgorithm {
|
||||
expected: &'static str,
|
||||
actual: String,
|
||||
},
|
||||
|
||||
#[error("CCR model validation failed after decode: {0}")]
|
||||
Validate(String),
|
||||
@@ -26,7 +32,9 @@ pub fn decode_content_info(der: &[u8]) -> Result<CcrContentInfo, CcrDecodeError>
|
||||
let mut top = DerReader::new(der);
|
||||
let mut seq = top.take_sequence().map_err(CcrDecodeError::Parse)?;
|
||||
if !top.is_empty() {
|
||||
return Err(CcrDecodeError::Parse("trailing bytes after ContentInfo".into()));
|
||||
return Err(CcrDecodeError::Parse(
|
||||
"trailing bytes after ContentInfo".into(),
|
||||
));
|
||||
}
|
||||
let content_type_raw = seq.take_tag(0x06).map_err(CcrDecodeError::Parse)?;
|
||||
if content_type_raw != OID_CT_RPKI_CCR_RAW {
|
||||
@@ -37,7 +45,9 @@ pub fn decode_content_info(der: &[u8]) -> Result<CcrContentInfo, CcrDecodeError>
|
||||
}
|
||||
let inner = seq.take_tag(0xA0).map_err(CcrDecodeError::Parse)?;
|
||||
if !seq.is_empty() {
|
||||
return Err(CcrDecodeError::Parse("trailing fields in ContentInfo".into()));
|
||||
return Err(CcrDecodeError::Parse(
|
||||
"trailing fields in ContentInfo".into(),
|
||||
));
|
||||
}
|
||||
let content = decode_ccr(inner)?;
|
||||
let ci = CcrContentInfo::new(content);
|
||||
@@ -87,7 +97,7 @@ pub fn decode_ccr(der: &[u8]) -> Result<RpkiCanonicalCacheRepresentation, CcrDec
|
||||
_ => {
|
||||
return Err(CcrDecodeError::Parse(format!(
|
||||
"unexpected CCR field tag 0x{tag:02X}"
|
||||
)))
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -110,7 +120,9 @@ fn decode_manifest_state(explicit_der: &[u8]) -> Result<ManifestState, CcrDecode
|
||||
let mut outer = DerReader::new(explicit_der);
|
||||
let mut seq = outer.take_sequence().map_err(CcrDecodeError::Parse)?;
|
||||
if !outer.is_empty() {
|
||||
return Err(CcrDecodeError::Parse("trailing bytes after ManifestState".into()));
|
||||
return Err(CcrDecodeError::Parse(
|
||||
"trailing bytes after ManifestState".into(),
|
||||
));
|
||||
}
|
||||
let mis_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?;
|
||||
let mut mis_reader = DerReader::new(mis_der);
|
||||
@@ -119,30 +131,50 @@ fn decode_manifest_state(explicit_der: &[u8]) -> Result<ManifestState, CcrDecode
|
||||
let (_tag, full, _value) = mis_reader.take_any_full().map_err(CcrDecodeError::Parse)?;
|
||||
mis.push(decode_manifest_instance(full)?);
|
||||
}
|
||||
let most_recent_update = parse_generalized_time(seq.take_tag(0x18).map_err(CcrDecodeError::Parse)?)?;
|
||||
let hash = seq.take_octet_string().map_err(CcrDecodeError::Parse)?.to_vec();
|
||||
let most_recent_update =
|
||||
parse_generalized_time(seq.take_tag(0x18).map_err(CcrDecodeError::Parse)?)?;
|
||||
let hash = seq
|
||||
.take_octet_string()
|
||||
.map_err(CcrDecodeError::Parse)?
|
||||
.to_vec();
|
||||
if !seq.is_empty() {
|
||||
return Err(CcrDecodeError::Parse("trailing fields in ManifestState".into()));
|
||||
return Err(CcrDecodeError::Parse(
|
||||
"trailing fields in ManifestState".into(),
|
||||
));
|
||||
}
|
||||
Ok(ManifestState { mis, most_recent_update, hash })
|
||||
Ok(ManifestState {
|
||||
mis,
|
||||
most_recent_update,
|
||||
hash,
|
||||
})
|
||||
}
|
||||
|
||||
fn decode_manifest_instance(der: &[u8]) -> Result<ManifestInstance, CcrDecodeError> {
|
||||
let mut top = DerReader::new(der);
|
||||
let mut seq = top.take_sequence().map_err(CcrDecodeError::Parse)?;
|
||||
if !top.is_empty() {
|
||||
return Err(CcrDecodeError::Parse("trailing bytes after ManifestInstance".into()));
|
||||
return Err(CcrDecodeError::Parse(
|
||||
"trailing bytes after ManifestInstance".into(),
|
||||
));
|
||||
}
|
||||
let hash = seq.take_octet_string().map_err(CcrDecodeError::Parse)?.to_vec();
|
||||
let hash = seq
|
||||
.take_octet_string()
|
||||
.map_err(CcrDecodeError::Parse)?
|
||||
.to_vec();
|
||||
let size = seq.take_uint_u64().map_err(CcrDecodeError::Parse)?;
|
||||
let aki = seq.take_octet_string().map_err(CcrDecodeError::Parse)?.to_vec();
|
||||
let aki = seq
|
||||
.take_octet_string()
|
||||
.map_err(CcrDecodeError::Parse)?
|
||||
.to_vec();
|
||||
let manifest_number = decode_big_unsigned(seq.take_tag(0x02).map_err(CcrDecodeError::Parse)?)?;
|
||||
let this_update = parse_generalized_time(seq.take_tag(0x18).map_err(CcrDecodeError::Parse)?)?;
|
||||
let locations_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?;
|
||||
let mut locations_reader = DerReader::new(locations_der);
|
||||
let mut locations = Vec::new();
|
||||
while !locations_reader.is_empty() {
|
||||
let (_tag, full, _value) = locations_reader.take_any_full().map_err(CcrDecodeError::Parse)?;
|
||||
let (_tag, full, _value) = locations_reader
|
||||
.take_any_full()
|
||||
.map_err(CcrDecodeError::Parse)?;
|
||||
locations.push(full.to_vec());
|
||||
}
|
||||
let subordinates = if !seq.is_empty() {
|
||||
@@ -150,20 +182,35 @@ fn decode_manifest_instance(der: &[u8]) -> Result<ManifestInstance, CcrDecodeErr
|
||||
let mut reader = DerReader::new(subordinate_der);
|
||||
let mut out = Vec::new();
|
||||
while !reader.is_empty() {
|
||||
out.push(reader.take_octet_string().map_err(CcrDecodeError::Parse)?.to_vec());
|
||||
out.push(
|
||||
reader
|
||||
.take_octet_string()
|
||||
.map_err(CcrDecodeError::Parse)?
|
||||
.to_vec(),
|
||||
);
|
||||
}
|
||||
out
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
Ok(ManifestInstance { hash, size, aki, manifest_number, this_update, locations, subordinates })
|
||||
Ok(ManifestInstance {
|
||||
hash,
|
||||
size,
|
||||
aki,
|
||||
manifest_number,
|
||||
this_update,
|
||||
locations,
|
||||
subordinates,
|
||||
})
|
||||
}
|
||||
|
||||
fn decode_roa_payload_state(explicit_der: &[u8]) -> Result<RoaPayloadState, CcrDecodeError> {
|
||||
let mut outer = DerReader::new(explicit_der);
|
||||
let mut seq = outer.take_sequence().map_err(CcrDecodeError::Parse)?;
|
||||
if !outer.is_empty() {
|
||||
return Err(CcrDecodeError::Parse("trailing bytes after ROAPayloadState".into()));
|
||||
return Err(CcrDecodeError::Parse(
|
||||
"trailing bytes after ROAPayloadState".into(),
|
||||
));
|
||||
}
|
||||
let payload_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?;
|
||||
let mut reader = DerReader::new(payload_der);
|
||||
@@ -172,7 +219,10 @@ fn decode_roa_payload_state(explicit_der: &[u8]) -> Result<RoaPayloadState, CcrD
|
||||
let (_tag, full, _value) = reader.take_any_full().map_err(CcrDecodeError::Parse)?;
|
||||
rps.push(decode_roa_payload_set(full)?);
|
||||
}
|
||||
let hash = seq.take_octet_string().map_err(CcrDecodeError::Parse)?.to_vec();
|
||||
let hash = seq
|
||||
.take_octet_string()
|
||||
.map_err(CcrDecodeError::Parse)?
|
||||
.to_vec();
|
||||
Ok(RoaPayloadState { rps, hash })
|
||||
}
|
||||
|
||||
@@ -180,7 +230,9 @@ fn decode_roa_payload_set(der: &[u8]) -> Result<RoaPayloadSet, CcrDecodeError> {
|
||||
let mut top = DerReader::new(der);
|
||||
let mut seq = top.take_sequence().map_err(CcrDecodeError::Parse)?;
|
||||
if !top.is_empty() {
|
||||
return Err(CcrDecodeError::Parse("trailing bytes after ROAPayloadSet".into()));
|
||||
return Err(CcrDecodeError::Parse(
|
||||
"trailing bytes after ROAPayloadSet".into(),
|
||||
));
|
||||
}
|
||||
let as_id = seq.take_uint_u64().map_err(CcrDecodeError::Parse)? as u32;
|
||||
let blocks_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?;
|
||||
@@ -190,14 +242,19 @@ fn decode_roa_payload_set(der: &[u8]) -> Result<RoaPayloadSet, CcrDecodeError> {
|
||||
let (_tag, full, _value) = reader.take_any_full().map_err(CcrDecodeError::Parse)?;
|
||||
ip_addr_blocks.push(full.to_vec());
|
||||
}
|
||||
Ok(RoaPayloadSet { as_id, ip_addr_blocks })
|
||||
Ok(RoaPayloadSet {
|
||||
as_id,
|
||||
ip_addr_blocks,
|
||||
})
|
||||
}
|
||||
|
||||
fn decode_aspa_payload_state(explicit_der: &[u8]) -> Result<AspaPayloadState, CcrDecodeError> {
|
||||
let mut outer = DerReader::new(explicit_der);
|
||||
let mut seq = outer.take_sequence().map_err(CcrDecodeError::Parse)?;
|
||||
if !outer.is_empty() {
|
||||
return Err(CcrDecodeError::Parse("trailing bytes after ASPAPayloadState".into()));
|
||||
return Err(CcrDecodeError::Parse(
|
||||
"trailing bytes after ASPAPayloadState".into(),
|
||||
));
|
||||
}
|
||||
let payload_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?;
|
||||
let mut reader = DerReader::new(payload_der);
|
||||
@@ -206,7 +263,10 @@ fn decode_aspa_payload_state(explicit_der: &[u8]) -> Result<AspaPayloadState, Cc
|
||||
let (_tag, full, _value) = reader.take_any_full().map_err(CcrDecodeError::Parse)?;
|
||||
aps.push(decode_aspa_payload_set(full)?);
|
||||
}
|
||||
let hash = seq.take_octet_string().map_err(CcrDecodeError::Parse)?.to_vec();
|
||||
let hash = seq
|
||||
.take_octet_string()
|
||||
.map_err(CcrDecodeError::Parse)?
|
||||
.to_vec();
|
||||
Ok(AspaPayloadState { aps, hash })
|
||||
}
|
||||
|
||||
@@ -214,7 +274,9 @@ fn decode_aspa_payload_set(der: &[u8]) -> Result<AspaPayloadSet, CcrDecodeError>
|
||||
let mut top = DerReader::new(der);
|
||||
let mut seq = top.take_sequence().map_err(CcrDecodeError::Parse)?;
|
||||
if !top.is_empty() {
|
||||
return Err(CcrDecodeError::Parse("trailing bytes after ASPAPayloadSet".into()));
|
||||
return Err(CcrDecodeError::Parse(
|
||||
"trailing bytes after ASPAPayloadSet".into(),
|
||||
));
|
||||
}
|
||||
let customer_as_id = seq.take_uint_u64().map_err(CcrDecodeError::Parse)? as u32;
|
||||
let providers_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?;
|
||||
@@ -223,22 +285,35 @@ fn decode_aspa_payload_set(der: &[u8]) -> Result<AspaPayloadSet, CcrDecodeError>
|
||||
while !reader.is_empty() {
|
||||
providers.push(reader.take_uint_u64().map_err(CcrDecodeError::Parse)? as u32);
|
||||
}
|
||||
Ok(AspaPayloadSet { customer_as_id, providers })
|
||||
Ok(AspaPayloadSet {
|
||||
customer_as_id,
|
||||
providers,
|
||||
})
|
||||
}
|
||||
|
||||
fn decode_trust_anchor_state(explicit_der: &[u8]) -> Result<TrustAnchorState, CcrDecodeError> {
|
||||
let mut outer = DerReader::new(explicit_der);
|
||||
let mut seq = outer.take_sequence().map_err(CcrDecodeError::Parse)?;
|
||||
if !outer.is_empty() {
|
||||
return Err(CcrDecodeError::Parse("trailing bytes after TrustAnchorState".into()));
|
||||
return Err(CcrDecodeError::Parse(
|
||||
"trailing bytes after TrustAnchorState".into(),
|
||||
));
|
||||
}
|
||||
let skis_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?;
|
||||
let mut reader = DerReader::new(skis_der);
|
||||
let mut skis = Vec::new();
|
||||
while !reader.is_empty() {
|
||||
skis.push(reader.take_octet_string().map_err(CcrDecodeError::Parse)?.to_vec());
|
||||
skis.push(
|
||||
reader
|
||||
.take_octet_string()
|
||||
.map_err(CcrDecodeError::Parse)?
|
||||
.to_vec(),
|
||||
);
|
||||
}
|
||||
let hash = seq.take_octet_string().map_err(CcrDecodeError::Parse)?.to_vec();
|
||||
let hash = seq
|
||||
.take_octet_string()
|
||||
.map_err(CcrDecodeError::Parse)?
|
||||
.to_vec();
|
||||
Ok(TrustAnchorState { skis, hash })
|
||||
}
|
||||
|
||||
@@ -246,7 +321,9 @@ fn decode_router_key_state(explicit_der: &[u8]) -> Result<RouterKeyState, CcrDec
|
||||
let mut outer = DerReader::new(explicit_der);
|
||||
let mut seq = outer.take_sequence().map_err(CcrDecodeError::Parse)?;
|
||||
if !outer.is_empty() {
|
||||
return Err(CcrDecodeError::Parse("trailing bytes after RouterKeyState".into()));
|
||||
return Err(CcrDecodeError::Parse(
|
||||
"trailing bytes after RouterKeyState".into(),
|
||||
));
|
||||
}
|
||||
let sets_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?;
|
||||
let mut reader = DerReader::new(sets_der);
|
||||
@@ -255,7 +332,10 @@ fn decode_router_key_state(explicit_der: &[u8]) -> Result<RouterKeyState, CcrDec
|
||||
let (_tag, full, _value) = reader.take_any_full().map_err(CcrDecodeError::Parse)?;
|
||||
rksets.push(decode_router_key_set(full)?);
|
||||
}
|
||||
let hash = seq.take_octet_string().map_err(CcrDecodeError::Parse)?.to_vec();
|
||||
let hash = seq
|
||||
.take_octet_string()
|
||||
.map_err(CcrDecodeError::Parse)?
|
||||
.to_vec();
|
||||
Ok(RouterKeyState { rksets, hash })
|
||||
}
|
||||
|
||||
@@ -263,7 +343,9 @@ fn decode_router_key_set(der: &[u8]) -> Result<RouterKeySet, CcrDecodeError> {
|
||||
let mut top = DerReader::new(der);
|
||||
let mut seq = top.take_sequence().map_err(CcrDecodeError::Parse)?;
|
||||
if !top.is_empty() {
|
||||
return Err(CcrDecodeError::Parse("trailing bytes after RouterKeySet".into()));
|
||||
return Err(CcrDecodeError::Parse(
|
||||
"trailing bytes after RouterKeySet".into(),
|
||||
));
|
||||
}
|
||||
let as_id = seq.take_uint_u64().map_err(CcrDecodeError::Parse)? as u32;
|
||||
let keys_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?;
|
||||
@@ -280,14 +362,22 @@ fn decode_router_key(der: &[u8]) -> Result<RouterKey, CcrDecodeError> {
|
||||
let mut top = DerReader::new(der);
|
||||
let mut seq = top.take_sequence().map_err(CcrDecodeError::Parse)?;
|
||||
if !top.is_empty() {
|
||||
return Err(CcrDecodeError::Parse("trailing bytes after RouterKey".into()));
|
||||
return Err(CcrDecodeError::Parse(
|
||||
"trailing bytes after RouterKey".into(),
|
||||
));
|
||||
}
|
||||
let ski = seq.take_octet_string().map_err(CcrDecodeError::Parse)?.to_vec();
|
||||
let ski = seq
|
||||
.take_octet_string()
|
||||
.map_err(CcrDecodeError::Parse)?
|
||||
.to_vec();
|
||||
let (_tag, full, _value) = seq.take_any_full().map_err(CcrDecodeError::Parse)?;
|
||||
if !seq.is_empty() {
|
||||
return Err(CcrDecodeError::Parse("trailing fields in RouterKey".into()));
|
||||
}
|
||||
Ok(RouterKey { ski, spki_der: full.to_vec() })
|
||||
Ok(RouterKey {
|
||||
ski,
|
||||
spki_der: full.to_vec(),
|
||||
})
|
||||
}
|
||||
|
||||
fn decode_digest_algorithm(mut seq: DerReader<'_>) -> Result<CcrDigestAlgorithm, CcrDecodeError> {
|
||||
@@ -354,8 +444,7 @@ fn parse_generalized_time(bytes: &[u8]) -> Result<time::OffsetDateTime, CcrDecod
|
||||
let hour = parse(8..10)? as u8;
|
||||
let minute = parse(10..12)? as u8;
|
||||
let second = parse(12..14)? as u8;
|
||||
let month = time::Month::try_from(month)
|
||||
.map_err(|e| CcrDecodeError::Parse(e.to_string()))?;
|
||||
let month = time::Month::try_from(month).map_err(|e| CcrDecodeError::Parse(e.to_string()))?;
|
||||
let date = time::Date::from_calendar_date(year, month, day)
|
||||
.map_err(|e| CcrDecodeError::Parse(e.to_string()))?;
|
||||
let timev = time::Time::from_hms(hour, minute, second)
|
||||
@@ -371,7 +460,9 @@ fn decode_big_unsigned(bytes: &[u8]) -> Result<BigUnsigned, CcrDecodeError> {
|
||||
return Err(CcrDecodeError::Parse("INTEGER must be non-negative".into()));
|
||||
}
|
||||
if bytes.len() > 1 && bytes[0] == 0x00 && (bytes[1] & 0x80) == 0 {
|
||||
return Err(CcrDecodeError::Parse("INTEGER not minimally encoded".into()));
|
||||
return Err(CcrDecodeError::Parse(
|
||||
"INTEGER not minimally encoded".into(),
|
||||
));
|
||||
}
|
||||
let bytes_be = if bytes.len() > 1 && bytes[0] == 0x00 {
|
||||
bytes[1..].to_vec()
|
||||
|
||||
@@ -34,34 +34,58 @@ pub fn dump_content_info_json(
|
||||
})
|
||||
}).unwrap_or_else(|| json!({"present": false}));
|
||||
|
||||
let vrps_total = content_info.content.vrps.as_ref().map(|state| {
|
||||
state.rps.iter().map(|set| set.ip_addr_blocks.len()).sum::<usize>()
|
||||
}).unwrap_or(0);
|
||||
let vrps = content_info.content.vrps.as_ref().map(|state| {
|
||||
json!({
|
||||
"present": true,
|
||||
"payload_sets": state.rps.len(),
|
||||
"hash_hex": hex::encode(&state.hash),
|
||||
"ip_addr_block_count": vrps_total,
|
||||
let vrps_total = content_info
|
||||
.content
|
||||
.vrps
|
||||
.as_ref()
|
||||
.map(|state| {
|
||||
state
|
||||
.rps
|
||||
.iter()
|
||||
.map(|set| set.ip_addr_blocks.len())
|
||||
.sum::<usize>()
|
||||
})
|
||||
}).unwrap_or_else(|| json!({"present": false}));
|
||||
.unwrap_or(0);
|
||||
let vrps = content_info
|
||||
.content
|
||||
.vrps
|
||||
.as_ref()
|
||||
.map(|state| {
|
||||
json!({
|
||||
"present": true,
|
||||
"payload_sets": state.rps.len(),
|
||||
"hash_hex": hex::encode(&state.hash),
|
||||
"ip_addr_block_count": vrps_total,
|
||||
})
|
||||
})
|
||||
.unwrap_or_else(|| json!({"present": false}));
|
||||
|
||||
let vaps = content_info.content.vaps.as_ref().map(|state| {
|
||||
json!({
|
||||
"present": true,
|
||||
"payload_sets": state.aps.len(),
|
||||
"hash_hex": hex::encode(&state.hash),
|
||||
"provider_count": state.aps.iter().map(|set| set.providers.len()).sum::<usize>(),
|
||||
let vaps = content_info
|
||||
.content
|
||||
.vaps
|
||||
.as_ref()
|
||||
.map(|state| {
|
||||
json!({
|
||||
"present": true,
|
||||
"payload_sets": state.aps.len(),
|
||||
"hash_hex": hex::encode(&state.hash),
|
||||
"provider_count": state.aps.iter().map(|set| set.providers.len()).sum::<usize>(),
|
||||
})
|
||||
})
|
||||
}).unwrap_or_else(|| json!({"present": false}));
|
||||
.unwrap_or_else(|| json!({"present": false}));
|
||||
|
||||
let tas = content_info.content.tas.as_ref().map(|state| {
|
||||
json!({
|
||||
"present": true,
|
||||
"ski_count": state.skis.len(),
|
||||
"hash_hex": hex::encode(&state.hash),
|
||||
let tas = content_info
|
||||
.content
|
||||
.tas
|
||||
.as_ref()
|
||||
.map(|state| {
|
||||
json!({
|
||||
"present": true,
|
||||
"ski_count": state.skis.len(),
|
||||
"hash_hex": hex::encode(&state.hash),
|
||||
})
|
||||
})
|
||||
}).unwrap_or_else(|| json!({"present": false}));
|
||||
.unwrap_or_else(|| json!({"present": false}));
|
||||
|
||||
let rks = content_info.content.rks.as_ref().map(|state| {
|
||||
json!({
|
||||
|
||||
@@ -24,9 +24,7 @@ pub fn encode_content_info(content_info: &CcrContentInfo) -> Result<Vec<u8>, Ccr
|
||||
]))
|
||||
}
|
||||
|
||||
pub fn encode_ccr(
|
||||
ccr: &RpkiCanonicalCacheRepresentation,
|
||||
) -> Result<Vec<u8>, CcrEncodeError> {
|
||||
pub fn encode_ccr(ccr: &RpkiCanonicalCacheRepresentation) -> Result<Vec<u8>, CcrEncodeError> {
|
||||
ccr.validate().map_err(CcrEncodeError::Validate)?;
|
||||
let mut fields = Vec::new();
|
||||
if ccr.version != CCR_VERSION_V0 {
|
||||
@@ -156,11 +154,12 @@ pub fn encode_trust_anchor_state(state: &TrustAnchorState) -> Result<Vec<u8>, Cc
|
||||
Ok(encode_sequence(&[skis, encode_octet_string(&state.hash)]))
|
||||
}
|
||||
|
||||
pub fn encode_trust_anchor_state_payload_der(
|
||||
skis: &[Vec<u8>],
|
||||
) -> Result<Vec<u8>, CcrEncodeError> {
|
||||
pub fn encode_trust_anchor_state_payload_der(skis: &[Vec<u8>]) -> Result<Vec<u8>, CcrEncodeError> {
|
||||
Ok(encode_sequence(
|
||||
&skis.iter().map(|ski| encode_octet_string(ski)).collect::<Vec<_>>(),
|
||||
&skis
|
||||
.iter()
|
||||
.map(|ski| encode_octet_string(ski))
|
||||
.collect::<Vec<_>>(),
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
@@ -54,7 +54,6 @@ pub fn build_ccr_from_run(
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
fn build_router_key_state_from_runtime(
|
||||
router_keys: &[RouterKeyPayload],
|
||||
) -> Result<crate::ccr::model::RouterKeyState, CcrBuildError> {
|
||||
@@ -109,17 +108,22 @@ pub fn write_ccr_file(
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::ccr::decode::decode_content_info;
|
||||
use crate::data_model::ta::TrustAnchor;
|
||||
use crate::data_model::tal::Tal;
|
||||
use crate::storage::{RawByHashEntry, RocksStore, VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, VcirChildEntry, VcirInstanceGate, VcirRelatedArtifact, VcirSummary, ValidatedCaInstanceResult, ValidatedManifestMeta, PackTime};
|
||||
use crate::validation::objects::{AspaAttestation, RouterKeyPayload, Vrp};
|
||||
use crate::data_model::manifest::ManifestObject;
|
||||
use crate::data_model::roa::{IpPrefix, RoaAfi};
|
||||
use crate::data_model::ta::TrustAnchor;
|
||||
use crate::data_model::tal::Tal;
|
||||
use crate::storage::{
|
||||
PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, ValidatedManifestMeta,
|
||||
VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary,
|
||||
VcirChildEntry, VcirInstanceGate, VcirRelatedArtifact, VcirSummary,
|
||||
};
|
||||
use crate::validation::objects::{AspaAttestation, RouterKeyPayload, Vrp};
|
||||
use sha2::Digest;
|
||||
|
||||
fn sample_trust_anchor() -> TrustAnchor {
|
||||
let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
|
||||
let tal_bytes = std::fs::read(base.join("tests/fixtures/tal/apnic-rfc7730-https.tal")).expect("read tal");
|
||||
let tal_bytes = std::fs::read(base.join("tests/fixtures/tal/apnic-rfc7730-https.tal"))
|
||||
.expect("read tal");
|
||||
let ta_der = std::fs::read(base.join("tests/fixtures/ta/apnic-ta.cer")).expect("read ta");
|
||||
let tal = Tal::decode_bytes(&tal_bytes).expect("decode tal");
|
||||
TrustAnchor::bind_der(tal, &ta_der, None).expect("bind ta")
|
||||
@@ -131,7 +135,8 @@ mod tests {
|
||||
let manifest = ManifestObject::decode_der(&manifest_der).expect("decode manifest");
|
||||
let hash = hex::encode(sha2::Sha256::digest(&manifest_der));
|
||||
let mut raw = RawByHashEntry::from_bytes(hash.clone(), manifest_der.clone());
|
||||
raw.origin_uris.push("rsync://example.test/repo/current.mft".to_string());
|
||||
raw.origin_uris
|
||||
.push("rsync://example.test/repo/current.mft".to_string());
|
||||
raw.object_type = Some("mft".to_string());
|
||||
raw.encoding = Some("der".to_string());
|
||||
store.put_raw_by_hash_entry(&raw).expect("put raw");
|
||||
@@ -142,19 +147,33 @@ mod tests {
|
||||
ca_subject_name: "CN=test".to_string(),
|
||||
ca_ski: "11".repeat(20),
|
||||
issuer_ski: "22".repeat(20),
|
||||
last_successful_validation_time: PackTime::from_utc_offset_datetime(manifest.manifest.this_update),
|
||||
last_successful_validation_time: PackTime::from_utc_offset_datetime(
|
||||
manifest.manifest.this_update,
|
||||
),
|
||||
current_manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(),
|
||||
current_crl_rsync_uri: "rsync://example.test/repo/current.crl".to_string(),
|
||||
validated_manifest_meta: ValidatedManifestMeta {
|
||||
validated_manifest_number: manifest.manifest.manifest_number.bytes_be.clone(),
|
||||
validated_manifest_this_update: PackTime::from_utc_offset_datetime(manifest.manifest.this_update),
|
||||
validated_manifest_next_update: PackTime::from_utc_offset_datetime(manifest.manifest.next_update),
|
||||
validated_manifest_this_update: PackTime::from_utc_offset_datetime(
|
||||
manifest.manifest.this_update,
|
||||
),
|
||||
validated_manifest_next_update: PackTime::from_utc_offset_datetime(
|
||||
manifest.manifest.next_update,
|
||||
),
|
||||
},
|
||||
instance_gate: VcirInstanceGate {
|
||||
manifest_next_update: PackTime::from_utc_offset_datetime(manifest.manifest.next_update),
|
||||
current_crl_next_update: PackTime::from_utc_offset_datetime(manifest.manifest.next_update),
|
||||
self_ca_not_after: PackTime::from_utc_offset_datetime(manifest.manifest.next_update),
|
||||
instance_effective_until: PackTime::from_utc_offset_datetime(manifest.manifest.next_update),
|
||||
manifest_next_update: PackTime::from_utc_offset_datetime(
|
||||
manifest.manifest.next_update,
|
||||
),
|
||||
current_crl_next_update: PackTime::from_utc_offset_datetime(
|
||||
manifest.manifest.next_update,
|
||||
),
|
||||
self_ca_not_after: PackTime::from_utc_offset_datetime(
|
||||
manifest.manifest.next_update,
|
||||
),
|
||||
instance_effective_until: PackTime::from_utc_offset_datetime(
|
||||
manifest.manifest.next_update,
|
||||
),
|
||||
},
|
||||
child_entries: vec![VcirChildEntry {
|
||||
child_manifest_rsync_uri: "rsync://example.test/repo/child.mft".to_string(),
|
||||
@@ -166,7 +185,9 @@ mod tests {
|
||||
child_rrdp_notification_uri: None,
|
||||
child_effective_ip_resources: None,
|
||||
child_effective_as_resources: None,
|
||||
accepted_at_validation_time: PackTime::from_utc_offset_datetime(manifest.manifest.this_update),
|
||||
accepted_at_validation_time: PackTime::from_utc_offset_datetime(
|
||||
manifest.manifest.this_update,
|
||||
),
|
||||
}],
|
||||
local_outputs: Vec::new(),
|
||||
related_artifacts: vec![VcirRelatedArtifact {
|
||||
@@ -204,10 +225,17 @@ mod tests {
|
||||
let trust_anchor = sample_trust_anchor();
|
||||
let vrps = vec![Vrp {
|
||||
asn: 64496,
|
||||
prefix: IpPrefix { afi: RoaAfi::Ipv4, prefix_len: 8, addr: [10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] },
|
||||
prefix: IpPrefix {
|
||||
afi: RoaAfi::Ipv4,
|
||||
prefix_len: 8,
|
||||
addr: [10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
|
||||
},
|
||||
max_length: 8,
|
||||
}];
|
||||
let aspas = vec![AspaAttestation { customer_as_id: 64496, provider_as_ids: vec![64497] }];
|
||||
let aspas = vec![AspaAttestation {
|
||||
customer_as_id: 64496,
|
||||
provider_as_ids: vec![64497],
|
||||
}];
|
||||
let router_keys = vec![RouterKeyPayload {
|
||||
as_id: 64496,
|
||||
ski: vec![0x11; 20],
|
||||
@@ -215,9 +243,19 @@ mod tests {
|
||||
source_object_uri: "rsync://example.test/repo/router.cer".to_string(),
|
||||
source_object_hash: hex::encode([0x11; 32]),
|
||||
source_ee_cert_hash: hex::encode([0x11; 32]),
|
||||
item_effective_until: PackTime::from_utc_offset_datetime(time::OffsetDateTime::now_utc() + time::Duration::hours(1)),
|
||||
item_effective_until: PackTime::from_utc_offset_datetime(
|
||||
time::OffsetDateTime::now_utc() + time::Duration::hours(1),
|
||||
),
|
||||
}];
|
||||
let ccr = build_ccr_from_run(&store, &[trust_anchor], &vrps, &aspas, &router_keys, time::OffsetDateTime::now_utc()).expect("build ccr");
|
||||
let ccr = build_ccr_from_run(
|
||||
&store,
|
||||
&[trust_anchor],
|
||||
&vrps,
|
||||
&aspas,
|
||||
&router_keys,
|
||||
time::OffsetDateTime::now_utc(),
|
||||
)
|
||||
.expect("build ccr");
|
||||
assert!(ccr.mfts.is_some());
|
||||
assert!(ccr.vrps.is_some());
|
||||
assert!(ccr.vaps.is_some());
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
pub mod decode;
|
||||
pub mod encode;
|
||||
pub mod dump;
|
||||
pub mod hash;
|
||||
pub mod model;
|
||||
#[cfg(feature = "full")]
|
||||
pub mod build;
|
||||
pub mod decode;
|
||||
pub mod dump;
|
||||
pub mod encode;
|
||||
#[cfg(feature = "full")]
|
||||
pub mod export;
|
||||
pub mod hash;
|
||||
pub mod model;
|
||||
#[cfg(feature = "full")]
|
||||
pub mod verify;
|
||||
|
||||
@@ -16,16 +16,19 @@ pub use build::{
|
||||
build_roa_payload_state, build_trust_anchor_state,
|
||||
};
|
||||
pub use decode::{CcrDecodeError, decode_content_info};
|
||||
pub use dump::{CcrDumpError, dump_content_info_json, dump_content_info_json_value};
|
||||
pub use encode::{CcrEncodeError, encode_content_info};
|
||||
#[cfg(feature = "full")]
|
||||
pub use export::{CcrExportError, build_ccr_from_run, write_ccr_file};
|
||||
pub use dump::{CcrDumpError, dump_content_info_json, dump_content_info_json_value};
|
||||
#[cfg(feature = "full")]
|
||||
pub use verify::{CcrVerifyError, CcrVerifySummary, extract_vrp_rows, verify_against_report_json_path, verify_against_vcir_store, verify_against_vcir_store_path, verify_content_info, verify_content_info_bytes};
|
||||
pub use hash::{compute_state_hash, verify_state_hash};
|
||||
pub use model::{
|
||||
AspaPayloadSet, AspaPayloadState, CcrContentInfo, CcrDigestAlgorithm,
|
||||
ManifestInstance, ManifestState, RoaPayloadSet, RoaPayloadState,
|
||||
RouterKey, RouterKeySet, RouterKeyState, RpkiCanonicalCacheRepresentation,
|
||||
TrustAnchorState,
|
||||
AspaPayloadSet, AspaPayloadState, CcrContentInfo, CcrDigestAlgorithm, ManifestInstance,
|
||||
ManifestState, RoaPayloadSet, RoaPayloadState, RouterKey, RouterKeySet, RouterKeyState,
|
||||
RpkiCanonicalCacheRepresentation, TrustAnchorState,
|
||||
};
|
||||
#[cfg(feature = "full")]
|
||||
pub use verify::{
|
||||
CcrVerifyError, CcrVerifySummary, extract_vrp_rows, verify_against_report_json_path,
|
||||
verify_against_vcir_store, verify_against_vcir_store_path, verify_content_info,
|
||||
verify_content_info_bytes,
|
||||
};
|
||||
|
||||
@@ -133,16 +133,17 @@ impl ManifestInstance {
|
||||
));
|
||||
}
|
||||
validate_key_identifier("ManifestInstance.aki", &self.aki)?;
|
||||
validate_big_unsigned_bytes("ManifestInstance.manifest_number", &self.manifest_number.bytes_be)?;
|
||||
validate_big_unsigned_bytes(
|
||||
"ManifestInstance.manifest_number",
|
||||
&self.manifest_number.bytes_be,
|
||||
)?;
|
||||
if self.locations.is_empty() {
|
||||
return Err("ManifestInstance.locations must contain at least one AccessDescription".into());
|
||||
return Err(
|
||||
"ManifestInstance.locations must contain at least one AccessDescription".into(),
|
||||
);
|
||||
}
|
||||
for location in &self.locations {
|
||||
validate_full_der_with_tag(
|
||||
"ManifestInstance.locations[]",
|
||||
location,
|
||||
Some(0x30),
|
||||
)?;
|
||||
validate_full_der_with_tag("ManifestInstance.locations[]", location, Some(0x30))?;
|
||||
}
|
||||
if !self.subordinates.is_empty() {
|
||||
validate_sorted_unique_bytes(
|
||||
@@ -339,7 +340,9 @@ fn validate_big_unsigned_bytes(field: &str, bytes: &[u8]) -> Result<(), String>
|
||||
return Err(format!("{field} must not be empty"));
|
||||
}
|
||||
if bytes.len() > 1 && bytes[0] == 0x00 {
|
||||
return Err(format!("{field} must be minimally encoded as an unsigned integer"));
|
||||
return Err(format!(
|
||||
"{field} must be minimally encoded as an unsigned integer"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -82,7 +82,9 @@ pub fn verify_content_info_bytes(der: &[u8]) -> Result<CcrVerifySummary, CcrVeri
|
||||
verify_content_info(&content_info)
|
||||
}
|
||||
|
||||
pub fn verify_content_info(content_info: &CcrContentInfo) -> Result<CcrVerifySummary, CcrVerifyError> {
|
||||
pub fn verify_content_info(
|
||||
content_info: &CcrContentInfo,
|
||||
) -> Result<CcrVerifySummary, CcrVerifyError> {
|
||||
content_info.validate().map_err(CcrDecodeError::Validate)?;
|
||||
let state_hashes_ok = true;
|
||||
let mut manifest_instances = 0usize;
|
||||
@@ -108,7 +110,11 @@ pub fn verify_content_info(content_info: &CcrContentInfo) -> Result<CcrVerifySum
|
||||
return Err(CcrVerifyError::RoaHashMismatch);
|
||||
}
|
||||
roa_payload_sets = vrps.rps.len();
|
||||
roa_vrp_count = vrps.rps.iter().map(|set| count_roa_block_entries(&set.ip_addr_blocks)).sum();
|
||||
roa_vrp_count = vrps
|
||||
.rps
|
||||
.iter()
|
||||
.map(|set| count_roa_block_entries(&set.ip_addr_blocks))
|
||||
.sum();
|
||||
}
|
||||
if let Some(vaps) = &content_info.content.vaps {
|
||||
let payload_der = encode_aspa_payload_state_payload_der(&vaps.aps)
|
||||
@@ -154,10 +160,11 @@ pub fn verify_against_report_json_path(
|
||||
content_info: &CcrContentInfo,
|
||||
report_json_path: &Path,
|
||||
) -> Result<(), CcrVerifyError> {
|
||||
let bytes = std::fs::read(report_json_path)
|
||||
.map_err(|e| CcrVerifyError::ReportRead(report_json_path.display().to_string(), e.to_string()))?;
|
||||
let json: serde_json::Value = serde_json::from_slice(&bytes)
|
||||
.map_err(|e| CcrVerifyError::ReportParse(e.to_string()))?;
|
||||
let bytes = std::fs::read(report_json_path).map_err(|e| {
|
||||
CcrVerifyError::ReportRead(report_json_path.display().to_string(), e.to_string())
|
||||
})?;
|
||||
let json: serde_json::Value =
|
||||
serde_json::from_slice(&bytes).map_err(|e| CcrVerifyError::ReportParse(e.to_string()))?;
|
||||
|
||||
let report_vrps = report_vrp_keys(&json)?;
|
||||
let ccr_vrps = extract_vrp_rows(content_info)?;
|
||||
@@ -198,7 +205,9 @@ pub fn verify_against_vcir_store(
|
||||
let Some(mfts) = &content_info.content.mfts else {
|
||||
return Ok(());
|
||||
};
|
||||
let vcirs = store.list_vcirs().map_err(|e| CcrVerifyError::ListVcirs(e.to_string()))?;
|
||||
let vcirs = store
|
||||
.list_vcirs()
|
||||
.map_err(|e| CcrVerifyError::ListVcirs(e.to_string()))?;
|
||||
let mut vcir_hashes = BTreeSet::new();
|
||||
for vcir in vcirs {
|
||||
if let Some(artifact) = vcir.related_artifacts.iter().find(|artifact| {
|
||||
@@ -244,7 +253,9 @@ fn verify_router_key_state_hash(state: &RouterKeyState) -> Result<(), CcrVerifyE
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn report_vrp_keys(json: &serde_json::Value) -> Result<BTreeSet<(u32, String, u16)>, CcrVerifyError> {
|
||||
fn report_vrp_keys(
|
||||
json: &serde_json::Value,
|
||||
) -> Result<BTreeSet<(u32, String, u16)>, CcrVerifyError> {
|
||||
let mut out = BTreeSet::new();
|
||||
let Some(items) = json.get("vrps").and_then(|v| v.as_array()) else {
|
||||
return Ok(out);
|
||||
@@ -253,7 +264,8 @@ fn report_vrp_keys(json: &serde_json::Value) -> Result<BTreeSet<(u32, String, u1
|
||||
let asn = item
|
||||
.get("asn")
|
||||
.and_then(|v| v.as_u64())
|
||||
.ok_or_else(|| CcrVerifyError::ReportParse("vrps[].asn missing".into()))? as u32;
|
||||
.ok_or_else(|| CcrVerifyError::ReportParse("vrps[].asn missing".into()))?
|
||||
as u32;
|
||||
let prefix = item
|
||||
.get("prefix")
|
||||
.and_then(|v| v.as_str())
|
||||
@@ -285,7 +297,11 @@ fn report_aspa_keys(json: &serde_json::Value) -> Result<BTreeSet<(u32, Vec<u32>)
|
||||
.and_then(|v| v.as_array())
|
||||
.ok_or_else(|| CcrVerifyError::ReportParse("aspas[].provider_as_ids missing".into()))?
|
||||
.iter()
|
||||
.map(|v| v.as_u64().ok_or_else(|| CcrVerifyError::ReportParse("provider_as_ids[] invalid".into())).map(|v| v as u32))
|
||||
.map(|v| {
|
||||
v.as_u64()
|
||||
.ok_or_else(|| CcrVerifyError::ReportParse("provider_as_ids[] invalid".into()))
|
||||
.map(|v| v as u32)
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
providers.sort_unstable();
|
||||
providers.dedup();
|
||||
@@ -294,7 +310,9 @@ fn report_aspa_keys(json: &serde_json::Value) -> Result<BTreeSet<(u32, Vec<u32>)
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub fn extract_vrp_rows(content_info: &CcrContentInfo) -> Result<BTreeSet<(u32, String, u16)>, CcrVerifyError> {
|
||||
pub fn extract_vrp_rows(
|
||||
content_info: &CcrContentInfo,
|
||||
) -> Result<BTreeSet<(u32, String, u16)>, CcrVerifyError> {
|
||||
let mut out = BTreeSet::new();
|
||||
let Some(vrps) = &content_info.content.vrps else {
|
||||
return Ok(out);
|
||||
@@ -311,7 +329,9 @@ pub fn extract_vrp_rows(content_info: &CcrContentInfo) -> Result<BTreeSet<(u32,
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn ccr_aspa_keys(content_info: &CcrContentInfo) -> Result<BTreeSet<(u32, Vec<u32>)>, CcrVerifyError> {
|
||||
fn ccr_aspa_keys(
|
||||
content_info: &CcrContentInfo,
|
||||
) -> Result<BTreeSet<(u32, Vec<u32>)>, CcrVerifyError> {
|
||||
let mut out = BTreeSet::new();
|
||||
let Some(vaps) = &content_info.content.vaps else {
|
||||
return Ok(out);
|
||||
@@ -322,24 +342,43 @@ fn ccr_aspa_keys(content_info: &CcrContentInfo) -> Result<BTreeSet<(u32, Vec<u32
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn decode_roa_family_block(block: &[u8]) -> Result<(u16, Vec<(u8, Vec<u8>, Option<u16>)>), CcrVerifyError> {
|
||||
fn decode_roa_family_block(
|
||||
block: &[u8],
|
||||
) -> Result<(u16, Vec<(u8, Vec<u8>, Option<u16>)>), CcrVerifyError> {
|
||||
let mut top = crate::data_model::common::DerReader::new(block);
|
||||
let mut seq = top.take_sequence().map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))?;
|
||||
let mut seq = top
|
||||
.take_sequence()
|
||||
.map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))?;
|
||||
if !top.is_empty() {
|
||||
return Err(CcrVerifyError::Decode(CcrDecodeError::Parse("trailing bytes after ROAIPAddressFamily".into())));
|
||||
return Err(CcrVerifyError::Decode(CcrDecodeError::Parse(
|
||||
"trailing bytes after ROAIPAddressFamily".into(),
|
||||
)));
|
||||
}
|
||||
let afi_bytes = seq.take_octet_string().map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))?;
|
||||
let afi_bytes = seq
|
||||
.take_octet_string()
|
||||
.map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))?;
|
||||
let afi = u16::from_be_bytes([afi_bytes[0], afi_bytes[1]]);
|
||||
let mut addrs = seq.take_sequence().map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))?;
|
||||
let mut addrs = seq
|
||||
.take_sequence()
|
||||
.map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))?;
|
||||
let mut entries = Vec::new();
|
||||
while !addrs.is_empty() {
|
||||
let mut addr_seq = addrs.take_sequence().map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))?;
|
||||
let (unused_bits, content) = addr_seq.take_bit_string().map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))?;
|
||||
let mut addr_seq = addrs
|
||||
.take_sequence()
|
||||
.map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))?;
|
||||
let (unused_bits, content) = addr_seq
|
||||
.take_bit_string()
|
||||
.map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))?;
|
||||
let prefix_len = (content.len() * 8) as u8 - unused_bits;
|
||||
let max_len = if addr_seq.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(addr_seq.take_uint_u64().map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))? as u16)
|
||||
Some(
|
||||
addr_seq
|
||||
.take_uint_u64()
|
||||
.map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))?
|
||||
as u16,
|
||||
)
|
||||
};
|
||||
entries.push((prefix_len, content.to_vec(), max_len));
|
||||
}
|
||||
@@ -358,37 +397,43 @@ fn format_prefix(afi: u16, addr_bytes: &[u8], prefix_len: u8) -> Result<String,
|
||||
full[..addr_bytes.len()].copy_from_slice(addr_bytes);
|
||||
Ok(format!("{}/{prefix_len}", std::net::Ipv6Addr::from(full)))
|
||||
}
|
||||
other => Err(CcrVerifyError::Decode(CcrDecodeError::Parse(format!("unsupported AFI {other}")))),
|
||||
other => Err(CcrVerifyError::Decode(CcrDecodeError::Parse(format!(
|
||||
"unsupported AFI {other}"
|
||||
)))),
|
||||
}
|
||||
}
|
||||
|
||||
fn count_roa_block_entries(blocks: &[Vec<u8>]) -> usize {
|
||||
blocks
|
||||
.iter()
|
||||
.map(|block| decode_roa_family_block(block).map(|(_, entries)| entries.len()).unwrap_or(0))
|
||||
.map(|block| {
|
||||
decode_roa_family_block(block)
|
||||
.map(|(_, entries)| entries.len())
|
||||
.unwrap_or(0)
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::ccr::build::{build_aspa_payload_state, build_roa_payload_state};
|
||||
use crate::ccr::encode::{
|
||||
encode_manifest_state_payload_der, encode_roa_payload_state_payload_der,
|
||||
encode_router_key_state_payload_der, encode_trust_anchor_state_payload_der,
|
||||
};
|
||||
use crate::ccr::build::{build_aspa_payload_state, build_roa_payload_state};
|
||||
use crate::ccr::model::{
|
||||
CcrDigestAlgorithm, ManifestInstance, ManifestState,
|
||||
RouterKey, RouterKeySet, RpkiCanonicalCacheRepresentation,
|
||||
CcrDigestAlgorithm, ManifestInstance, ManifestState, RouterKey, RouterKeySet,
|
||||
RpkiCanonicalCacheRepresentation,
|
||||
};
|
||||
use crate::data_model::roa::{IpPrefix, RoaAfi};
|
||||
use crate::validation::objects::{AspaAttestation, Vrp};
|
||||
use crate::data_model::common::BigUnsigned;
|
||||
use crate::data_model::roa::{IpPrefix, RoaAfi};
|
||||
use crate::storage::{
|
||||
PackTime, ValidatedCaInstanceResult, ValidatedManifestMeta, VcirArtifactKind,
|
||||
VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, VcirChildEntry,
|
||||
VcirInstanceGate, VcirRelatedArtifact, VcirSummary,
|
||||
};
|
||||
use crate::validation::objects::{AspaAttestation, Vrp};
|
||||
|
||||
fn sample_time() -> time::OffsetDateTime {
|
||||
time::OffsetDateTime::parse(
|
||||
@@ -415,21 +460,37 @@ mod tests {
|
||||
};
|
||||
let vrps = build_roa_payload_state(&[Vrp {
|
||||
asn: 64496,
|
||||
prefix: IpPrefix { afi: RoaAfi::Ipv4, prefix_len: 0, addr: [0; 16] },
|
||||
prefix: IpPrefix {
|
||||
afi: RoaAfi::Ipv4,
|
||||
prefix_len: 0,
|
||||
addr: [0; 16],
|
||||
},
|
||||
max_length: 0,
|
||||
}]).expect("build roa state");
|
||||
}])
|
||||
.expect("build roa state");
|
||||
let vaps = build_aspa_payload_state(&[AspaAttestation {
|
||||
customer_as_id: 64496,
|
||||
provider_as_ids: vec![64497],
|
||||
}]).expect("build aspa state");
|
||||
}])
|
||||
.expect("build aspa state");
|
||||
let skis = vec![vec![0x11; 20]];
|
||||
let tas = TrustAnchorState {
|
||||
hash: crate::ccr::compute_state_hash(&encode_trust_anchor_state_payload_der(&skis).unwrap()),
|
||||
hash: crate::ccr::compute_state_hash(
|
||||
&encode_trust_anchor_state_payload_der(&skis).unwrap(),
|
||||
),
|
||||
skis,
|
||||
};
|
||||
let rksets = vec![RouterKeySet { as_id: 64496, router_keys: vec![RouterKey { ski: vec![0x22;20], spki_der: vec![0x30,0x00] }] }];
|
||||
let rksets = vec![RouterKeySet {
|
||||
as_id: 64496,
|
||||
router_keys: vec![RouterKey {
|
||||
ski: vec![0x22; 20],
|
||||
spki_der: vec![0x30, 0x00],
|
||||
}],
|
||||
}];
|
||||
let rks = RouterKeyState {
|
||||
hash: crate::ccr::compute_state_hash(&encode_router_key_state_payload_der(&rksets).unwrap()),
|
||||
hash: crate::ccr::compute_state_hash(
|
||||
&encode_router_key_state_payload_der(&rksets).unwrap(),
|
||||
),
|
||||
rksets,
|
||||
};
|
||||
CcrContentInfo::new(RpkiCanonicalCacheRepresentation {
|
||||
@@ -448,19 +509,31 @@ mod tests {
|
||||
fn verify_detects_each_state_hash_mismatch() {
|
||||
let mut ci = sample_content_info();
|
||||
ci.content.vrps.as_mut().unwrap().hash[0] ^= 0x01;
|
||||
assert!(matches!(verify_content_info(&ci), Err(CcrVerifyError::RoaHashMismatch)));
|
||||
assert!(matches!(
|
||||
verify_content_info(&ci),
|
||||
Err(CcrVerifyError::RoaHashMismatch)
|
||||
));
|
||||
|
||||
let mut ci = sample_content_info();
|
||||
ci.content.vaps.as_mut().unwrap().hash[0] ^= 0x01;
|
||||
assert!(matches!(verify_content_info(&ci), Err(CcrVerifyError::AspaHashMismatch)));
|
||||
assert!(matches!(
|
||||
verify_content_info(&ci),
|
||||
Err(CcrVerifyError::AspaHashMismatch)
|
||||
));
|
||||
|
||||
let mut ci = sample_content_info();
|
||||
ci.content.tas.as_mut().unwrap().hash[0] ^= 0x01;
|
||||
assert!(matches!(verify_content_info(&ci), Err(CcrVerifyError::TrustAnchorHashMismatch)));
|
||||
assert!(matches!(
|
||||
verify_content_info(&ci),
|
||||
Err(CcrVerifyError::TrustAnchorHashMismatch)
|
||||
));
|
||||
|
||||
let mut ci = sample_content_info();
|
||||
ci.content.rks.as_mut().unwrap().hash[0] ^= 0x01;
|
||||
assert!(matches!(verify_content_info(&ci), Err(CcrVerifyError::RouterKeyHashMismatch)));
|
||||
assert!(matches!(
|
||||
verify_content_info(&ci),
|
||||
Err(CcrVerifyError::RouterKeyHashMismatch)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -472,20 +545,28 @@ mod tests {
|
||||
});
|
||||
let report_path = td.path().join("report.json");
|
||||
std::fs::write(&report_path, serde_json::to_vec(&report).unwrap()).unwrap();
|
||||
verify_against_report_json_path(&sample_content_info(), &report_path).expect("matching report");
|
||||
verify_against_report_json_path(&sample_content_info(), &report_path)
|
||||
.expect("matching report");
|
||||
|
||||
let bad_path = td.path().join("bad.json");
|
||||
std::fs::write(&bad_path, b"not-json").unwrap();
|
||||
assert!(matches!(verify_against_report_json_path(&sample_content_info(), &bad_path), Err(CcrVerifyError::ReportParse(_))));
|
||||
assert!(matches!(
|
||||
verify_against_report_json_path(&sample_content_info(), &bad_path),
|
||||
Err(CcrVerifyError::ReportParse(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_against_report_json_rejects_missing_fields_and_aspa_mismatch() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let missing = serde_json::json!({"vrps":[{"prefix":"0.0.0.0/0","max_length":0}],"aspas":[]});
|
||||
let missing =
|
||||
serde_json::json!({"vrps":[{"prefix":"0.0.0.0/0","max_length":0}],"aspas":[]});
|
||||
let missing_path = td.path().join("missing.json");
|
||||
std::fs::write(&missing_path, serde_json::to_vec(&missing).unwrap()).unwrap();
|
||||
assert!(matches!(verify_against_report_json_path(&sample_content_info(), &missing_path), Err(CcrVerifyError::ReportParse(_))));
|
||||
assert!(matches!(
|
||||
verify_against_report_json_path(&sample_content_info(), &missing_path),
|
||||
Err(CcrVerifyError::ReportParse(_))
|
||||
));
|
||||
|
||||
let mismatch = serde_json::json!({
|
||||
"vrps": [{"asn": 64496, "prefix": "0.0.0.0/0", "max_length": 0}],
|
||||
@@ -493,7 +574,10 @@ mod tests {
|
||||
});
|
||||
let mismatch_path = td.path().join("mismatch.json");
|
||||
std::fs::write(&mismatch_path, serde_json::to_vec(&mismatch).unwrap()).unwrap();
|
||||
assert!(matches!(verify_against_report_json_path(&sample_content_info(), &mismatch_path), Err(CcrVerifyError::ReportAspaMismatch { .. })));
|
||||
assert!(matches!(
|
||||
verify_against_report_json_path(&sample_content_info(), &mismatch_path),
|
||||
Err(CcrVerifyError::ReportAspaMismatch { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -542,11 +626,26 @@ mod tests {
|
||||
object_type: Some("mft".to_string()),
|
||||
validation_status: VcirArtifactValidationStatus::Accepted,
|
||||
}],
|
||||
summary: VcirSummary { local_vrp_count: 0, local_aspa_count: 0, local_router_key_count: 0, child_count: 1, accepted_object_count: 1, rejected_object_count: 0 },
|
||||
audit_summary: VcirAuditSummary { failed_fetch_eligible: true, last_failed_fetch_reason: None, warning_count: 0, audit_flags: Vec::new() },
|
||||
summary: VcirSummary {
|
||||
local_vrp_count: 0,
|
||||
local_aspa_count: 0,
|
||||
local_router_key_count: 0,
|
||||
child_count: 1,
|
||||
accepted_object_count: 1,
|
||||
rejected_object_count: 0,
|
||||
},
|
||||
audit_summary: VcirAuditSummary {
|
||||
failed_fetch_eligible: true,
|
||||
last_failed_fetch_reason: None,
|
||||
warning_count: 0,
|
||||
audit_flags: Vec::new(),
|
||||
},
|
||||
};
|
||||
store.put_vcir(&vcir).unwrap();
|
||||
assert!(matches!(verify_against_vcir_store(&sample_content_info(), &store), Err(CcrVerifyError::VcirManifestMismatch { .. })));
|
||||
assert!(matches!(
|
||||
verify_against_vcir_store(&sample_content_info(), &store),
|
||||
Err(CcrVerifyError::VcirManifestMismatch { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -557,12 +656,27 @@ mod tests {
|
||||
hash_alg: CcrDigestAlgorithm::Sha256,
|
||||
produced_at: sample_time(),
|
||||
mfts: None,
|
||||
vrps: Some(crate::ccr::model::RoaPayloadState { rps: vec![crate::ccr::model::RoaPayloadSet { as_id: 64496, ip_addr_blocks: vec![block.clone()] }], hash: crate::ccr::compute_state_hash(&encode_roa_payload_state_payload_der(&[crate::ccr::model::RoaPayloadSet { as_id: 64496, ip_addr_blocks: vec![block] }]).unwrap()) }),
|
||||
vrps: Some(crate::ccr::model::RoaPayloadState {
|
||||
rps: vec![crate::ccr::model::RoaPayloadSet {
|
||||
as_id: 64496,
|
||||
ip_addr_blocks: vec![block.clone()],
|
||||
}],
|
||||
hash: crate::ccr::compute_state_hash(
|
||||
&encode_roa_payload_state_payload_der(&[crate::ccr::model::RoaPayloadSet {
|
||||
as_id: 64496,
|
||||
ip_addr_blocks: vec![block],
|
||||
}])
|
||||
.unwrap(),
|
||||
),
|
||||
}),
|
||||
vaps: None,
|
||||
tas: None,
|
||||
rks: None,
|
||||
});
|
||||
assert!(matches!(extract_vrp_rows(&ci), Err(CcrVerifyError::Decode(_))));
|
||||
assert!(matches!(
|
||||
extract_vrp_rows(&ci),
|
||||
Err(CcrVerifyError::Decode(_))
|
||||
));
|
||||
let bad_count = count_roa_block_entries(&[vec![0x04, 0x00]]);
|
||||
assert_eq!(bad_count, 0);
|
||||
}
|
||||
|
||||
@@ -14,7 +14,10 @@ pub enum CirDecodeError {
|
||||
UnexpectedVersion { expected: u32, actual: u32 },
|
||||
|
||||
#[error("unexpected digest algorithm OID: expected {expected}, got {actual}")]
|
||||
UnexpectedDigestAlgorithm { expected: &'static str, actual: String },
|
||||
UnexpectedDigestAlgorithm {
|
||||
expected: &'static str,
|
||||
actual: String,
|
||||
},
|
||||
|
||||
#[error("CIR model validation failed after decode: {0}")]
|
||||
Validate(String),
|
||||
@@ -85,12 +88,17 @@ fn decode_object(der: &[u8]) -> Result<CirObject, CirDecodeError> {
|
||||
let mut top = DerReader::new(der);
|
||||
let mut seq = top.take_sequence().map_err(CirDecodeError::Parse)?;
|
||||
if !top.is_empty() {
|
||||
return Err(CirDecodeError::Parse("trailing bytes after CirObject".into()));
|
||||
return Err(CirDecodeError::Parse(
|
||||
"trailing bytes after CirObject".into(),
|
||||
));
|
||||
}
|
||||
let rsync_uri = std::str::from_utf8(seq.take_tag(0x16).map_err(CirDecodeError::Parse)?)
|
||||
.map_err(|e| CirDecodeError::Parse(e.to_string()))?
|
||||
.to_string();
|
||||
let sha256 = seq.take_octet_string().map_err(CirDecodeError::Parse)?.to_vec();
|
||||
let sha256 = seq
|
||||
.take_octet_string()
|
||||
.map_err(CirDecodeError::Parse)?
|
||||
.to_vec();
|
||||
if !seq.is_empty() {
|
||||
return Err(CirDecodeError::Parse("trailing fields in CirObject".into()));
|
||||
}
|
||||
@@ -106,7 +114,10 @@ fn decode_tal(der: &[u8]) -> Result<CirTal, CirDecodeError> {
|
||||
let tal_uri = std::str::from_utf8(seq.take_tag(0x16).map_err(CirDecodeError::Parse)?)
|
||||
.map_err(|e| CirDecodeError::Parse(e.to_string()))?
|
||||
.to_string();
|
||||
let tal_bytes = seq.take_octet_string().map_err(CirDecodeError::Parse)?.to_vec();
|
||||
let tal_bytes = seq
|
||||
.take_octet_string()
|
||||
.map_err(CirDecodeError::Parse)?
|
||||
.to_vec();
|
||||
if !seq.is_empty() {
|
||||
return Err(CirDecodeError::Parse("trailing fields in CirTal".into()));
|
||||
}
|
||||
@@ -150,12 +161,10 @@ fn parse_generalized_time(bytes: &[u8]) -> Result<time::OffsetDateTime, CirDecod
|
||||
let hour = parse(8..10)? as u8;
|
||||
let minute = parse(10..12)? as u8;
|
||||
let second = parse(12..14)? as u8;
|
||||
let month = time::Month::try_from(month)
|
||||
.map_err(|e| CirDecodeError::Parse(e.to_string()))?;
|
||||
let month = time::Month::try_from(month).map_err(|e| CirDecodeError::Parse(e.to_string()))?;
|
||||
let date = time::Date::from_calendar_date(year, month, day)
|
||||
.map_err(|e| CirDecodeError::Parse(e.to_string()))?;
|
||||
let timev = time::Time::from_hms(hour, minute, second)
|
||||
.map_err(|e| CirDecodeError::Parse(e.to_string()))?;
|
||||
Ok(time::PrimitiveDateTime::new(date, timev).assume_utc())
|
||||
}
|
||||
|
||||
|
||||
@@ -9,9 +9,7 @@ pub enum CirEncodeError {
|
||||
Validate(String),
|
||||
}
|
||||
|
||||
pub fn encode_cir(
|
||||
cir: &CanonicalInputRepresentation,
|
||||
) -> Result<Vec<u8>, CirEncodeError> {
|
||||
pub fn encode_cir(cir: &CanonicalInputRepresentation) -> Result<Vec<u8>, CirEncodeError> {
|
||||
cir.validate().map_err(CirEncodeError::Validate)?;
|
||||
Ok(encode_sequence(&[
|
||||
encode_integer_u32(cir.version),
|
||||
@@ -134,7 +132,10 @@ fn encode_len_into(len: usize, out: &mut Vec<u8>) {
|
||||
return;
|
||||
}
|
||||
let bytes = len.to_be_bytes();
|
||||
let first_non_zero = bytes.iter().position(|&b| b != 0).unwrap_or(bytes.len() - 1);
|
||||
let first_non_zero = bytes
|
||||
.iter()
|
||||
.position(|&b| b != 0)
|
||||
.unwrap_or(bytes.len() - 1);
|
||||
let len_bytes = &bytes[first_non_zero..];
|
||||
out.push(0x80 | (len_bytes.len() as u8));
|
||||
out.extend_from_slice(len_bytes);
|
||||
@@ -144,4 +145,3 @@ fn encode_len_into(len: usize, out: &mut Vec<u8>) {
|
||||
const _: () = {
|
||||
let _ = CIR_VERSION_V1;
|
||||
};
|
||||
|
||||
|
||||
@@ -3,13 +3,14 @@ use std::collections::BTreeSet;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::audit::{AuditObjectResult, PublicationPointAudit};
|
||||
use crate::blob_store::RawObjectStore;
|
||||
use crate::cir::encode::{CirEncodeError, encode_cir};
|
||||
use crate::cir::model::{
|
||||
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal,
|
||||
};
|
||||
use crate::cir::static_pool::{
|
||||
CirStaticPoolError, CirStaticPoolExportSummary, write_bytes_to_static_pool,
|
||||
export_hashes_from_store,
|
||||
CirStaticPoolError, CirStaticPoolExportSummary, export_hashes_from_store,
|
||||
write_bytes_to_static_pool,
|
||||
};
|
||||
use crate::data_model::ta::TrustAnchor;
|
||||
use crate::storage::{RepositoryViewState, RocksStore};
|
||||
@@ -46,11 +47,24 @@ pub enum CirExportError {
|
||||
Write(String, String),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum CirObjectPoolExportSummary {
|
||||
Static(CirStaticPoolExportSummary),
|
||||
RawStore(CirRawStoreExportSummary),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct CirRawStoreExportSummary {
|
||||
pub unique_hashes: usize,
|
||||
pub written_entries: usize,
|
||||
pub reused_entries: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct CirExportSummary {
|
||||
pub object_count: usize,
|
||||
pub tal_count: usize,
|
||||
pub static_pool: CirStaticPoolExportSummary,
|
||||
pub object_pool: CirObjectPoolExportSummary,
|
||||
pub timing: CirExportTiming,
|
||||
}
|
||||
|
||||
@@ -132,7 +146,10 @@ pub fn build_cir_from_run(
|
||||
Ok(cir)
|
||||
}
|
||||
|
||||
pub fn write_cir_file(path: &Path, cir: &CanonicalInputRepresentation) -> Result<(), CirExportError> {
|
||||
pub fn write_cir_file(
|
||||
path: &Path,
|
||||
cir: &CanonicalInputRepresentation,
|
||||
) -> Result<(), CirExportError> {
|
||||
let der = encode_cir(cir)?;
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)
|
||||
@@ -175,6 +192,58 @@ pub fn export_cir_static_pool(
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
pub fn export_cir_raw_store(
|
||||
store: &RocksStore,
|
||||
raw_store_path: &Path,
|
||||
cir: &CanonicalInputRepresentation,
|
||||
trust_anchor: &TrustAnchor,
|
||||
) -> Result<CirRawStoreExportSummary, CirExportError> {
|
||||
let ta_hash = ta_sha256_hex(&trust_anchor.ta_certificate.raw_der);
|
||||
let unique: BTreeSet<String> = cir
|
||||
.objects
|
||||
.iter()
|
||||
.map(|item| hex::encode(&item.sha256))
|
||||
.collect();
|
||||
|
||||
let mut written_entries = 0usize;
|
||||
let mut reused_entries = 0usize;
|
||||
for sha256_hex in &unique {
|
||||
if store
|
||||
.get_raw_entry(sha256_hex)
|
||||
.map_err(|e| CirExportError::Write(raw_store_path.display().to_string(), e.to_string()))?
|
||||
.is_some()
|
||||
{
|
||||
reused_entries += 1;
|
||||
continue;
|
||||
}
|
||||
if sha256_hex == &ta_hash {
|
||||
let mut entry =
|
||||
crate::storage::RawByHashEntry::from_bytes(ta_hash.clone(), trust_anchor.ta_certificate.raw_der.clone());
|
||||
entry.object_type = Some("cer".to_string());
|
||||
for object in &cir.objects {
|
||||
if hex::encode(&object.sha256) == ta_hash {
|
||||
entry.origin_uris.push(object.rsync_uri.clone());
|
||||
}
|
||||
}
|
||||
store
|
||||
.put_raw_by_hash_entry(&entry)
|
||||
.map_err(|e| CirExportError::Write(raw_store_path.display().to_string(), e.to_string()))?;
|
||||
written_entries += 1;
|
||||
continue;
|
||||
}
|
||||
return Err(CirExportError::Write(
|
||||
raw_store_path.display().to_string(),
|
||||
format!("raw store missing object for sha256={sha256_hex}"),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(CirRawStoreExportSummary {
|
||||
unique_hashes: unique.len(),
|
||||
written_entries,
|
||||
reused_entries,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn export_cir_from_run(
|
||||
store: &RocksStore,
|
||||
trust_anchor: &TrustAnchor,
|
||||
@@ -182,9 +251,19 @@ pub fn export_cir_from_run(
|
||||
validation_time: time::OffsetDateTime,
|
||||
publication_points: &[PublicationPointAudit],
|
||||
cir_out: &Path,
|
||||
static_root: &Path,
|
||||
static_root: Option<&Path>,
|
||||
raw_store_path: Option<&Path>,
|
||||
capture_date_utc: time::Date,
|
||||
) -> Result<CirExportSummary, CirExportError> {
|
||||
let backend_count = static_root.is_some() as u8 + raw_store_path.is_some() as u8;
|
||||
match backend_count {
|
||||
1 => {}
|
||||
_ => {
|
||||
return Err(CirExportError::Validate(
|
||||
"must specify exactly one CIR object pool backend".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let total_started = std::time::Instant::now();
|
||||
|
||||
let started = std::time::Instant::now();
|
||||
@@ -198,7 +277,22 @@ pub fn export_cir_from_run(
|
||||
let build_cir_ms = started.elapsed().as_millis() as u64;
|
||||
|
||||
let started = std::time::Instant::now();
|
||||
let static_pool = export_cir_static_pool(store, static_root, capture_date_utc, &cir, trust_anchor)?;
|
||||
let object_pool = if let Some(static_root) = static_root {
|
||||
CirObjectPoolExportSummary::Static(export_cir_static_pool(
|
||||
store,
|
||||
static_root,
|
||||
capture_date_utc,
|
||||
&cir,
|
||||
trust_anchor,
|
||||
)?)
|
||||
} else {
|
||||
CirObjectPoolExportSummary::RawStore(export_cir_raw_store(
|
||||
store,
|
||||
raw_store_path.expect("validated"),
|
||||
&cir,
|
||||
trust_anchor,
|
||||
)?)
|
||||
};
|
||||
let static_pool_ms = started.elapsed().as_millis() as u64;
|
||||
|
||||
let started = std::time::Instant::now();
|
||||
@@ -208,7 +302,7 @@ pub fn export_cir_from_run(
|
||||
Ok(CirExportSummary {
|
||||
object_count: cir.objects.len(),
|
||||
tal_count: cir.tals.len(),
|
||||
static_pool,
|
||||
object_pool,
|
||||
timing: CirExportTiming {
|
||||
build_cir_ms,
|
||||
static_pool_ms,
|
||||
@@ -246,7 +340,8 @@ mod tests {
|
||||
|
||||
fn sample_trust_anchor() -> TrustAnchor {
|
||||
let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
|
||||
let tal_bytes = std::fs::read(base.join("tests/fixtures/tal/apnic-rfc7730-https.tal")).unwrap();
|
||||
let tal_bytes =
|
||||
std::fs::read(base.join("tests/fixtures/tal/apnic-rfc7730-https.tal")).unwrap();
|
||||
let ta_der = std::fs::read(base.join("tests/fixtures/ta/apnic-ta.cer")).unwrap();
|
||||
let tal = Tal::decode_bytes(&tal_bytes).unwrap();
|
||||
TrustAnchor::bind_der(tal, &ta_der, None).unwrap()
|
||||
@@ -264,7 +359,8 @@ mod tests {
|
||||
let bytes = b"object-a".to_vec();
|
||||
let hash = sha256_hex(&bytes);
|
||||
let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone());
|
||||
raw.origin_uris.push("rsync://example.test/repo/a.cer".into());
|
||||
raw.origin_uris
|
||||
.push("rsync://example.test/repo/a.cer".into());
|
||||
store.put_raw_by_hash_entry(&raw).unwrap();
|
||||
store
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
@@ -284,18 +380,20 @@ mod tests {
|
||||
sample_time(),
|
||||
&[],
|
||||
)
|
||||
.expect("build cir");
|
||||
.expect("build cir");
|
||||
assert_eq!(cir.version, CIR_VERSION_V1);
|
||||
assert_eq!(cir.tals.len(), 1);
|
||||
assert_eq!(cir.tals[0].tal_uri, "https://example.test/root.tal");
|
||||
assert!(cir
|
||||
.objects
|
||||
.iter()
|
||||
.any(|item| item.rsync_uri == "rsync://example.test/repo/a.cer"));
|
||||
assert!(cir
|
||||
.objects
|
||||
.iter()
|
||||
.any(|item| item.rsync_uri.contains("apnic-rpki-root-iana-origin.cer")));
|
||||
assert!(
|
||||
cir.objects
|
||||
.iter()
|
||||
.any(|item| item.rsync_uri == "rsync://example.test/repo/a.cer")
|
||||
);
|
||||
assert!(
|
||||
cir.objects
|
||||
.iter()
|
||||
.any(|item| item.rsync_uri.contains("apnic-rpki-root-iana-origin.cer"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -309,7 +407,8 @@ mod tests {
|
||||
let bytes = b"object-b".to_vec();
|
||||
let hash = sha256_hex(&bytes);
|
||||
let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone());
|
||||
raw.origin_uris.push("rsync://example.test/repo/b.roa".into());
|
||||
raw.origin_uris
|
||||
.push("rsync://example.test/repo/b.roa".into());
|
||||
store.put_raw_by_hash_entry(&raw).unwrap();
|
||||
store
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
@@ -330,12 +429,17 @@ mod tests {
|
||||
sample_time(),
|
||||
&[],
|
||||
&cir_path,
|
||||
&static_root,
|
||||
Some(&static_root),
|
||||
None,
|
||||
sample_date(),
|
||||
)
|
||||
.expect("export cir");
|
||||
assert_eq!(summary.tal_count, 1);
|
||||
assert!(summary.object_count >= 2);
|
||||
match summary.object_pool {
|
||||
CirObjectPoolExportSummary::Static(_) => {}
|
||||
other => panic!("unexpected backend: {other:?}"),
|
||||
}
|
||||
|
||||
let der = std::fs::read(&cir_path).unwrap();
|
||||
let cir = decode_cir(&der).unwrap();
|
||||
@@ -345,6 +449,51 @@ mod tests {
|
||||
assert_eq!(std::fs::read(object_path).unwrap(), bytes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_cir_from_run_uses_raw_store_backend_without_pool_export() {
|
||||
let td = tempfile::tempdir().unwrap();
|
||||
let store_dir = td.path().join("db");
|
||||
let raw_store = td.path().join("raw-store.db");
|
||||
let out_dir = td.path().join("out");
|
||||
let store = RocksStore::open_with_external_raw_store(&store_dir, &raw_store).unwrap();
|
||||
|
||||
let bytes = b"object-d".to_vec();
|
||||
let hash = sha256_hex(&bytes);
|
||||
let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone());
|
||||
raw.origin_uris
|
||||
.push("rsync://example.test/repo/d.roa".into());
|
||||
store.put_raw_by_hash_entry(&raw).unwrap();
|
||||
store
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
rsync_uri: "rsync://example.test/repo/d.roa".to_string(),
|
||||
current_hash: Some(hash.clone()),
|
||||
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let ta = sample_trust_anchor();
|
||||
let cir_path = out_dir.join("example.cir");
|
||||
let summary = export_cir_from_run(
|
||||
&store,
|
||||
&ta,
|
||||
"https://example.test/root.tal",
|
||||
sample_time(),
|
||||
&[],
|
||||
&cir_path,
|
||||
None,
|
||||
Some(&raw_store),
|
||||
sample_date(),
|
||||
)
|
||||
.expect("export cir to raw store");
|
||||
match summary.object_pool {
|
||||
CirObjectPoolExportSummary::RawStore(ref s) => assert!(s.unique_hashes >= 2),
|
||||
other => panic!("unexpected backend: {other:?}"),
|
||||
}
|
||||
assert!(raw_store.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_cir_from_run_includes_vcir_current_instance_objects_from_audit() {
|
||||
let td = tempfile::tempdir().unwrap();
|
||||
@@ -357,14 +506,16 @@ mod tests {
|
||||
};
|
||||
pp.objects.push(crate::audit::ObjectAuditEntry {
|
||||
rsync_uri: "rsync://example.test/repo/fallback.mft".to_string(),
|
||||
sha256_hex: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string(),
|
||||
sha256_hex: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
.to_string(),
|
||||
kind: crate::audit::AuditObjectKind::Manifest,
|
||||
result: crate::audit::AuditObjectResult::Ok,
|
||||
detail: None,
|
||||
});
|
||||
pp.objects.push(crate::audit::ObjectAuditEntry {
|
||||
rsync_uri: "rsync://example.test/repo/fallback.roa".to_string(),
|
||||
sha256_hex: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(),
|
||||
sha256_hex: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
.to_string(),
|
||||
kind: crate::audit::AuditObjectKind::Roa,
|
||||
result: crate::audit::AuditObjectResult::Ok,
|
||||
detail: None,
|
||||
@@ -379,13 +530,15 @@ mod tests {
|
||||
)
|
||||
.expect("build cir");
|
||||
|
||||
assert!(cir
|
||||
.objects
|
||||
.iter()
|
||||
.any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.mft"));
|
||||
assert!(cir
|
||||
.objects
|
||||
.iter()
|
||||
.any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.roa"));
|
||||
assert!(
|
||||
cir.objects
|
||||
.iter()
|
||||
.any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.mft")
|
||||
);
|
||||
assert!(
|
||||
cir.objects
|
||||
.iter()
|
||||
.any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.roa")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::blob_store::{ExternalRawStoreDb, RawObjectStore};
|
||||
use crate::cir::model::CanonicalInputRepresentation;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
@@ -27,13 +28,30 @@ pub enum CirMaterializeError {
|
||||
MissingStaticObject { sha256_hex: String },
|
||||
|
||||
#[error("link target failed: {src} -> {dst}: {detail}")]
|
||||
Link { src: String, dst: String, detail: String },
|
||||
Link {
|
||||
src: String,
|
||||
dst: String,
|
||||
detail: String,
|
||||
},
|
||||
|
||||
#[error("copy target failed: {src} -> {dst}: {detail}")]
|
||||
Copy { src: String, dst: String, detail: String },
|
||||
Copy {
|
||||
src: String,
|
||||
dst: String,
|
||||
detail: String,
|
||||
},
|
||||
|
||||
#[error("mirror tree mismatch after materialize: {0}")]
|
||||
TreeMismatch(String),
|
||||
|
||||
#[error("open raw store failed: {path}: {detail}")]
|
||||
OpenRawStore { path: String, detail: String },
|
||||
|
||||
#[error("raw object not found for sha256={sha256_hex}")]
|
||||
MissingRawStoreObject { sha256_hex: String },
|
||||
|
||||
#[error("read raw store failed for sha256={sha256_hex}: {detail}")]
|
||||
ReadRawStore { sha256_hex: String, detail: String },
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
@@ -49,8 +67,7 @@ pub fn materialize_cir(
|
||||
mirror_root: &Path,
|
||||
clean_rebuild: bool,
|
||||
) -> Result<CirMaterializeSummary, CirMaterializeError> {
|
||||
cir.validate()
|
||||
.map_err(CirMaterializeError::TreeMismatch)?;
|
||||
cir.validate().map_err(CirMaterializeError::TreeMismatch)?;
|
||||
|
||||
if clean_rebuild && mirror_root.exists() {
|
||||
fs::remove_dir_all(mirror_root).map_err(|e| CirMaterializeError::RemoveMirrorRoot {
|
||||
@@ -120,6 +137,89 @@ pub fn materialize_cir(
|
||||
})
|
||||
}
|
||||
|
||||
pub fn materialize_cir_from_raw_store(
|
||||
cir: &CanonicalInputRepresentation,
|
||||
raw_store_db: &Path,
|
||||
mirror_root: &Path,
|
||||
clean_rebuild: bool,
|
||||
) -> Result<CirMaterializeSummary, CirMaterializeError> {
|
||||
cir.validate().map_err(CirMaterializeError::TreeMismatch)?;
|
||||
|
||||
if clean_rebuild && mirror_root.exists() {
|
||||
fs::remove_dir_all(mirror_root).map_err(|e| CirMaterializeError::RemoveMirrorRoot {
|
||||
path: mirror_root.display().to_string(),
|
||||
detail: e.to_string(),
|
||||
})?;
|
||||
}
|
||||
fs::create_dir_all(mirror_root).map_err(|e| CirMaterializeError::CreateMirrorRoot {
|
||||
path: mirror_root.display().to_string(),
|
||||
detail: e.to_string(),
|
||||
})?;
|
||||
|
||||
let raw_store =
|
||||
ExternalRawStoreDb::open(raw_store_db).map_err(|e| CirMaterializeError::OpenRawStore {
|
||||
path: raw_store_db.display().to_string(),
|
||||
detail: e.to_string(),
|
||||
})?;
|
||||
|
||||
let mut copied_files = 0usize;
|
||||
for object in &cir.objects {
|
||||
let sha256_hex = hex::encode(&object.sha256);
|
||||
let entry = raw_store
|
||||
.get_raw_entry(&sha256_hex)
|
||||
.map_err(|e| CirMaterializeError::ReadRawStore {
|
||||
sha256_hex: sha256_hex.clone(),
|
||||
detail: e.to_string(),
|
||||
})?
|
||||
.ok_or_else(|| CirMaterializeError::MissingRawStoreObject {
|
||||
sha256_hex: sha256_hex.clone(),
|
||||
})?;
|
||||
let relative = mirror_relative_path_for_rsync_uri(&object.rsync_uri)?;
|
||||
let target = mirror_root.join(&relative);
|
||||
|
||||
if let Some(parent) = target.parent() {
|
||||
fs::create_dir_all(parent).map_err(|e| CirMaterializeError::CreateParent {
|
||||
path: parent.display().to_string(),
|
||||
detail: e.to_string(),
|
||||
})?;
|
||||
}
|
||||
|
||||
if target.exists() {
|
||||
fs::remove_file(&target).map_err(|e| CirMaterializeError::RemoveExistingTarget {
|
||||
path: target.display().to_string(),
|
||||
detail: e.to_string(),
|
||||
})?;
|
||||
}
|
||||
|
||||
fs::write(&target, &entry.bytes).map_err(|e| CirMaterializeError::Copy {
|
||||
src: raw_store_db.display().to_string(),
|
||||
dst: target.display().to_string(),
|
||||
detail: e.to_string(),
|
||||
})?;
|
||||
copied_files += 1;
|
||||
}
|
||||
|
||||
let actual = collect_materialized_uris(mirror_root)?;
|
||||
let expected = cir
|
||||
.objects
|
||||
.iter()
|
||||
.map(|item| item.rsync_uri.clone())
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
if actual != expected {
|
||||
return Err(CirMaterializeError::TreeMismatch(format!(
|
||||
"expected {} files, got {} files",
|
||||
expected.len(),
|
||||
actual.len()
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(CirMaterializeSummary {
|
||||
object_count: cir.objects.len(),
|
||||
linked_files: 0,
|
||||
copied_files,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn mirror_relative_path_for_rsync_uri(rsync_uri: &str) -> Result<PathBuf, CirMaterializeError> {
|
||||
let url = url::Url::parse(rsync_uri)
|
||||
.map_err(|_| CirMaterializeError::InvalidRsyncUri(rsync_uri.to_string()))?;
|
||||
@@ -160,8 +260,8 @@ pub fn resolve_static_pool_file(
|
||||
let prefix1 = &sha256_hex[0..2];
|
||||
let prefix2 = &sha256_hex[2..4];
|
||||
|
||||
let entries = fs::read_dir(static_root)
|
||||
.map_err(|_| CirMaterializeError::MissingStaticObject {
|
||||
let entries =
|
||||
fs::read_dir(static_root).map_err(|_| CirMaterializeError::MissingStaticObject {
|
||||
sha256_hex: sha256_hex.to_string(),
|
||||
})?;
|
||||
let mut dates = entries
|
||||
@@ -215,10 +315,15 @@ fn collect_materialized_uris(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{CirMaterializeError, materialize_cir, mirror_relative_path_for_rsync_uri, resolve_static_pool_file};
|
||||
use super::{
|
||||
CirMaterializeError, materialize_cir, materialize_cir_from_raw_store,
|
||||
mirror_relative_path_for_rsync_uri, resolve_static_pool_file,
|
||||
};
|
||||
use crate::blob_store::ExternalRawStoreDb;
|
||||
use crate::cir::model::{
|
||||
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal,
|
||||
};
|
||||
use sha2::Digest;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
fn sample_time() -> time::OffsetDateTime {
|
||||
@@ -257,21 +362,45 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn cir_with_real_hashes(a: &[u8], b: &[u8]) -> CanonicalInputRepresentation {
|
||||
CanonicalInputRepresentation {
|
||||
version: CIR_VERSION_V1,
|
||||
hash_alg: CirHashAlgorithm::Sha256,
|
||||
validation_time: sample_time(),
|
||||
objects: vec![
|
||||
CirObject {
|
||||
rsync_uri: "rsync://example.net/repo/a.cer".to_string(),
|
||||
sha256: sha2::Sha256::digest(a).to_vec(),
|
||||
},
|
||||
CirObject {
|
||||
rsync_uri: "rsync://example.net/repo/nested/b.roa".to_string(),
|
||||
sha256: sha2::Sha256::digest(b).to_vec(),
|
||||
},
|
||||
],
|
||||
tals: vec![CirTal {
|
||||
tal_uri: "https://tal.example.net/root.tal".to_string(),
|
||||
tal_bytes: b"x".to_vec(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mirror_relative_path_for_rsync_uri_maps_host_and_path() {
|
||||
let path =
|
||||
mirror_relative_path_for_rsync_uri("rsync://example.net/repo/nested/b.roa").unwrap();
|
||||
assert_eq!(path, PathBuf::from("example.net").join("repo").join("nested").join("b.roa"));
|
||||
assert_eq!(
|
||||
path,
|
||||
PathBuf::from("example.net")
|
||||
.join("repo")
|
||||
.join("nested")
|
||||
.join("b.roa")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_static_pool_file_finds_hash_across_dates() {
|
||||
let td = tempfile::tempdir().unwrap();
|
||||
let path = td
|
||||
.path()
|
||||
.join("20260407")
|
||||
.join("11")
|
||||
.join("11");
|
||||
let path = td.path().join("20260407").join("11").join("11");
|
||||
std::fs::create_dir_all(&path).unwrap();
|
||||
let file = path.join("1111111111111111111111111111111111111111111111111111111111111111");
|
||||
std::fs::write(&file, b"x").unwrap();
|
||||
@@ -289,14 +418,20 @@ mod tests {
|
||||
let td = tempfile::tempdir().unwrap();
|
||||
let err = resolve_static_pool_file(td.path(), "not-a-hash")
|
||||
.expect_err("invalid hash should fail");
|
||||
assert!(matches!(err, CirMaterializeError::MissingStaticObject { .. }));
|
||||
assert!(matches!(
|
||||
err,
|
||||
CirMaterializeError::MissingStaticObject { .. }
|
||||
));
|
||||
|
||||
let err = resolve_static_pool_file(
|
||||
td.path(),
|
||||
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
)
|
||||
.expect_err("missing hash should fail");
|
||||
assert!(matches!(err, CirMaterializeError::MissingStaticObject { .. }));
|
||||
assert!(matches!(
|
||||
err,
|
||||
CirMaterializeError::MissingStaticObject { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -333,7 +468,10 @@ mod tests {
|
||||
|
||||
let summary = materialize_cir(&sample_cir(), &static_root, &mirror_root, true).unwrap();
|
||||
assert_eq!(summary.object_count, 2);
|
||||
assert_eq!(std::fs::read(mirror_root.join("example.net/repo/a.cer")).unwrap(), b"a");
|
||||
assert_eq!(
|
||||
std::fs::read(mirror_root.join("example.net/repo/a.cer")).unwrap(),
|
||||
b"a"
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(mirror_root.join("example.net/repo/nested/b.roa")).unwrap(),
|
||||
b"b"
|
||||
@@ -344,14 +482,12 @@ mod tests {
|
||||
#[test]
|
||||
fn materialize_fails_when_static_object_missing() {
|
||||
let td = tempfile::tempdir().unwrap();
|
||||
let err = materialize_cir(
|
||||
&sample_cir(),
|
||||
td.path(),
|
||||
&td.path().join("mirror"),
|
||||
true,
|
||||
)
|
||||
.expect_err("missing static object must fail");
|
||||
assert!(matches!(err, CirMaterializeError::MissingStaticObject { .. }));
|
||||
let err = materialize_cir(&sample_cir(), td.path(), &td.path().join("mirror"), true)
|
||||
.expect_err("missing static object must fail");
|
||||
assert!(matches!(
|
||||
err,
|
||||
CirMaterializeError::MissingStaticObject { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -380,6 +516,192 @@ mod tests {
|
||||
assert!(matches!(err, CirMaterializeError::TreeMismatch(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn materialize_from_raw_store_creates_expected_tree() {
|
||||
let td = tempfile::tempdir().unwrap();
|
||||
let raw_store_path = td.path().join("raw-store.db");
|
||||
let mirror_root = td.path().join("mirror");
|
||||
let a = b"a".to_vec();
|
||||
let b = b"b".to_vec();
|
||||
let cir = CanonicalInputRepresentation {
|
||||
version: CIR_VERSION_V1,
|
||||
hash_alg: CirHashAlgorithm::Sha256,
|
||||
validation_time: sample_time(),
|
||||
objects: vec![
|
||||
CirObject {
|
||||
rsync_uri: "rsync://example.net/repo/a.cer".to_string(),
|
||||
sha256: sha2::Sha256::digest(&a).to_vec(),
|
||||
},
|
||||
CirObject {
|
||||
rsync_uri: "rsync://example.net/repo/nested/b.roa".to_string(),
|
||||
sha256: sha2::Sha256::digest(&b).to_vec(),
|
||||
},
|
||||
],
|
||||
tals: vec![CirTal {
|
||||
tal_uri: "https://tal.example.net/root.tal".to_string(),
|
||||
tal_bytes: b"x".to_vec(),
|
||||
}],
|
||||
};
|
||||
|
||||
{
|
||||
let raw_store = ExternalRawStoreDb::open(&raw_store_path).unwrap();
|
||||
let mut entry_a =
|
||||
crate::storage::RawByHashEntry::from_bytes(hex::encode(&cir.objects[0].sha256), a);
|
||||
entry_a.origin_uris.push(cir.objects[0].rsync_uri.clone());
|
||||
raw_store.put_raw_entry(&entry_a).unwrap();
|
||||
|
||||
let mut entry_b =
|
||||
crate::storage::RawByHashEntry::from_bytes(hex::encode(&cir.objects[1].sha256), b);
|
||||
entry_b.origin_uris.push(cir.objects[1].rsync_uri.clone());
|
||||
raw_store.put_raw_entry(&entry_b).unwrap();
|
||||
}
|
||||
|
||||
let summary =
|
||||
materialize_cir_from_raw_store(&cir, &raw_store_path, &mirror_root, true).unwrap();
|
||||
assert_eq!(summary.object_count, 2);
|
||||
assert_eq!(summary.linked_files, 0);
|
||||
assert_eq!(summary.copied_files, 2);
|
||||
assert_eq!(
|
||||
std::fs::read(mirror_root.join("example.net/repo/a.cer")).unwrap(),
|
||||
b"a"
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(mirror_root.join("example.net/repo/nested/b.roa")).unwrap(),
|
||||
b"b"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn materialize_from_raw_store_fails_when_object_missing() {
|
||||
let td = tempfile::tempdir().unwrap();
|
||||
let raw_store_path = td.path().join("raw-store.db");
|
||||
let mirror_root = td.path().join("mirror");
|
||||
let cir = cir_with_real_hashes(b"a", b"b");
|
||||
{
|
||||
let raw_store = ExternalRawStoreDb::open(&raw_store_path).unwrap();
|
||||
let only = crate::storage::RawByHashEntry::from_bytes(
|
||||
hex::encode(&cir.objects[0].sha256),
|
||||
b"a".to_vec(),
|
||||
);
|
||||
raw_store.put_raw_entry(&only).unwrap();
|
||||
}
|
||||
|
||||
let err = materialize_cir_from_raw_store(&cir, &raw_store_path, &mirror_root, true)
|
||||
.expect_err("missing second object should fail");
|
||||
assert!(matches!(
|
||||
err,
|
||||
CirMaterializeError::MissingRawStoreObject { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn materialize_from_raw_store_detects_stale_tree_when_not_clean_rebuild() {
|
||||
let td = tempfile::tempdir().unwrap();
|
||||
let raw_store_path = td.path().join("raw-store.db");
|
||||
let mirror_root = td.path().join("mirror");
|
||||
let cir = cir_with_real_hashes(b"a", b"b");
|
||||
{
|
||||
let raw_store = ExternalRawStoreDb::open(&raw_store_path).unwrap();
|
||||
let mut entry_a = crate::storage::RawByHashEntry::from_bytes(
|
||||
hex::encode(&cir.objects[0].sha256),
|
||||
b"a".to_vec(),
|
||||
);
|
||||
entry_a.origin_uris.push(cir.objects[0].rsync_uri.clone());
|
||||
raw_store.put_raw_entry(&entry_a).unwrap();
|
||||
let mut entry_b = crate::storage::RawByHashEntry::from_bytes(
|
||||
hex::encode(&cir.objects[1].sha256),
|
||||
b"b".to_vec(),
|
||||
);
|
||||
entry_b.origin_uris.push(cir.objects[1].rsync_uri.clone());
|
||||
raw_store.put_raw_entry(&entry_b).unwrap();
|
||||
}
|
||||
std::fs::create_dir_all(mirror_root.join("extra")).unwrap();
|
||||
std::fs::write(mirror_root.join("extra/stale.txt"), b"stale").unwrap();
|
||||
|
||||
let err =
|
||||
materialize_cir_from_raw_store(&cir, &raw_store_path, &mirror_root, false)
|
||||
.expect_err("stale file should fail exact tree check");
|
||||
assert!(matches!(err, CirMaterializeError::TreeMismatch(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn materialize_from_raw_store_overwrites_existing_targets() {
|
||||
let td = tempfile::tempdir().unwrap();
|
||||
let raw_store_path = td.path().join("raw-store.db");
|
||||
let mirror_root = td.path().join("mirror");
|
||||
let a = b"new-a".to_vec();
|
||||
let b = b"new-b".to_vec();
|
||||
let cir = cir_with_real_hashes(&a, &b);
|
||||
{
|
||||
let raw_store = ExternalRawStoreDb::open(&raw_store_path).unwrap();
|
||||
let mut entry_a = crate::storage::RawByHashEntry::from_bytes(
|
||||
hex::encode(&cir.objects[0].sha256),
|
||||
a.clone(),
|
||||
);
|
||||
entry_a.origin_uris.push(cir.objects[0].rsync_uri.clone());
|
||||
raw_store.put_raw_entry(&entry_a).unwrap();
|
||||
let mut entry_b = crate::storage::RawByHashEntry::from_bytes(
|
||||
hex::encode(&cir.objects[1].sha256),
|
||||
b.clone(),
|
||||
);
|
||||
entry_b.origin_uris.push(cir.objects[1].rsync_uri.clone());
|
||||
raw_store.put_raw_entry(&entry_b).unwrap();
|
||||
}
|
||||
let target = mirror_root.join("example.net/repo/a.cer");
|
||||
std::fs::create_dir_all(target.parent().unwrap()).unwrap();
|
||||
std::fs::write(&target, b"old").unwrap();
|
||||
|
||||
let summary =
|
||||
materialize_cir_from_raw_store(&cir, &raw_store_path, &mirror_root, false).unwrap();
|
||||
assert_eq!(summary.copied_files, 2);
|
||||
assert_eq!(std::fs::read(&target).unwrap(), a);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn materialize_from_raw_store_reports_codec_errors() {
|
||||
let td = tempfile::tempdir().unwrap();
|
||||
let raw_store_path = td.path().join("raw-store.db");
|
||||
let mirror_root = td.path().join("mirror");
|
||||
{
|
||||
let _raw_store = ExternalRawStoreDb::open(&raw_store_path).unwrap();
|
||||
}
|
||||
{
|
||||
let db = rocksdb::DB::open_default(&raw_store_path).unwrap();
|
||||
db.put(
|
||||
b"rawbyhash:1111111111111111111111111111111111111111111111111111111111111111",
|
||||
b"bad-cbor",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let err = materialize_cir_from_raw_store(
|
||||
&CanonicalInputRepresentation {
|
||||
version: CIR_VERSION_V1,
|
||||
hash_alg: CirHashAlgorithm::Sha256,
|
||||
validation_time: sample_time(),
|
||||
objects: vec![CirObject {
|
||||
rsync_uri: "rsync://example.net/repo/a.cer".to_string(),
|
||||
sha256: hex::decode(
|
||||
"1111111111111111111111111111111111111111111111111111111111111111",
|
||||
)
|
||||
.unwrap(),
|
||||
}],
|
||||
tals: vec![CirTal {
|
||||
tal_uri: "https://tal.example.net/root.tal".to_string(),
|
||||
tal_bytes: b"x".to_vec(),
|
||||
}],
|
||||
},
|
||||
&raw_store_path,
|
||||
&mirror_root,
|
||||
true,
|
||||
)
|
||||
.expect_err("corrupt raw-store object should fail");
|
||||
assert!(matches!(
|
||||
err,
|
||||
CirMaterializeError::ReadRawStore { .. } | CirMaterializeError::MissingRawStoreObject { .. }
|
||||
));
|
||||
}
|
||||
|
||||
fn write_static(root: &Path, date: &str, hash: &str, bytes: &[u8]) {
|
||||
let path = root.join(date).join(&hash[0..2]).join(&hash[2..4]);
|
||||
std::fs::create_dir_all(&path).unwrap();
|
||||
|
||||
@@ -1,26 +1,29 @@
|
||||
pub mod decode;
|
||||
pub mod encode;
|
||||
#[cfg(feature = "full")]
|
||||
pub mod export;
|
||||
pub mod materialize;
|
||||
pub mod model;
|
||||
pub mod sequence;
|
||||
#[cfg(feature = "full")]
|
||||
pub mod export;
|
||||
#[cfg(feature = "full")]
|
||||
pub mod static_pool;
|
||||
|
||||
pub use decode::{CirDecodeError, decode_cir};
|
||||
pub use encode::{CirEncodeError, encode_cir};
|
||||
#[cfg(feature = "full")]
|
||||
pub use export::{
|
||||
CirExportError, CirExportSummary, build_cir_from_run, export_cir_from_run, write_cir_file,
|
||||
};
|
||||
pub use materialize::{
|
||||
CirMaterializeError, CirMaterializeSummary, materialize_cir, mirror_relative_path_for_rsync_uri,
|
||||
resolve_static_pool_file,
|
||||
CirMaterializeError, CirMaterializeSummary, materialize_cir,
|
||||
materialize_cir_from_raw_store,
|
||||
mirror_relative_path_for_rsync_uri, resolve_static_pool_file,
|
||||
};
|
||||
pub use model::{
|
||||
CIR_VERSION_V1, CirHashAlgorithm, CirObject, CirTal, CanonicalInputRepresentation,
|
||||
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal,
|
||||
};
|
||||
pub use sequence::{CirSequenceManifest, CirSequenceStep, CirSequenceStepKind};
|
||||
#[cfg(feature = "full")]
|
||||
pub use export::{CirExportError, CirExportSummary, build_cir_from_run, export_cir_from_run, write_cir_file};
|
||||
#[cfg(feature = "full")]
|
||||
pub use static_pool::{
|
||||
CirStaticPoolError, CirStaticPoolExportSummary, CirStaticPoolWriteResult,
|
||||
export_hashes_from_store, static_pool_path, static_pool_relative_path,
|
||||
@@ -59,9 +62,8 @@ mod tests {
|
||||
],
|
||||
tals: vec![CirTal {
|
||||
tal_uri: "https://tal.example.net/root.tal".to_string(),
|
||||
tal_bytes:
|
||||
b"https://tal.example.net/ta.cer\nrsync://example.net/repo/ta.cer\nMIIB"
|
||||
.to_vec(),
|
||||
tal_bytes: b"https://tal.example.net/ta.cer\nrsync://example.net/repo/ta.cer\nMIIB"
|
||||
.to_vec(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
@@ -74,7 +76,10 @@ mod tests {
|
||||
} else {
|
||||
let len = value.len();
|
||||
let bytes = len.to_be_bytes();
|
||||
let first_non_zero = bytes.iter().position(|&b| b != 0).unwrap_or(bytes.len() - 1);
|
||||
let first_non_zero = bytes
|
||||
.iter()
|
||||
.position(|&b| b != 0)
|
||||
.unwrap_or(bytes.len() - 1);
|
||||
let len_bytes = &bytes[first_non_zero..];
|
||||
out.push(0x80 | len_bytes.len() as u8);
|
||||
out.extend_from_slice(len_bytes);
|
||||
@@ -178,8 +183,7 @@ mod tests {
|
||||
der[idx + sha256_bytes.len() - 1] ^= 0x01;
|
||||
let err = decode_cir(&der).expect_err("wrong oid must fail");
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains(crate::data_model::oid::OID_SHA256),
|
||||
err.to_string().contains(crate::data_model::oid::OID_SHA256),
|
||||
"{err}"
|
||||
);
|
||||
}
|
||||
@@ -222,7 +226,10 @@ mod tests {
|
||||
tals: Vec::new(),
|
||||
};
|
||||
let err = encode_cir(&no_tals).expect_err("empty tals must fail");
|
||||
assert!(err.to_string().contains("CIR.tals must be non-empty"), "{err}");
|
||||
assert!(
|
||||
err.to_string().contains("CIR.tals must be non-empty"),
|
||||
"{err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -276,7 +283,10 @@ mod tests {
|
||||
let mut der = encode_cir(&cir).expect("encode cir");
|
||||
der.push(0);
|
||||
let err = decode_cir(&der).expect_err("trailing bytes after cir must fail");
|
||||
assert!(err.to_string().contains("trailing bytes after CIR"), "{err}");
|
||||
assert!(
|
||||
err.to_string().contains("trailing bytes after CIR"),
|
||||
"{err}"
|
||||
);
|
||||
|
||||
let object = test_encode_tlv(
|
||||
0x30,
|
||||
@@ -307,7 +317,10 @@ mod tests {
|
||||
.concat(),
|
||||
);
|
||||
let err = decode_cir(&bad).expect_err("trailing field in object must fail");
|
||||
assert!(err.to_string().contains("trailing fields in CirObject"), "{err}");
|
||||
assert!(
|
||||
err.to_string().contains("trailing fields in CirObject"),
|
||||
"{err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -111,7 +111,9 @@ fn validate_sorted_unique_strings<'a>(
|
||||
) -> Result<(), String> {
|
||||
let mut prev: Option<&'a str> = None;
|
||||
for key in items {
|
||||
if let Some(prev_key) = prev && key <= prev_key {
|
||||
if let Some(prev_key) = prev
|
||||
&& key <= prev_key
|
||||
{
|
||||
return Err(message.into());
|
||||
}
|
||||
prev = Some(key);
|
||||
|
||||
@@ -15,13 +15,24 @@ pub struct CirSequenceStep {
|
||||
pub cir_path: String,
|
||||
pub ccr_path: String,
|
||||
pub report_path: String,
|
||||
#[serde(default)]
|
||||
pub timing_path: Option<String>,
|
||||
#[serde(default)]
|
||||
pub stdout_log_path: Option<String>,
|
||||
#[serde(default)]
|
||||
pub stderr_log_path: Option<String>,
|
||||
#[serde(default)]
|
||||
pub artifact_prefix: Option<String>,
|
||||
pub previous_step_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct CirSequenceManifest {
|
||||
pub version: u32,
|
||||
pub static_root: String,
|
||||
#[serde(default)]
|
||||
pub static_root: Option<String>,
|
||||
#[serde(default)]
|
||||
pub raw_store_db_path: Option<String>,
|
||||
pub steps: Vec<CirSequenceStep>,
|
||||
}
|
||||
|
||||
@@ -30,8 +41,16 @@ impl CirSequenceManifest {
|
||||
if self.version == 0 {
|
||||
return Err("sequence.version must be positive".to_string());
|
||||
}
|
||||
if self.static_root.trim().is_empty() {
|
||||
return Err("sequence.static_root must not be empty".to_string());
|
||||
let backend_count = self.static_root.is_some() as u8 + self.raw_store_db_path.is_some() as u8;
|
||||
if backend_count != 1 {
|
||||
return Err("sequence must set exactly one of static_root or raw_store_db_path".to_string());
|
||||
}
|
||||
match (self.static_root.as_ref(), self.raw_store_db_path.as_ref()) {
|
||||
(Some(static_root), None) if !static_root.trim().is_empty() => {}
|
||||
(None, Some(raw_store_db_path)) if !raw_store_db_path.trim().is_empty() => {}
|
||||
_ => {
|
||||
return Err("sequence backend path must not be empty".to_string());
|
||||
}
|
||||
}
|
||||
if self.steps.is_empty() {
|
||||
return Err("sequence.steps must not be empty".to_string());
|
||||
@@ -49,6 +68,34 @@ impl CirSequenceManifest {
|
||||
"sequence.steps[{idx}].validation_time must not be empty"
|
||||
));
|
||||
}
|
||||
if let Some(timing_path) = &step.timing_path
|
||||
&& timing_path.trim().is_empty()
|
||||
{
|
||||
return Err(format!(
|
||||
"sequence.steps[{idx}].timing_path must not be empty when set"
|
||||
));
|
||||
}
|
||||
if let Some(stdout_log_path) = &step.stdout_log_path
|
||||
&& stdout_log_path.trim().is_empty()
|
||||
{
|
||||
return Err(format!(
|
||||
"sequence.steps[{idx}].stdout_log_path must not be empty when set"
|
||||
));
|
||||
}
|
||||
if let Some(stderr_log_path) = &step.stderr_log_path
|
||||
&& stderr_log_path.trim().is_empty()
|
||||
{
|
||||
return Err(format!(
|
||||
"sequence.steps[{idx}].stderr_log_path must not be empty when set"
|
||||
));
|
||||
}
|
||||
if let Some(artifact_prefix) = &step.artifact_prefix
|
||||
&& artifact_prefix.trim().is_empty()
|
||||
{
|
||||
return Err(format!(
|
||||
"sequence.steps[{idx}].artifact_prefix must not be empty when set"
|
||||
));
|
||||
}
|
||||
if step.cir_path.trim().is_empty()
|
||||
|| step.ccr_path.trim().is_empty()
|
||||
|| step.report_path.trim().is_empty()
|
||||
@@ -92,7 +139,8 @@ mod tests {
|
||||
fn sample_manifest() -> CirSequenceManifest {
|
||||
CirSequenceManifest {
|
||||
version: 1,
|
||||
static_root: "static".to_string(),
|
||||
static_root: Some("static".to_string()),
|
||||
raw_store_db_path: None,
|
||||
steps: vec![
|
||||
CirSequenceStep {
|
||||
step_id: "full".to_string(),
|
||||
@@ -101,6 +149,10 @@ mod tests {
|
||||
cir_path: "full/input.cir".to_string(),
|
||||
ccr_path: "full/result.ccr".to_string(),
|
||||
report_path: "full/report.json".to_string(),
|
||||
timing_path: Some("full/timing.json".to_string()),
|
||||
stdout_log_path: Some("full/stdout.log".to_string()),
|
||||
stderr_log_path: Some("full/stderr.log".to_string()),
|
||||
artifact_prefix: Some("2026-04-09T00:00:00Z-test".to_string()),
|
||||
previous_step_id: None,
|
||||
},
|
||||
CirSequenceStep {
|
||||
@@ -110,6 +162,10 @@ mod tests {
|
||||
cir_path: "delta-001/input.cir".to_string(),
|
||||
ccr_path: "delta-001/result.ccr".to_string(),
|
||||
report_path: "delta-001/report.json".to_string(),
|
||||
timing_path: Some("delta-001/timing.json".to_string()),
|
||||
stdout_log_path: Some("delta-001/stdout.log".to_string()),
|
||||
stderr_log_path: Some("delta-001/stderr.log".to_string()),
|
||||
artifact_prefix: Some("2026-04-09T00:10:00Z-test".to_string()),
|
||||
previous_step_id: Some("full".to_string()),
|
||||
},
|
||||
],
|
||||
@@ -144,4 +200,12 @@ mod tests {
|
||||
let err = bad.validate().expect_err("missing previous must fail");
|
||||
assert!(err.contains("previous_step_id"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sequence_manifest_validate_accepts_raw_store_backend() {
|
||||
let mut manifest = sample_manifest();
|
||||
manifest.static_root = None;
|
||||
manifest.raw_store_db_path = Some("raw-store.db".to_string());
|
||||
manifest.validate().expect("raw store sequence");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ use std::fs::{self, OpenOptions};
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::blob_store::RawObjectStore;
|
||||
use crate::storage::{RawByHashEntry, RocksStore};
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
@@ -75,10 +76,7 @@ pub fn static_pool_path(
|
||||
capture_date_utc: time::Date,
|
||||
sha256_hex: &str,
|
||||
) -> Result<PathBuf, CirStaticPoolError> {
|
||||
Ok(static_root.join(static_pool_relative_path(
|
||||
capture_date_utc,
|
||||
sha256_hex,
|
||||
)?))
|
||||
Ok(static_root.join(static_pool_relative_path(capture_date_utc, sha256_hex)?))
|
||||
}
|
||||
|
||||
pub fn write_bytes_to_static_pool(
|
||||
@@ -191,12 +189,13 @@ pub fn export_hashes_from_store(
|
||||
let mut written_files = 0usize;
|
||||
let mut reused_files = 0usize;
|
||||
for sha256_hex in &unique {
|
||||
let entry = store
|
||||
.get_raw_by_hash_entry(sha256_hex)
|
||||
let bytes = store
|
||||
.get_blob_bytes(sha256_hex)
|
||||
.map_err(|e| CirStaticPoolError::Storage(e.to_string()))?
|
||||
.ok_or_else(|| CirStaticPoolError::MissingRawByHash {
|
||||
sha256_hex: sha256_hex.clone(),
|
||||
})?;
|
||||
let entry = RawByHashEntry::from_bytes(sha256_hex.clone(), bytes);
|
||||
let result = write_raw_entry_to_static_pool(static_root, capture_date_utc, &entry)?;
|
||||
if result.written {
|
||||
written_files += 1;
|
||||
@@ -223,9 +222,7 @@ fn format_utc_date(date: time::Date) -> String {
|
||||
|
||||
fn validate_sha256_hex(sha256_hex: &str) -> Result<(), CirStaticPoolError> {
|
||||
if sha256_hex.len() != 64 || !sha256_hex.as_bytes().iter().all(u8::is_ascii_hexdigit) {
|
||||
return Err(CirStaticPoolError::InvalidSha256Hex(
|
||||
sha256_hex.to_string(),
|
||||
));
|
||||
return Err(CirStaticPoolError::InvalidSha256Hex(sha256_hex.to_string()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -238,8 +235,8 @@ fn compute_sha256_hex(bytes: &[u8]) -> String {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
CirStaticPoolError, compute_sha256_hex, export_hashes_from_store, static_pool_relative_path,
|
||||
write_bytes_to_static_pool,
|
||||
CirStaticPoolError, compute_sha256_hex, export_hashes_from_store,
|
||||
static_pool_relative_path, write_bytes_to_static_pool,
|
||||
};
|
||||
use crate::storage::{RawByHashEntry, RepositoryViewEntry, RepositoryViewState, RocksStore};
|
||||
use std::fs;
|
||||
@@ -270,8 +267,8 @@ mod tests {
|
||||
let bytes = b"static-pool-object";
|
||||
let sha = compute_sha256_hex(bytes);
|
||||
|
||||
let first = write_bytes_to_static_pool(td.path(), sample_date(), &sha, bytes)
|
||||
.expect("first write");
|
||||
let first =
|
||||
write_bytes_to_static_pool(td.path(), sample_date(), &sha, bytes).expect("first write");
|
||||
let second = write_bytes_to_static_pool(td.path(), sample_date(), &sha, bytes)
|
||||
.expect("second write");
|
||||
|
||||
@@ -281,11 +278,13 @@ mod tests {
|
||||
|
||||
let all_files: Vec<_> = walk_files(td.path());
|
||||
assert_eq!(all_files.len(), 1);
|
||||
assert!(!all_files[0]
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or_default()
|
||||
.contains(".tmp."));
|
||||
assert!(
|
||||
!all_files[0]
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or_default()
|
||||
.contains(".tmp.")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -315,7 +314,9 @@ mod tests {
|
||||
let bytes = b"store-object".to_vec();
|
||||
let sha = compute_sha256_hex(&bytes);
|
||||
let mut entry = RawByHashEntry::from_bytes(sha.clone(), bytes.clone());
|
||||
entry.origin_uris.push("rsync://example.test/repo/object.cer".to_string());
|
||||
entry
|
||||
.origin_uris
|
||||
.push("rsync://example.test/repo/object.cer".to_string());
|
||||
store.put_raw_by_hash_entry(&entry).expect("put raw entry");
|
||||
store
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
|
||||
202
src/cli.rs
202
src/cli.rs
@@ -4,8 +4,8 @@ use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::analysis::timing::{TimingHandle, TimingMeta, TimingMetaUpdate};
|
||||
use crate::audit::{
|
||||
AspaOutput, AuditReportV2, AuditRunMeta, AuditWarning, TreeSummary, VrpOutput,
|
||||
format_roa_ip_prefix,
|
||||
AspaOutput, AuditReportV2, AuditRepoSyncStats, AuditRunMeta, AuditWarning, TreeSummary,
|
||||
VrpOutput, format_roa_ip_prefix,
|
||||
};
|
||||
use crate::fetch::http::{BlockingHttpFetcher, HttpFetcherConfig};
|
||||
use crate::fetch::rsync::LocalDirRsyncFetcher;
|
||||
@@ -51,6 +51,7 @@ pub struct CliArgs {
|
||||
pub ta_path: Option<PathBuf>,
|
||||
|
||||
pub db_path: PathBuf,
|
||||
pub raw_store_db: Option<PathBuf>,
|
||||
pub policy_path: Option<PathBuf>,
|
||||
pub report_json_path: Option<PathBuf>,
|
||||
pub ccr_out_path: Option<PathBuf>,
|
||||
@@ -92,12 +93,13 @@ Usage:
|
||||
|
||||
Options:
|
||||
--db <path> RocksDB directory path (required)
|
||||
--raw-store-db <path> External raw-by-hash store DB path (optional)
|
||||
--policy <path> Policy TOML path (optional)
|
||||
--report-json <path> Write full audit report as JSON (optional)
|
||||
--ccr-out <path> Write CCR DER ContentInfo to this path (optional)
|
||||
--cir-enable Export CIR after the run completes
|
||||
--cir-out <path> Write CIR DER to this path (requires --cir-enable)
|
||||
--cir-static-root <path> Shared static pool root for CIR export (requires --cir-enable)
|
||||
--cir-static-root <path> Shared static pool root for CIR export (requires --cir-enable unless --raw-store-db is used)
|
||||
--cir-tal-uri <url> Override TAL URI for CIR export when using --tal-path (optional)
|
||||
--payload-replay-archive <path> Use local payload replay archive root (offline replay mode)
|
||||
--payload-replay-locks <path> Use local payload replay locks.json (offline replay mode)
|
||||
@@ -134,6 +136,7 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
||||
let mut ta_path: Option<PathBuf> = None;
|
||||
|
||||
let mut db_path: Option<PathBuf> = None;
|
||||
let mut raw_store_db: Option<PathBuf> = None;
|
||||
let mut policy_path: Option<PathBuf> = None;
|
||||
let mut report_json_path: Option<PathBuf> = None;
|
||||
let mut ccr_out_path: Option<PathBuf> = None;
|
||||
@@ -152,8 +155,8 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
||||
let mut rsync_local_dir: Option<PathBuf> = None;
|
||||
let mut disable_rrdp: bool = false;
|
||||
let mut rsync_command: Option<PathBuf> = None;
|
||||
let mut http_timeout_secs: u64 = 20;
|
||||
let mut rsync_timeout_secs: u64 = 60;
|
||||
let mut http_timeout_secs: u64 = 30;
|
||||
let mut rsync_timeout_secs: u64 = 30;
|
||||
let mut rsync_mirror_root: Option<PathBuf> = None;
|
||||
let mut max_depth: Option<usize> = None;
|
||||
let mut max_instances: Option<usize> = None;
|
||||
@@ -186,6 +189,11 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
||||
let v = argv.get(i).ok_or("--db requires a value")?;
|
||||
db_path = Some(PathBuf::from(v));
|
||||
}
|
||||
"--raw-store-db" => {
|
||||
i += 1;
|
||||
let v = argv.get(i).ok_or("--raw-store-db requires a value")?;
|
||||
raw_store_db = Some(PathBuf::from(v));
|
||||
}
|
||||
"--policy" => {
|
||||
i += 1;
|
||||
let v = argv.get(i).ok_or("--policy requires a value")?;
|
||||
@@ -247,7 +255,9 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
||||
}
|
||||
"--payload-base-validation-time" => {
|
||||
i += 1;
|
||||
let v = argv.get(i).ok_or("--payload-base-validation-time requires a value")?;
|
||||
let v = argv
|
||||
.get(i)
|
||||
.ok_or("--payload-base-validation-time requires a value")?;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
let t = time::OffsetDateTime::parse(v, &Rfc3339).map_err(|e| {
|
||||
format!("invalid --payload-base-validation-time (RFC3339 expected): {e}")
|
||||
@@ -350,13 +360,17 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
||||
usage()
|
||||
));
|
||||
}
|
||||
if cir_enabled && (cir_out_path.is_none() || cir_static_root.is_none()) {
|
||||
let cir_backend_count = cir_static_root.is_some() as u8 + raw_store_db.is_some() as u8;
|
||||
if cir_enabled && (cir_out_path.is_none() || cir_backend_count != 1) {
|
||||
return Err(format!(
|
||||
"--cir-enable requires both --cir-out and --cir-static-root\n\n{}",
|
||||
"--cir-enable requires --cir-out and exactly one of --cir-static-root or --raw-store-db\n\n{}",
|
||||
usage()
|
||||
));
|
||||
}
|
||||
if !cir_enabled && (cir_out_path.is_some() || cir_static_root.is_some() || cir_tal_uri.is_some())
|
||||
if !cir_enabled
|
||||
&& (cir_out_path.is_some()
|
||||
|| cir_static_root.is_some()
|
||||
|| cir_tal_uri.is_some())
|
||||
{
|
||||
return Err(format!(
|
||||
"--cir-out/--cir-static-root/--cir-tal-uri require --cir-enable\n\n{}",
|
||||
@@ -459,6 +473,7 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
||||
tal_path,
|
||||
ta_path,
|
||||
db_path,
|
||||
raw_store_db,
|
||||
policy_path,
|
||||
report_json_path,
|
||||
ccr_out_path,
|
||||
@@ -578,6 +593,8 @@ fn build_report(
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let repo_sync_stats = build_repo_sync_stats(&out.publication_points);
|
||||
|
||||
AuditReportV2 {
|
||||
format_version: 2,
|
||||
meta: AuditRunMeta {
|
||||
@@ -594,9 +611,36 @@ fn build_report(
|
||||
aspas,
|
||||
downloads: out.downloads,
|
||||
download_stats: out.download_stats,
|
||||
repo_sync_stats,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_repo_sync_stats(
|
||||
publication_points: &[crate::audit::PublicationPointAudit],
|
||||
) -> AuditRepoSyncStats {
|
||||
let mut stats = AuditRepoSyncStats {
|
||||
publication_points_total: publication_points.len() as u64,
|
||||
..AuditRepoSyncStats::default()
|
||||
};
|
||||
|
||||
for pp in publication_points {
|
||||
let duration = pp.repo_sync_duration_ms.unwrap_or(0);
|
||||
if let Some(phase) = pp.repo_sync_phase.as_ref() {
|
||||
let entry = stats.by_phase.entry(phase.clone()).or_default();
|
||||
entry.count += 1;
|
||||
entry.duration_ms_total += duration;
|
||||
}
|
||||
let entry = stats
|
||||
.by_terminal_state
|
||||
.entry(pp.repo_terminal_state.clone())
|
||||
.or_default();
|
||||
entry.count += 1;
|
||||
entry.duration_ms_total += duration;
|
||||
}
|
||||
|
||||
stats
|
||||
}
|
||||
|
||||
pub fn run(argv: &[String]) -> Result<(), String> {
|
||||
let args = parse_args(argv)?;
|
||||
|
||||
@@ -608,7 +652,12 @@ pub fn run(argv: &[String]) -> Result<(), String> {
|
||||
.validation_time
|
||||
.unwrap_or_else(time::OffsetDateTime::now_utc);
|
||||
|
||||
let store = RocksStore::open(&args.db_path).map_err(|e| e.to_string())?;
|
||||
let store = if let Some(raw_store_db) = args.raw_store_db.as_ref() {
|
||||
RocksStore::open_with_external_raw_store(&args.db_path, raw_store_db)
|
||||
.map_err(|e| e.to_string())?
|
||||
} else {
|
||||
RocksStore::open(&args.db_path).map_err(|e| e.to_string())?
|
||||
};
|
||||
let config = TreeRunConfig {
|
||||
max_depth: args.max_depth,
|
||||
max_instances: args.max_instances,
|
||||
@@ -1094,10 +1143,6 @@ pub fn run(argv: &[String]) -> Result<(), String> {
|
||||
.cir_out_path
|
||||
.as_deref()
|
||||
.expect("validated by parse_args for cir");
|
||||
let cir_static_root = args
|
||||
.cir_static_root
|
||||
.as_deref()
|
||||
.expect("validated by parse_args for cir");
|
||||
let summary = export_cir_from_run(
|
||||
&store,
|
||||
&out.discovery.trust_anchor,
|
||||
@@ -1105,7 +1150,8 @@ pub fn run(argv: &[String]) -> Result<(), String> {
|
||||
validation_time,
|
||||
&out.publication_points,
|
||||
cir_out_path,
|
||||
cir_static_root,
|
||||
args.cir_static_root.as_deref(),
|
||||
args.raw_store_db.as_deref(),
|
||||
time::OffsetDateTime::now_utc().date(),
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
@@ -1113,13 +1159,22 @@ pub fn run(argv: &[String]) -> Result<(), String> {
|
||||
cir_static_pool_ms = Some(summary.timing.static_pool_ms);
|
||||
cir_write_cir_ms = Some(summary.timing.write_cir_ms);
|
||||
cir_total_ms = Some(summary.timing.total_ms);
|
||||
let (backend_name, written_entries, reused_entries) = match &summary.object_pool {
|
||||
crate::cir::export::CirObjectPoolExportSummary::Static(s) => {
|
||||
("static", s.written_files, s.reused_files)
|
||||
}
|
||||
crate::cir::export::CirObjectPoolExportSummary::RawStore(s) => {
|
||||
("raw-store", s.written_entries, s.reused_entries)
|
||||
}
|
||||
};
|
||||
eprintln!(
|
||||
"wrote CIR: {} (objects={}, tals={}, static_written={}, static_reused={}, build_cir_ms={}, static_pool_ms={}, write_cir_ms={}, total_ms={})",
|
||||
"wrote CIR: {} (objects={}, tals={}, backend={}, written={}, reused={}, build_cir_ms={}, static_pool_ms={}, write_cir_ms={}, total_ms={})",
|
||||
cir_out_path.display(),
|
||||
summary.object_count,
|
||||
summary.tal_count,
|
||||
summary.static_pool.written_files,
|
||||
summary.static_pool.reused_files,
|
||||
backend_name,
|
||||
written_entries,
|
||||
reused_entries,
|
||||
summary.timing.build_cir_ms,
|
||||
summary.timing.static_pool_ms,
|
||||
summary.timing.write_cir_ms,
|
||||
@@ -1160,7 +1215,12 @@ pub fn run(argv: &[String]) -> Result<(), String> {
|
||||
&stage_timing_path,
|
||||
serde_json::to_vec_pretty(&stage_timing).map_err(|e| e.to_string())?,
|
||||
)
|
||||
.map_err(|e| format!("write stage timing failed: {}: {e}", stage_timing_path.display()))?;
|
||||
.map_err(|e| {
|
||||
format!(
|
||||
"write stage timing failed: {}: {e}",
|
||||
stage_timing_path.display()
|
||||
)
|
||||
})?;
|
||||
eprintln!("analysis: wrote {}", stage_timing_path.display());
|
||||
}
|
||||
}
|
||||
@@ -1290,7 +1350,57 @@ mod tests {
|
||||
"out/example.ccr".to_string(),
|
||||
];
|
||||
let args = parse_args(&argv).expect("parse args");
|
||||
assert_eq!(args.ccr_out_path.as_deref(), Some(std::path::Path::new("out/example.ccr")));
|
||||
assert_eq!(
|
||||
args.ccr_out_path.as_deref(),
|
||||
Some(std::path::Path::new("out/example.ccr"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_accepts_external_raw_store_db() {
|
||||
let argv = vec![
|
||||
"rpki".to_string(),
|
||||
"--db".to_string(),
|
||||
"db".to_string(),
|
||||
"--raw-store-db".to_string(),
|
||||
"raw-store.db".to_string(),
|
||||
"--tal-url".to_string(),
|
||||
"https://example.test/x.tal".to_string(),
|
||||
];
|
||||
let args = parse_args(&argv).expect("parse args");
|
||||
assert_eq!(
|
||||
args.raw_store_db.as_deref(),
|
||||
Some(std::path::Path::new("raw-store.db"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_accepts_cir_enable_with_raw_store_backend() {
|
||||
let argv = vec![
|
||||
"rpki".to_string(),
|
||||
"--db".to_string(),
|
||||
"db".to_string(),
|
||||
"--raw-store-db".to_string(),
|
||||
"raw-store.db".to_string(),
|
||||
"--tal-path".to_string(),
|
||||
"x.tal".to_string(),
|
||||
"--ta-path".to_string(),
|
||||
"x.cer".to_string(),
|
||||
"--rsync-local-dir".to_string(),
|
||||
"repo".to_string(),
|
||||
"--cir-enable".to_string(),
|
||||
"--cir-out".to_string(),
|
||||
"out/example.cir".to_string(),
|
||||
"--cir-tal-uri".to_string(),
|
||||
"https://example.test/root.tal".to_string(),
|
||||
];
|
||||
let args = parse_args(&argv).expect("parse args");
|
||||
assert!(args.cir_enabled);
|
||||
assert_eq!(
|
||||
args.raw_store_db.as_deref(),
|
||||
Some(std::path::Path::new("raw-store.db"))
|
||||
);
|
||||
assert_eq!(args.cir_static_root, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1315,9 +1425,18 @@ mod tests {
|
||||
];
|
||||
let args = parse_args(&argv).expect("parse args");
|
||||
assert!(args.cir_enabled);
|
||||
assert_eq!(args.cir_out_path.as_deref(), Some(std::path::Path::new("out/example.cir")));
|
||||
assert_eq!(args.cir_static_root.as_deref(), Some(std::path::Path::new("out/static")));
|
||||
assert_eq!(args.cir_tal_uri.as_deref(), Some("https://example.test/root.tal"));
|
||||
assert_eq!(
|
||||
args.cir_out_path.as_deref(),
|
||||
Some(std::path::Path::new("out/example.cir"))
|
||||
);
|
||||
assert_eq!(
|
||||
args.cir_static_root.as_deref(),
|
||||
Some(std::path::Path::new("out/static"))
|
||||
);
|
||||
assert_eq!(
|
||||
args.cir_tal_uri.as_deref(),
|
||||
Some("https://example.test/root.tal")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1333,7 +1452,10 @@ mod tests {
|
||||
"out/example.cir".to_string(),
|
||||
];
|
||||
let err = parse_args(&argv_missing).unwrap_err();
|
||||
assert!(err.contains("--cir-enable requires both --cir-out and --cir-static-root"), "{err}");
|
||||
assert!(
|
||||
err.contains("--cir-enable requires --cir-out and exactly one of --cir-static-root or --raw-store-db"),
|
||||
"{err}"
|
||||
);
|
||||
|
||||
let argv_needs_enable = vec![
|
||||
"rpki".to_string(),
|
||||
@@ -1835,6 +1957,7 @@ mod tests {
|
||||
aspas: Vec::new(),
|
||||
downloads: Vec::new(),
|
||||
download_stats: crate::audit::AuditDownloadStats::default(),
|
||||
repo_sync_stats: crate::audit::AuditRepoSyncStats::default(),
|
||||
};
|
||||
|
||||
let dir = tempfile::tempdir().expect("tmpdir");
|
||||
@@ -1844,4 +1967,35 @@ mod tests {
|
||||
assert!(s.contains("\"format_version\""));
|
||||
assert!(s.contains("\"policy\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_repo_sync_stats_aggregates_phase_and_terminal_state() {
|
||||
let mut pp1 = crate::audit::PublicationPointAudit::default();
|
||||
pp1.repo_sync_phase = Some("rrdp_ok".to_string());
|
||||
pp1.repo_sync_duration_ms = Some(10);
|
||||
pp1.repo_terminal_state = "fresh".to_string();
|
||||
|
||||
let mut pp2 = crate::audit::PublicationPointAudit::default();
|
||||
pp2.repo_sync_phase = Some("rrdp_failed_rsync_failed".to_string());
|
||||
pp2.repo_sync_duration_ms = Some(20);
|
||||
pp2.repo_terminal_state = "failed_no_cache".to_string();
|
||||
|
||||
let mut pp3 = crate::audit::PublicationPointAudit::default();
|
||||
pp3.repo_sync_phase = Some("rrdp_failed_rsync_failed".to_string());
|
||||
pp3.repo_sync_duration_ms = Some(30);
|
||||
pp3.repo_terminal_state = "failed_no_cache".to_string();
|
||||
|
||||
let stats = build_repo_sync_stats(&[pp1, pp2, pp3]);
|
||||
assert_eq!(stats.publication_points_total, 3);
|
||||
assert_eq!(stats.by_phase["rrdp_ok"].count, 1);
|
||||
assert_eq!(stats.by_phase["rrdp_ok"].duration_ms_total, 10);
|
||||
assert_eq!(stats.by_phase["rrdp_failed_rsync_failed"].count, 2);
|
||||
assert_eq!(stats.by_phase["rrdp_failed_rsync_failed"].duration_ms_total, 50);
|
||||
assert_eq!(stats.by_terminal_state["fresh"].count, 1);
|
||||
assert_eq!(stats.by_terminal_state["failed_no_cache"].count, 2);
|
||||
assert_eq!(
|
||||
stats.by_terminal_state["failed_no_cache"].duration_ms_total,
|
||||
50
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,12 +3,9 @@ use crate::data_model::oid::{
|
||||
};
|
||||
use crate::data_model::rc::{
|
||||
AsIdOrRange, AsIdentifierChoice, ResourceCertKind, ResourceCertificate,
|
||||
ResourceCertificateParseError, ResourceCertificateParsed,
|
||||
ResourceCertificateProfileError,
|
||||
};
|
||||
use crate::validation::cert_path::{
|
||||
CertPathError, validate_ee_cert_path_with_predecoded_ee,
|
||||
ResourceCertificateParseError, ResourceCertificateParsed, ResourceCertificateProfileError,
|
||||
};
|
||||
use crate::validation::cert_path::{CertPathError, validate_ee_cert_path_with_predecoded_ee};
|
||||
use x509_parser::extensions::ParsedExtension;
|
||||
use x509_parser::prelude::{FromDer, X509Certificate};
|
||||
use x509_parser::public_key::PublicKey;
|
||||
@@ -33,7 +30,7 @@ pub enum BgpsecRouterCertificateParseError {
|
||||
#[error("resource certificate parse error: {0} (RFC 5280 §4.1; RFC 6487 §4; RFC 8209 §3.1)")]
|
||||
ResourceCertificate(#[from] ResourceCertificateParseError),
|
||||
|
||||
#[error("X.509 parse error: {0} (RFC 5280 §4.1; RFC 8209 §3.1)" )]
|
||||
#[error("X.509 parse error: {0} (RFC 5280 §4.1; RFC 8209 §3.1)")]
|
||||
X509(String),
|
||||
|
||||
#[error("trailing bytes after router certificate DER: {0} bytes (DER; RFC 5280 §4.1)")]
|
||||
@@ -54,46 +51,72 @@ pub enum BgpsecRouterCertificateProfileError {
|
||||
#[error("BGPsec router certificate must be an EE certificate (RFC 8209 §3.1)")]
|
||||
NotEe,
|
||||
|
||||
#[error("BGPsec router certificate must contain SubjectKeyIdentifier (RFC 6487 §4.8.2; RFC 8209 §3.3)")]
|
||||
#[error(
|
||||
"BGPsec router certificate must contain SubjectKeyIdentifier (RFC 6487 §4.8.2; RFC 8209 §3.3)"
|
||||
)]
|
||||
MissingSki,
|
||||
|
||||
#[error("BGPsec router certificate must include ExtendedKeyUsage (RFC 8209 §3.1.3.2; RFC 8209 §3.3)")]
|
||||
#[error(
|
||||
"BGPsec router certificate must include ExtendedKeyUsage (RFC 8209 §3.1.3.2; RFC 8209 §3.3)"
|
||||
)]
|
||||
MissingExtendedKeyUsage,
|
||||
|
||||
#[error("BGPsec router certificate ExtendedKeyUsage must be non-critical (RFC 6487 §4.8.4; RFC 8209 §3.1.3.2)")]
|
||||
#[error(
|
||||
"BGPsec router certificate ExtendedKeyUsage must be non-critical (RFC 6487 §4.8.4; RFC 8209 §3.1.3.2)"
|
||||
)]
|
||||
ExtendedKeyUsageCriticality,
|
||||
|
||||
#[error("BGPsec router certificate ExtendedKeyUsage must contain id-kp-bgpsec-router ({OID_KP_BGPSEC_ROUTER}) (RFC 8209 §3.1.3.2; RFC 8209 §3.3)")]
|
||||
#[error(
|
||||
"BGPsec router certificate ExtendedKeyUsage must contain id-kp-bgpsec-router ({OID_KP_BGPSEC_ROUTER}) (RFC 8209 §3.1.3.2; RFC 8209 §3.3)"
|
||||
)]
|
||||
MissingBgpsecRouterEku,
|
||||
|
||||
#[error("BGPsec router certificate MUST NOT include Subject Information Access (RFC 8209 §3.1.3.3; RFC 8209 §3.3)")]
|
||||
#[error(
|
||||
"BGPsec router certificate MUST NOT include Subject Information Access (RFC 8209 §3.1.3.3; RFC 8209 §3.3)"
|
||||
)]
|
||||
SubjectInfoAccessPresent,
|
||||
|
||||
#[error("BGPsec router certificate MUST NOT include IP resources extension (RFC 8209 §3.1.3.4; RFC 8209 §3.3)")]
|
||||
#[error(
|
||||
"BGPsec router certificate MUST NOT include IP resources extension (RFC 8209 §3.1.3.4; RFC 8209 §3.3)"
|
||||
)]
|
||||
IpResourcesPresent,
|
||||
|
||||
#[error("BGPsec router certificate MUST include AS resources extension (RFC 8209 §3.1.3.5; RFC 8209 §3.3)")]
|
||||
#[error(
|
||||
"BGPsec router certificate MUST include AS resources extension (RFC 8209 §3.1.3.5; RFC 8209 §3.3)"
|
||||
)]
|
||||
AsResourcesMissing,
|
||||
|
||||
#[error("BGPsec router certificate AS resources MUST include one or more ASNs (RFC 8209 §3.1.3.5)")]
|
||||
#[error(
|
||||
"BGPsec router certificate AS resources MUST include one or more ASNs (RFC 8209 §3.1.3.5)"
|
||||
)]
|
||||
AsResourcesAsnumMissing,
|
||||
|
||||
#[error("BGPsec router certificate AS resources MUST NOT use inherit (RFC 8209 §3.1.3.5)")]
|
||||
AsResourcesInherit,
|
||||
|
||||
#[error("BGPsec router certificate AS resources MUST contain explicit ASNs, not ranges (RFC 8209 §3.1.3.5)")]
|
||||
#[error(
|
||||
"BGPsec router certificate AS resources MUST contain explicit ASNs, not ranges (RFC 8209 §3.1.3.5)"
|
||||
)]
|
||||
AsResourcesRangeNotAllowed,
|
||||
|
||||
#[error("BGPsec router certificate subjectPublicKeyInfo.algorithm must be id-ecPublicKey ({OID_EC_PUBLIC_KEY}) (RFC 8208 §3.1)")]
|
||||
#[error(
|
||||
"BGPsec router certificate subjectPublicKeyInfo.algorithm must be id-ecPublicKey ({OID_EC_PUBLIC_KEY}) (RFC 8208 §3.1)"
|
||||
)]
|
||||
SpkiAlgorithmNotEcPublicKey,
|
||||
|
||||
#[error("BGPsec router certificate subjectPublicKeyInfo.parameters must be secp256r1 ({OID_SECP256R1}) (RFC 8208 §3.1)")]
|
||||
#[error(
|
||||
"BGPsec router certificate subjectPublicKeyInfo.parameters must be secp256r1 ({OID_SECP256R1}) (RFC 8208 §3.1)"
|
||||
)]
|
||||
SpkiWrongCurve,
|
||||
|
||||
#[error("BGPsec router certificate subjectPublicKeyInfo.parameters missing or invalid (RFC 8208 §3.1)")]
|
||||
#[error(
|
||||
"BGPsec router certificate subjectPublicKeyInfo.parameters missing or invalid (RFC 8208 §3.1)"
|
||||
)]
|
||||
SpkiParametersMissingOrInvalid,
|
||||
|
||||
#[error("BGPsec router certificate subjectPublicKey MUST be uncompressed P-256 ECPoint (RFC 8208 §3.1)")]
|
||||
#[error(
|
||||
"BGPsec router certificate subjectPublicKey MUST be uncompressed P-256 ECPoint (RFC 8208 §3.1)"
|
||||
)]
|
||||
SpkiEcPointNotUncompressedP256,
|
||||
}
|
||||
|
||||
@@ -116,16 +139,21 @@ pub enum BgpsecRouterCertificatePathError {
|
||||
}
|
||||
|
||||
impl BgpsecRouterCertificate {
|
||||
pub fn parse_der(der: &[u8]) -> Result<BgpsecRouterCertificateParsed, BgpsecRouterCertificateParseError> {
|
||||
pub fn parse_der(
|
||||
der: &[u8],
|
||||
) -> Result<BgpsecRouterCertificateParsed, BgpsecRouterCertificateParseError> {
|
||||
let (rem, cert) = X509Certificate::from_der(der)
|
||||
.map_err(|e| BgpsecRouterCertificateParseError::X509(e.to_string()))?;
|
||||
if !rem.is_empty() {
|
||||
return Err(BgpsecRouterCertificateParseError::TrailingBytes(rem.len()));
|
||||
}
|
||||
let (spki_rem, _spki) = SubjectPublicKeyInfo::from_der(cert.tbs_certificate.subject_pki.raw)
|
||||
.map_err(|e| BgpsecRouterCertificateParseError::SpkiParse(e.to_string()))?;
|
||||
let (spki_rem, _spki) =
|
||||
SubjectPublicKeyInfo::from_der(cert.tbs_certificate.subject_pki.raw)
|
||||
.map_err(|e| BgpsecRouterCertificateParseError::SpkiParse(e.to_string()))?;
|
||||
if !spki_rem.is_empty() {
|
||||
return Err(BgpsecRouterCertificateParseError::SpkiTrailingBytes(spki_rem.len()));
|
||||
return Err(BgpsecRouterCertificateParseError::SpkiTrailingBytes(
|
||||
spki_rem.len(),
|
||||
));
|
||||
}
|
||||
let rc_parsed = ResourceCertificate::parse_der(der)?;
|
||||
Ok(BgpsecRouterCertificateParsed { rc_parsed })
|
||||
@@ -170,7 +198,9 @@ impl BgpsecRouterCertificate {
|
||||
}
|
||||
|
||||
impl BgpsecRouterCertificateParsed {
|
||||
pub fn validate_profile(self) -> Result<BgpsecRouterCertificate, BgpsecRouterCertificateProfileError> {
|
||||
pub fn validate_profile(
|
||||
self,
|
||||
) -> Result<BgpsecRouterCertificate, BgpsecRouterCertificateProfileError> {
|
||||
let rc = self.rc_parsed.validate_profile()?;
|
||||
if rc.kind != ResourceCertKind::Ee {
|
||||
return Err(BgpsecRouterCertificateProfileError::NotEe);
|
||||
@@ -196,15 +226,17 @@ impl BgpsecRouterCertificateParsed {
|
||||
.ok_or(BgpsecRouterCertificateProfileError::AsResourcesMissing)?;
|
||||
let asns = extract_router_asns(as_resources)?;
|
||||
|
||||
let (rem, cert) = X509Certificate::from_der(&rc.raw_der)
|
||||
.map_err(|e| BgpsecRouterCertificateProfileError::ResourceCertificate(
|
||||
ResourceCertificateProfileError::InvalidCertificatePolicy(e.to_string())
|
||||
))?;
|
||||
let (rem, cert) = X509Certificate::from_der(&rc.raw_der).map_err(|e| {
|
||||
BgpsecRouterCertificateProfileError::ResourceCertificate(
|
||||
ResourceCertificateProfileError::InvalidCertificatePolicy(e.to_string()),
|
||||
)
|
||||
})?;
|
||||
if !rem.is_empty() {
|
||||
return Err(BgpsecRouterCertificateProfileError::ResourceCertificate(
|
||||
ResourceCertificateProfileError::InvalidCertificatePolicy(
|
||||
format!("trailing bytes after router certificate DER: {}", rem.len()),
|
||||
),
|
||||
ResourceCertificateProfileError::InvalidCertificatePolicy(format!(
|
||||
"trailing bytes after router certificate DER: {}",
|
||||
rem.len()
|
||||
)),
|
||||
));
|
||||
}
|
||||
validate_router_eku(&cert)?;
|
||||
@@ -243,7 +275,7 @@ fn extract_router_asns(
|
||||
match item {
|
||||
AsIdOrRange::Id(v) => asns.push(*v),
|
||||
AsIdOrRange::Range { .. } => {
|
||||
return Err(BgpsecRouterCertificateProfileError::AsResourcesRangeNotAllowed)
|
||||
return Err(BgpsecRouterCertificateProfileError::AsResourcesRangeNotAllowed);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -252,7 +284,9 @@ fn extract_router_asns(
|
||||
Ok(asns)
|
||||
}
|
||||
|
||||
fn validate_router_eku(cert: &X509Certificate<'_>) -> Result<(), BgpsecRouterCertificateProfileError> {
|
||||
fn validate_router_eku(
|
||||
cert: &X509Certificate<'_>,
|
||||
) -> Result<(), BgpsecRouterCertificateProfileError> {
|
||||
let mut matches = cert
|
||||
.tbs_certificate
|
||||
.extensions()
|
||||
|
||||
@@ -326,10 +326,7 @@ impl RpkiSignedObject {
|
||||
/// Verify the CMS signature using the embedded EE certificate public key.
|
||||
pub fn verify_signature(&self) -> Result<(), SignedObjectVerifyError> {
|
||||
let ee = &self.signed_data.certificates[0];
|
||||
self.verify_signature_with_rsa_components(
|
||||
&ee.rsa_public_modulus,
|
||||
&ee.rsa_public_exponent,
|
||||
)
|
||||
self.verify_signature_with_rsa_components(&ee.rsa_public_modulus, &ee.rsa_public_exponent)
|
||||
}
|
||||
|
||||
/// Verify the CMS signature using a DER-encoded SubjectPublicKeyInfo.
|
||||
@@ -451,9 +448,7 @@ impl<'a> CmsReader<'a> {
|
||||
let (rem, any) = Any::from_ber(self.buf).map_err(|e| format!("BER parse error: {e}"))?;
|
||||
let tag = header_to_single_byte_tag(&any.header)?;
|
||||
if tag != 0x04 && tag != 0x24 {
|
||||
return Err(format!(
|
||||
"unexpected tag: got 0x{tag:02X}, expected 0x04"
|
||||
));
|
||||
return Err(format!("unexpected tag: got 0x{tag:02X}, expected 0x04"));
|
||||
}
|
||||
let octets = flatten_octet_string(any)?;
|
||||
self.buf = rem;
|
||||
@@ -516,7 +511,9 @@ fn header_to_single_byte_tag(header: &Header<'_>) -> Result<u8, String> {
|
||||
if tag_no > 30 {
|
||||
return Err(format!("high-tag-number form not supported: {tag_no}"));
|
||||
}
|
||||
Ok(((header.class() as u8) << 6) | if header.constructed() { 0x20 } else { 0x00 } | tag_no as u8)
|
||||
Ok(((header.class() as u8) << 6)
|
||||
| if header.constructed() { 0x20 } else { 0x00 }
|
||||
| tag_no as u8)
|
||||
}
|
||||
|
||||
fn cms_take_tlv(input: &[u8]) -> Result<(u8, &[u8], &[u8]), String> {
|
||||
@@ -759,20 +756,18 @@ fn validate_ee_certificate(der: &[u8]) -> Result<ResourceEeCertificate, SignedOb
|
||||
"trailing bytes after EE SubjectPublicKeyInfo DER".to_string(),
|
||||
));
|
||||
}
|
||||
let parsed_pk = spki
|
||||
.parsed()
|
||||
.map_err(|_e| SignedObjectValidateError::EeCertificateParse(
|
||||
let parsed_pk = spki.parsed().map_err(|_e| {
|
||||
SignedObjectValidateError::EeCertificateParse(
|
||||
"unsupported EE public key algorithm".to_string(),
|
||||
))?;
|
||||
)
|
||||
})?;
|
||||
let (rsa_public_modulus, rsa_public_exponent) = match parsed_pk {
|
||||
PublicKey::RSA(rsa) => {
|
||||
let modulus = strip_leading_zeros(rsa.modulus).to_vec();
|
||||
let exponent = strip_leading_zeros(rsa.exponent).to_vec();
|
||||
let _ = rsa
|
||||
.try_exponent()
|
||||
.map_err(|_e| SignedObjectValidateError::EeCertificateParse(
|
||||
"invalid EE RSA exponent".to_string(),
|
||||
))?;
|
||||
let _ = rsa.try_exponent().map_err(|_e| {
|
||||
SignedObjectValidateError::EeCertificateParse("invalid EE RSA exponent".to_string())
|
||||
})?;
|
||||
(modulus, exponent)
|
||||
}
|
||||
_ => {
|
||||
|
||||
@@ -7,6 +7,8 @@ use crate::sync::rrdp::Fetcher;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct HttpFetcherConfig {
|
||||
/// Connection-establishment timeout for HTTP requests.
|
||||
pub connect_timeout: Duration,
|
||||
/// Short timeout used for connection establishment and small metadata objects.
|
||||
pub timeout: Duration,
|
||||
/// Larger timeout used for RRDP snapshot / delta bodies.
|
||||
@@ -17,7 +19,8 @@ pub struct HttpFetcherConfig {
|
||||
impl Default for HttpFetcherConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
timeout: Duration::from_secs(20),
|
||||
connect_timeout: Duration::from_secs(15),
|
||||
timeout: Duration::from_secs(30),
|
||||
large_body_timeout: Duration::from_secs(180),
|
||||
user_agent: "rpki-dev/0.1 (stage2)".to_string(),
|
||||
}
|
||||
@@ -41,14 +44,15 @@ impl BlockingHttpFetcher {
|
||||
pub fn new(config: HttpFetcherConfig) -> Result<Self, String> {
|
||||
let short_timeout = config.timeout;
|
||||
let large_body_timeout = std::cmp::max(config.large_body_timeout, config.timeout);
|
||||
let connect_timeout = std::cmp::min(config.connect_timeout, config.timeout);
|
||||
let short_client = Client::builder()
|
||||
.connect_timeout(config.timeout)
|
||||
.connect_timeout(connect_timeout)
|
||||
.timeout(config.timeout)
|
||||
.user_agent(config.user_agent.clone())
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let large_body_client = Client::builder()
|
||||
.connect_timeout(config.timeout)
|
||||
.connect_timeout(connect_timeout)
|
||||
.timeout(large_body_timeout)
|
||||
.user_agent(config.user_agent)
|
||||
.build()
|
||||
@@ -64,24 +68,21 @@ impl BlockingHttpFetcher {
|
||||
pub fn fetch_bytes(&self, uri: &str) -> Result<Vec<u8>, String> {
|
||||
let started = std::time::Instant::now();
|
||||
let (client, timeout_profile, timeout_value) = self.client_for_uri(uri);
|
||||
let resp = client
|
||||
.get(uri)
|
||||
.send()
|
||||
.map_err(|e| {
|
||||
let msg = format!("http request failed: {e}");
|
||||
crate::progress_log::emit(
|
||||
"http_fetch_failed",
|
||||
serde_json::json!({
|
||||
"uri": uri,
|
||||
"stage": "request",
|
||||
"timeout_profile": timeout_profile,
|
||||
"request_timeout_ms": timeout_value.as_millis() as u64,
|
||||
"duration_ms": started.elapsed().as_millis() as u64,
|
||||
"error": msg,
|
||||
}),
|
||||
);
|
||||
msg
|
||||
})?;
|
||||
let resp = client.get(uri).send().map_err(|e| {
|
||||
let msg = format!("http request failed: {e}");
|
||||
crate::progress_log::emit(
|
||||
"http_fetch_failed",
|
||||
serde_json::json!({
|
||||
"uri": uri,
|
||||
"stage": "request",
|
||||
"timeout_profile": timeout_profile,
|
||||
"request_timeout_ms": timeout_value.as_millis() as u64,
|
||||
"duration_ms": started.elapsed().as_millis() as u64,
|
||||
"error": msg,
|
||||
}),
|
||||
);
|
||||
msg
|
||||
})?;
|
||||
|
||||
let status = resp.status();
|
||||
let headers = resp.headers().clone();
|
||||
@@ -175,7 +176,11 @@ impl BlockingHttpFetcher {
|
||||
|
||||
fn client_for_uri(&self, uri: &str) -> (&Client, &'static str, Duration) {
|
||||
if uses_large_body_timeout(uri) {
|
||||
(&self.large_body_client, "large_body", self.large_body_timeout)
|
||||
(
|
||||
&self.large_body_client,
|
||||
"large_body",
|
||||
self.large_body_timeout,
|
||||
)
|
||||
} else {
|
||||
(&self.short_client, "short", self.short_timeout)
|
||||
}
|
||||
@@ -200,9 +205,7 @@ fn header_value_opt(headers: &HeaderMap, name: &str) -> Option<String> {
|
||||
}
|
||||
|
||||
fn uses_large_body_timeout(uri: &str) -> bool {
|
||||
uri.starts_with("https://")
|
||||
&& uri.ends_with(".xml")
|
||||
&& !uri.ends_with("notification.xml")
|
||||
uri.starts_with("https://") && uri.ends_with(".xml") && !uri.ends_with("notification.xml")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -280,13 +283,17 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn uses_large_body_timeout_selects_rrdp_snapshot_and_delta_not_notification() {
|
||||
assert!(!uses_large_body_timeout("https://rrdp.example.test/notification.xml"));
|
||||
assert!(!uses_large_body_timeout(
|
||||
"https://rrdp.example.test/notification.xml"
|
||||
));
|
||||
assert!(uses_large_body_timeout(
|
||||
"https://rrdp.example.test/session/123/snapshot.xml"
|
||||
));
|
||||
assert!(uses_large_body_timeout(
|
||||
"https://rrdp.example.test/session/123/delta-42.xml"
|
||||
));
|
||||
assert!(!uses_large_body_timeout("https://tal.example.test/example.tal"));
|
||||
assert!(!uses_large_body_timeout(
|
||||
"https://tal.example.test/example.tal"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
use std::cell::RefCell;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::process::Stdio;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
|
||||
use sha2::Digest;
|
||||
use uuid::Uuid;
|
||||
@@ -12,6 +16,7 @@ use crate::fetch::rsync::{
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SystemRsyncConfig {
|
||||
pub rsync_bin: PathBuf,
|
||||
pub connect_timeout: Duration,
|
||||
pub timeout: Duration,
|
||||
pub extra_args: Vec<String>,
|
||||
/// Optional root directory for persistent rsync mirrors.
|
||||
@@ -28,7 +33,8 @@ impl Default for SystemRsyncConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
rsync_bin: PathBuf::from("rsync"),
|
||||
timeout: Duration::from_secs(60),
|
||||
connect_timeout: Duration::from_secs(15),
|
||||
timeout: Duration::from_secs(30),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: None,
|
||||
}
|
||||
@@ -44,6 +50,39 @@ pub struct SystemRsyncFetcher {
|
||||
config: SystemRsyncConfig,
|
||||
}
|
||||
|
||||
thread_local! {
|
||||
static RSYNC_TIMEOUT_OVERRIDE: RefCell<Option<Duration>> = const { RefCell::new(None) };
|
||||
static RSYNC_FAIL_FAST_PROFILE: RefCell<Option<RsyncFailFastProfile>> = const { RefCell::new(None) };
|
||||
}
|
||||
|
||||
pub fn with_scoped_rsync_timeout_override<R>(timeout: Duration, f: impl FnOnce() -> R) -> R {
|
||||
RSYNC_TIMEOUT_OVERRIDE.with(|cell| {
|
||||
let previous = cell.replace(Some(timeout));
|
||||
let result = f();
|
||||
let _ = cell.replace(previous);
|
||||
result
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct RsyncFailFastProfile {
|
||||
pub initial_wall_clock_timeout: Duration,
|
||||
pub max_wall_clock_timeout: Duration,
|
||||
pub max_attempts: usize,
|
||||
}
|
||||
|
||||
pub fn with_scoped_rsync_fail_fast_profile<R>(
|
||||
profile: RsyncFailFastProfile,
|
||||
f: impl FnOnce() -> R,
|
||||
) -> R {
|
||||
RSYNC_FAIL_FAST_PROFILE.with(|cell| {
|
||||
let previous = cell.replace(Some(profile));
|
||||
let result = f();
|
||||
let _ = cell.replace(previous);
|
||||
result
|
||||
})
|
||||
}
|
||||
|
||||
impl SystemRsyncFetcher {
|
||||
pub fn new(config: SystemRsyncConfig) -> Self {
|
||||
Self { config }
|
||||
@@ -69,32 +108,164 @@ impl SystemRsyncFetcher {
|
||||
}
|
||||
|
||||
fn run_rsync(&self, src: &str, dst: &Path) -> Result<(), String> {
|
||||
let fail_fast = RSYNC_FAIL_FAST_PROFILE.with(|cell| *cell.borrow());
|
||||
if let Some(profile) = fail_fast {
|
||||
return self.run_rsync_fail_fast(src, dst, profile);
|
||||
}
|
||||
self.run_rsync_once(src, dst, None, false)
|
||||
}
|
||||
|
||||
fn run_rsync_once(
|
||||
&self,
|
||||
src: &str,
|
||||
dst: &Path,
|
||||
wall_clock_timeout: Option<Duration>,
|
||||
keep_partial: bool,
|
||||
) -> Result<(), String> {
|
||||
// `--timeout` is I/O timeout in seconds (applies to network reads/writes).
|
||||
let timeout_secs = self.config.timeout.as_secs().max(1).to_string();
|
||||
let timeout =
|
||||
RSYNC_TIMEOUT_OVERRIDE.with(|cell| cell.borrow().unwrap_or(self.config.timeout));
|
||||
let connect_timeout_secs = self.config.connect_timeout.as_secs().max(1).to_string();
|
||||
let timeout_secs = timeout.as_secs().max(1).to_string();
|
||||
let is_remote_rsync = src.starts_with("rsync://");
|
||||
|
||||
let mut cmd = Command::new(&self.config.rsync_bin);
|
||||
cmd.arg("-rt")
|
||||
.arg("--delete")
|
||||
.arg("--timeout")
|
||||
.arg(timeout_secs)
|
||||
.args(&self.config.extra_args)
|
||||
.arg(src)
|
||||
.arg(dst);
|
||||
|
||||
let out = cmd
|
||||
.output()
|
||||
.map_err(|e| format!("rsync spawn failed: {e}"))?;
|
||||
if !out.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
return Err(format!(
|
||||
"rsync failed: status={} stdout={} stderr={}",
|
||||
out.status,
|
||||
stdout.trim(),
|
||||
stderr.trim()
|
||||
));
|
||||
.args(&self.config.extra_args);
|
||||
if is_remote_rsync {
|
||||
cmd.arg("--contimeout").arg(connect_timeout_secs);
|
||||
}
|
||||
if keep_partial {
|
||||
cmd.arg("--partial");
|
||||
}
|
||||
cmd.arg(src)
|
||||
.arg(dst)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
|
||||
let mut child = cmd
|
||||
.spawn()
|
||||
.map_err(|e| format!("rsync spawn failed: {e}"))?;
|
||||
if let Some(limit) = wall_clock_timeout {
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
match child
|
||||
.try_wait()
|
||||
.map_err(|e| format!("rsync wait failed: {e}"))?
|
||||
{
|
||||
Some(_status) => {
|
||||
let out = child
|
||||
.wait_with_output()
|
||||
.map_err(|e| format!("rsync wait_with_output failed: {e}"))?;
|
||||
if out.status.success() {
|
||||
return Ok(());
|
||||
}
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
return Err(format!(
|
||||
"rsync failed: status={} stdout={} stderr={}",
|
||||
out.status,
|
||||
stdout.trim(),
|
||||
stderr.trim()
|
||||
));
|
||||
}
|
||||
None => {
|
||||
if started.elapsed() >= limit {
|
||||
let _ = child.kill();
|
||||
let out = child
|
||||
.wait_with_output()
|
||||
.map_err(|e| format!("rsync wait_with_output failed: {e}"))?;
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
return Err(format!(
|
||||
"rsync wall-clock timeout after {}s: stdout={} stderr={}",
|
||||
limit.as_secs(),
|
||||
stdout.trim(),
|
||||
stderr.trim()
|
||||
));
|
||||
}
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let out = child
|
||||
.wait_with_output()
|
||||
.map_err(|e| format!("rsync wait_with_output failed: {e}"))?;
|
||||
if out.status.success() {
|
||||
return Ok(());
|
||||
}
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
Err(format!(
|
||||
"rsync failed: status={} stdout={} stderr={}",
|
||||
out.status,
|
||||
stdout.trim(),
|
||||
stderr.trim()
|
||||
))
|
||||
}
|
||||
|
||||
fn run_rsync_fail_fast(
|
||||
&self,
|
||||
src: &str,
|
||||
dst: &Path,
|
||||
profile: RsyncFailFastProfile,
|
||||
) -> Result<(), String> {
|
||||
let mut attempt = 0usize;
|
||||
let mut timeout = profile.initial_wall_clock_timeout;
|
||||
let mut previous_progress = (0usize, 0u64);
|
||||
let mut zero_progress_attempts = 0usize;
|
||||
let max_timeout = std::cmp::max(
|
||||
profile.max_wall_clock_timeout,
|
||||
profile.initial_wall_clock_timeout,
|
||||
);
|
||||
|
||||
loop {
|
||||
attempt += 1;
|
||||
match self.run_rsync_once(src, dst, Some(timeout), true) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(err) => {
|
||||
if is_hard_fail_rsync_error(&err) {
|
||||
return Err(format!(
|
||||
"rsync fail-fast hard-fail on attempt {}: {}",
|
||||
attempt, err
|
||||
));
|
||||
}
|
||||
if !err.contains("wall-clock timeout") {
|
||||
return Err(err);
|
||||
}
|
||||
let progress = dir_progress(dst)
|
||||
.map_err(|e| format!("rsync fail-fast progress stat failed: {e}"))?;
|
||||
if progress == (0, 0) {
|
||||
zero_progress_attempts += 1;
|
||||
if zero_progress_attempts >= 2 || attempt >= profile.max_attempts {
|
||||
return Err(format!(
|
||||
"rsync fail-fast gave up after {} attempts with no progress: {}",
|
||||
attempt, err
|
||||
));
|
||||
}
|
||||
} else if progress == previous_progress {
|
||||
return Err(format!(
|
||||
"rsync fail-fast gave up after {} attempts with no additional progress: {}",
|
||||
attempt, err
|
||||
));
|
||||
} else {
|
||||
previous_progress = progress;
|
||||
}
|
||||
|
||||
if attempt >= profile.max_attempts {
|
||||
return Err(format!(
|
||||
"rsync fail-fast exhausted {} attempts: {}",
|
||||
profile.max_attempts, err
|
||||
));
|
||||
}
|
||||
timeout = std::cmp::min(timeout.saturating_mul(2), max_timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn module_fetch_uri(&self, rsync_base_uri: &str) -> String {
|
||||
@@ -140,7 +311,8 @@ impl RsyncFetcher for SystemRsyncFetcher {
|
||||
let tmp = TempDir::new().map_err(|e| RsyncFetchError::Fetch(e.to_string()))?;
|
||||
self.run_rsync(&base, tmp.path())
|
||||
.map_err(RsyncFetchError::Fetch)?;
|
||||
walk_dir_visit(tmp.path(), tmp.path(), &base, &mut wrapped).map_err(RsyncFetchError::Fetch)?;
|
||||
walk_dir_visit(tmp.path(), tmp.path(), &base, &mut wrapped)
|
||||
.map_err(RsyncFetchError::Fetch)?;
|
||||
Ok((count, bytes_total))
|
||||
}
|
||||
|
||||
@@ -178,7 +350,10 @@ fn rsync_module_root_uri(s: &str) -> Option<String> {
|
||||
let mut host_and_path = rest.splitn(2, '/');
|
||||
let authority = host_and_path.next()?;
|
||||
let path = host_and_path.next()?;
|
||||
let mut segments: Vec<&str> = path.split('/').filter(|segment| !segment.is_empty()).collect();
|
||||
let mut segments: Vec<&str> = path
|
||||
.split('/')
|
||||
.filter(|segment| !segment.is_empty())
|
||||
.collect();
|
||||
if segments.is_empty() {
|
||||
return None;
|
||||
}
|
||||
@@ -216,6 +391,38 @@ fn walk_dir_collect(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn dir_progress(root: &Path) -> Result<(usize, u64), String> {
|
||||
if !root.exists() {
|
||||
return Ok((0, 0));
|
||||
}
|
||||
let mut files = 0usize;
|
||||
let mut bytes = 0u64;
|
||||
let mut stack = vec![root.to_path_buf()];
|
||||
while let Some(path) = stack.pop() {
|
||||
let rd = std::fs::read_dir(&path).map_err(|e| e.to_string())?;
|
||||
for entry in rd {
|
||||
let entry = entry.map_err(|e| e.to_string())?;
|
||||
let path = entry.path();
|
||||
let meta = entry.metadata().map_err(|e| e.to_string())?;
|
||||
if meta.is_dir() {
|
||||
stack.push(path);
|
||||
} else if meta.is_file() {
|
||||
files += 1;
|
||||
bytes += meta.len();
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok((files, bytes))
|
||||
}
|
||||
|
||||
fn is_hard_fail_rsync_error(msg: &str) -> bool {
|
||||
let lower = msg.to_ascii_lowercase();
|
||||
lower.contains("no route to host")
|
||||
|| lower.contains("network is unreachable")
|
||||
|| lower.contains("connection refused")
|
||||
|| lower.contains("name or service not known")
|
||||
}
|
||||
|
||||
fn walk_dir_visit(
|
||||
root: &Path,
|
||||
current: &Path,
|
||||
@@ -314,6 +521,7 @@ mod tests {
|
||||
// 1) Spawn error.
|
||||
let f = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
rsync_bin: PathBuf::from("/this/does/not/exist/rsync"),
|
||||
connect_timeout: Duration::from_secs(1),
|
||||
timeout: Duration::from_secs(1),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: None,
|
||||
@@ -326,6 +534,7 @@ mod tests {
|
||||
// 2) Non-zero exit status.
|
||||
let f = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
rsync_bin: PathBuf::from("false"),
|
||||
connect_timeout: Duration::from_secs(1),
|
||||
timeout: Duration::from_secs(1),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: None,
|
||||
@@ -345,6 +554,7 @@ mod tests {
|
||||
|
||||
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
rsync_bin: PathBuf::from("rsync"),
|
||||
connect_timeout: Duration::from_secs(1),
|
||||
timeout: Duration::from_secs(1),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: Some(root_file.clone()),
|
||||
@@ -371,6 +581,7 @@ mod tests {
|
||||
|
||||
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
rsync_bin: PathBuf::from("rsync"),
|
||||
connect_timeout: Duration::from_secs(1),
|
||||
timeout: Duration::from_secs(1),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: Some(root.clone()),
|
||||
@@ -406,4 +617,168 @@ mod tests {
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].0, "rsync://example.net/repo/a.cer");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn rsync_fail_fast_retries_when_progress_is_made() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let script = temp.path().join("fake-rsync.sh");
|
||||
let state = temp.path().join("state.txt");
|
||||
std::fs::write(
|
||||
&script,
|
||||
format!(
|
||||
"#!/usr/bin/env bash\nset -euo pipefail\nSTATE=\"{}\"\nDST=\"${{@: -1}}\"\nCOUNT=0\nif [[ -f \"$STATE\" ]]; then COUNT=$(cat \"$STATE\"); fi\nCOUNT=$((COUNT+1))\necho \"$COUNT\" > \"$STATE\"\nmkdir -p \"$DST\"\nif [[ \"$COUNT\" -eq 1 ]]; then\n echo first > \"$DST/part1\"\n sleep 2\nelse\n echo second > \"$DST/part2\"\nfi\n",
|
||||
state.display()
|
||||
),
|
||||
)
|
||||
.expect("write script");
|
||||
let mut perms = std::fs::metadata(&script).unwrap().permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&script, perms).unwrap();
|
||||
|
||||
let dst = temp.path().join("dst");
|
||||
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
rsync_bin: script,
|
||||
connect_timeout: Duration::from_secs(15),
|
||||
timeout: Duration::from_secs(60),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: None,
|
||||
});
|
||||
|
||||
fetcher
|
||||
.run_rsync_fail_fast(
|
||||
"rsync://example.net/repo/",
|
||||
&dst,
|
||||
RsyncFailFastProfile {
|
||||
initial_wall_clock_timeout: Duration::from_secs(1),
|
||||
max_wall_clock_timeout: Duration::from_secs(4),
|
||||
max_attempts: 3,
|
||||
},
|
||||
)
|
||||
.expect("eventual success");
|
||||
|
||||
assert!(dst.join("part1").exists());
|
||||
assert!(dst.join("part2").exists());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn rsync_fail_fast_gives_up_after_two_zero_progress_timeouts() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let script = temp.path().join("fake-rsync.sh");
|
||||
std::fs::write(&script, "#!/usr/bin/env bash\nset -euo pipefail\nsleep 5\n")
|
||||
.expect("write script");
|
||||
let mut perms = std::fs::metadata(&script).unwrap().permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&script, perms).unwrap();
|
||||
|
||||
let dst = temp.path().join("dst");
|
||||
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
rsync_bin: script,
|
||||
connect_timeout: Duration::from_secs(15),
|
||||
timeout: Duration::from_secs(60),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: None,
|
||||
});
|
||||
|
||||
let err = fetcher
|
||||
.run_rsync_fail_fast(
|
||||
"rsync://example.net/repo/",
|
||||
&dst,
|
||||
RsyncFailFastProfile {
|
||||
initial_wall_clock_timeout: Duration::from_secs(1),
|
||||
max_wall_clock_timeout: Duration::from_secs(2),
|
||||
max_attempts: 4,
|
||||
},
|
||||
)
|
||||
.expect_err("must fail");
|
||||
assert!(err.contains("no progress"), "{err}");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn rsync_fail_fast_hard_fail_stops_after_first_attempt() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let script = temp.path().join("fake-rsync.sh");
|
||||
let state = temp.path().join("state.txt");
|
||||
std::fs::write(
|
||||
&script,
|
||||
format!(
|
||||
"#!/usr/bin/env bash\nset -euo pipefail\nSTATE=\"{}\"\nCOUNT=0\nif [[ -f \"$STATE\" ]]; then COUNT=$(cat \"$STATE\"); fi\nCOUNT=$((COUNT+1))\necho \"$COUNT\" > \"$STATE\"\necho 'rsync: [Receiver] failed to connect to host (1.2.3.4): Connection refused (111)' >&2\nexit 10\n",
|
||||
state.display()
|
||||
),
|
||||
)
|
||||
.expect("write script");
|
||||
let mut perms = std::fs::metadata(&script).unwrap().permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&script, perms).unwrap();
|
||||
|
||||
let dst = temp.path().join("dst");
|
||||
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
rsync_bin: script,
|
||||
connect_timeout: Duration::from_secs(15),
|
||||
timeout: Duration::from_secs(60),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: None,
|
||||
});
|
||||
|
||||
let err = fetcher
|
||||
.run_rsync_fail_fast(
|
||||
"rsync://example.net/repo/",
|
||||
&dst,
|
||||
RsyncFailFastProfile {
|
||||
initial_wall_clock_timeout: Duration::from_secs(10),
|
||||
max_wall_clock_timeout: Duration::from_secs(80),
|
||||
max_attempts: 4,
|
||||
},
|
||||
)
|
||||
.expect_err("must hard fail");
|
||||
assert!(err.contains("hard-fail"), "{err}");
|
||||
let count = std::fs::read_to_string(&state).unwrap();
|
||||
assert_eq!(count.trim(), "1");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn run_rsync_once_passes_contimeout_and_timeout_args() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let script = temp.path().join("capture-rsync.sh");
|
||||
let args_file = temp.path().join("args.txt");
|
||||
std::fs::write(
|
||||
&script,
|
||||
format!(
|
||||
"#!/usr/bin/env bash\nset -euo pipefail\nprintf '%s\\n' \"$@\" > \"{}\"\nDST=\"${{@: -1}}\"\nmkdir -p \"$DST\"\n",
|
||||
args_file.display()
|
||||
),
|
||||
)
|
||||
.expect("write script");
|
||||
let mut perms = std::fs::metadata(&script).unwrap().permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&script, perms).unwrap();
|
||||
|
||||
let dst = temp.path().join("dst");
|
||||
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
rsync_bin: script,
|
||||
connect_timeout: Duration::from_secs(15),
|
||||
timeout: Duration::from_secs(30),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: None,
|
||||
});
|
||||
|
||||
fetcher
|
||||
.run_rsync_once("rsync://example.net/repo/", &dst, None, false)
|
||||
.expect("rsync");
|
||||
|
||||
let args = std::fs::read_to_string(&args_file).expect("read args");
|
||||
assert!(args.contains("--contimeout\n15\n"), "{args}");
|
||||
assert!(args.contains("--timeout\n30\n"), "{args}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,9 +3,9 @@ pub mod cir;
|
||||
pub mod data_model;
|
||||
|
||||
#[cfg(feature = "full")]
|
||||
pub mod analysis;
|
||||
pub mod blob_store;
|
||||
#[cfg(feature = "full")]
|
||||
pub mod bundle;
|
||||
pub mod analysis;
|
||||
#[cfg(feature = "full")]
|
||||
pub mod audit;
|
||||
#[cfg(feature = "full")]
|
||||
@@ -13,6 +13,8 @@ pub mod audit_downloads;
|
||||
#[cfg(feature = "full")]
|
||||
pub mod audit_trace;
|
||||
#[cfg(feature = "full")]
|
||||
pub mod bundle;
|
||||
#[cfg(feature = "full")]
|
||||
pub mod cli;
|
||||
#[cfg(feature = "full")]
|
||||
pub mod fetch;
|
||||
|
||||
@@ -874,9 +874,13 @@ mod tests {
|
||||
.join("meta.json");
|
||||
std::fs::remove_file(&meta_path).expect("remove rsync module meta");
|
||||
|
||||
let index = ReplayArchiveIndex::load_allow_missing_rsync_modules(&archive_root, &locks_path)
|
||||
.expect("load replay index without rsync meta");
|
||||
let module = index.rsync_modules.get(&module_uri).expect("module present");
|
||||
let index =
|
||||
ReplayArchiveIndex::load_allow_missing_rsync_modules(&archive_root, &locks_path)
|
||||
.expect("load replay index without rsync meta");
|
||||
let module = index
|
||||
.rsync_modules
|
||||
.get(&module_uri)
|
||||
.expect("module present");
|
||||
assert_eq!(module.meta.module, module_uri);
|
||||
assert_eq!(module.meta.version, 1);
|
||||
}
|
||||
|
||||
@@ -916,7 +916,10 @@ mod tests {
|
||||
|
||||
let index = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path)
|
||||
.expect("load delta replay index without rsync meta");
|
||||
let module = index.rsync_modules.get(&module_uri).expect("module present");
|
||||
let module = index
|
||||
.rsync_modules
|
||||
.get(&module_uri)
|
||||
.expect("module present");
|
||||
assert_eq!(module.meta.module, module_uri);
|
||||
assert_eq!(module.meta.version, 1);
|
||||
}
|
||||
|
||||
140
src/storage.rs
140
src/storage.rs
@@ -8,6 +8,7 @@ use rocksdb::{
|
||||
use serde::{Deserialize, Serialize, de::DeserializeOwned};
|
||||
use sha2::Digest;
|
||||
|
||||
use crate::blob_store::{ExternalRawStoreDb, RawObjectStore};
|
||||
use crate::data_model::rc::{AsResourceSet, IpResourceSet};
|
||||
|
||||
pub const CF_REPOSITORY_VIEW: &str = "repository_view";
|
||||
@@ -77,6 +78,7 @@ pub type StorageResult<T> = Result<T, StorageError>;
|
||||
|
||||
pub struct RocksStore {
|
||||
db: DB,
|
||||
external_raw_store: Option<ExternalRawStoreDb>,
|
||||
}
|
||||
|
||||
pub mod pack {
|
||||
@@ -769,7 +771,19 @@ impl RocksStore {
|
||||
let db = DB::open_cf_descriptors(&base_opts, path, column_family_descriptors())
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
|
||||
Ok(Self { db })
|
||||
Ok(Self {
|
||||
db,
|
||||
external_raw_store: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn open_with_external_raw_store(
|
||||
path: &Path,
|
||||
raw_store_path: &Path,
|
||||
) -> StorageResult<Self> {
|
||||
let mut store = Self::open(path)?;
|
||||
store.external_raw_store = Some(ExternalRawStoreDb::open(raw_store_path)?);
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
fn cf(&self, name: &'static str) -> StorageResult<&ColumnFamily> {
|
||||
@@ -822,7 +836,10 @@ impl RocksStore {
|
||||
member_records: &[RrdpSourceMemberRecord],
|
||||
owner_records: &[RrdpUriOwnerRecord],
|
||||
) -> StorageResult<()> {
|
||||
if repository_view_entries.is_empty() && member_records.is_empty() && owner_records.is_empty() {
|
||||
if repository_view_entries.is_empty()
|
||||
&& member_records.is_empty()
|
||||
&& owner_records.is_empty()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -877,6 +894,9 @@ impl RocksStore {
|
||||
|
||||
pub fn put_raw_by_hash_entry(&self, entry: &RawByHashEntry) -> StorageResult<()> {
|
||||
entry.validate_internal()?;
|
||||
if let Some(raw_store) = self.external_raw_store.as_ref() {
|
||||
return raw_store.put_raw_entry(entry);
|
||||
}
|
||||
let cf = self.cf(CF_RAW_BY_HASH)?;
|
||||
let key = raw_by_hash_key(&entry.sha256_hex);
|
||||
let value = encode_cbor(entry, "raw_by_hash")?;
|
||||
@@ -890,6 +910,9 @@ impl RocksStore {
|
||||
if entries.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(raw_store) = self.external_raw_store.as_ref() {
|
||||
return raw_store.put_raw_entries_batch(entries);
|
||||
}
|
||||
|
||||
let cf = self.cf(CF_RAW_BY_HASH)?;
|
||||
let mut batch = WriteBatch::default();
|
||||
@@ -909,6 +932,9 @@ impl RocksStore {
|
||||
if entries.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(raw_store) = self.external_raw_store.as_ref() {
|
||||
return raw_store.put_raw_entries_batch(entries);
|
||||
}
|
||||
|
||||
let cf = self.cf(CF_RAW_BY_HASH)?;
|
||||
let mut batch = WriteBatch::default();
|
||||
@@ -922,6 +948,9 @@ impl RocksStore {
|
||||
|
||||
pub fn delete_raw_by_hash_entry(&self, sha256_hex: &str) -> StorageResult<()> {
|
||||
validate_sha256_hex("raw_by_hash.sha256_hex", sha256_hex)?;
|
||||
if let Some(raw_store) = self.external_raw_store.as_ref() {
|
||||
return raw_store.delete_raw_entry(sha256_hex);
|
||||
}
|
||||
let cf = self.cf(CF_RAW_BY_HASH)?;
|
||||
let key = raw_by_hash_key(sha256_hex);
|
||||
self.db
|
||||
@@ -931,6 +960,9 @@ impl RocksStore {
|
||||
}
|
||||
|
||||
pub fn get_raw_by_hash_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>> {
|
||||
if let Some(raw_store) = self.external_raw_store.as_ref() {
|
||||
return raw_store.get_raw_entry(sha256_hex);
|
||||
}
|
||||
let cf = self.cf(CF_RAW_BY_HASH)?;
|
||||
let key = raw_by_hash_key(sha256_hex);
|
||||
let Some(bytes) = self
|
||||
@@ -952,9 +984,15 @@ impl RocksStore {
|
||||
if sha256_hexes.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
if let Some(raw_store) = self.external_raw_store.as_ref() {
|
||||
return raw_store.get_raw_entries_batch(sha256_hexes);
|
||||
}
|
||||
|
||||
let cf = self.cf(CF_RAW_BY_HASH)?;
|
||||
let keys: Vec<String> = sha256_hexes.iter().map(|hash| raw_by_hash_key(hash)).collect();
|
||||
let keys: Vec<String> = sha256_hexes
|
||||
.iter()
|
||||
.map(|hash| raw_by_hash_key(hash))
|
||||
.collect();
|
||||
self.db
|
||||
.multi_get_cf(keys.iter().map(|key| (cf, key.as_bytes())))
|
||||
.into_iter()
|
||||
@@ -1217,7 +1255,10 @@ impl RocksStore {
|
||||
))
|
||||
}
|
||||
|
||||
pub fn load_current_object_bytes_by_uri(&self, rsync_uri: &str) -> StorageResult<Option<Vec<u8>>> {
|
||||
pub fn load_current_object_bytes_by_uri(
|
||||
&self,
|
||||
rsync_uri: &str,
|
||||
) -> StorageResult<Option<Vec<u8>>> {
|
||||
let Some(view) = self.get_repository_view_entry(rsync_uri)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -1230,17 +1271,15 @@ impl RocksStore {
|
||||
.as_deref()
|
||||
.ok_or(StorageError::InvalidData {
|
||||
entity: "repository_view",
|
||||
detail: format!(
|
||||
"current_hash missing for current object URI: {rsync_uri}"
|
||||
),
|
||||
detail: format!("current_hash missing for current object URI: {rsync_uri}"),
|
||||
})?;
|
||||
let raw = self.get_raw_by_hash_entry(hash)?.ok_or(StorageError::InvalidData {
|
||||
let bytes = self.get_blob_bytes(hash)?.ok_or(StorageError::InvalidData {
|
||||
entity: "repository_view",
|
||||
detail: format!(
|
||||
"raw_by_hash entry missing for current object URI: {rsync_uri} (hash={hash})"
|
||||
),
|
||||
})?;
|
||||
Ok(Some(raw.bytes))
|
||||
Ok(Some(bytes))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1283,7 +1322,6 @@ impl RocksStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
#[allow(dead_code)]
|
||||
|
||||
pub fn write_batch(&self, batch: WriteBatch) -> StorageResult<()> {
|
||||
@@ -1721,6 +1759,37 @@ mod tests {
|
||||
assert_eq!(got_raw, raw);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_by_hash_routes_to_external_raw_store_when_configured() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let main_db = td.path().join("main-db");
|
||||
let raw_db = td.path().join("raw-store.db");
|
||||
|
||||
let raw = sample_raw_by_hash_entry(b"external-raw".to_vec());
|
||||
{
|
||||
let store =
|
||||
RocksStore::open_with_external_raw_store(&main_db, &raw_db).expect("open store");
|
||||
store
|
||||
.put_raw_by_hash_entry(&raw)
|
||||
.expect("put external raw");
|
||||
|
||||
let got = store
|
||||
.get_raw_by_hash_entry(&raw.sha256_hex)
|
||||
.expect("get external raw")
|
||||
.expect("raw exists");
|
||||
assert_eq!(got, raw);
|
||||
}
|
||||
|
||||
let main_store = RocksStore::open(&main_db).expect("open main only");
|
||||
assert!(
|
||||
main_store
|
||||
.get_raw_by_hash_entry(&raw.sha256_hex)
|
||||
.expect("read main store")
|
||||
.is_none(),
|
||||
"main db should not contain raw entry when external raw store is configured"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repository_view_and_raw_by_hash_validation_errors_are_reported() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
@@ -1880,10 +1949,15 @@ mod tests {
|
||||
store
|
||||
.replace_vcir_and_audit_rule_indexes(None, &previous)
|
||||
.expect("store previous vcir");
|
||||
assert!(store
|
||||
.get_audit_rule_index_entry(AuditRuleKind::Roa, &previous.local_outputs[0].rule_hash)
|
||||
.expect("get old audit entry")
|
||||
.is_some());
|
||||
assert!(
|
||||
store
|
||||
.get_audit_rule_index_entry(
|
||||
AuditRuleKind::Roa,
|
||||
&previous.local_outputs[0].rule_hash
|
||||
)
|
||||
.expect("get old audit entry")
|
||||
.is_some()
|
||||
);
|
||||
|
||||
let mut current = sample_vcir("rsync://example.test/repo/current.mft");
|
||||
current.local_outputs = vec![VcirLocalOutput {
|
||||
@@ -1909,14 +1983,24 @@ mod tests {
|
||||
.expect("get replaced vcir")
|
||||
.expect("vcir exists");
|
||||
assert_eq!(got, current);
|
||||
assert!(store
|
||||
.get_audit_rule_index_entry(AuditRuleKind::Roa, &previous.local_outputs[0].rule_hash)
|
||||
.expect("get deleted old audit entry")
|
||||
.is_none());
|
||||
assert!(store
|
||||
.get_audit_rule_index_entry(AuditRuleKind::Aspa, ¤t.local_outputs[0].rule_hash)
|
||||
.expect("get new audit entry")
|
||||
.is_some());
|
||||
assert!(
|
||||
store
|
||||
.get_audit_rule_index_entry(
|
||||
AuditRuleKind::Roa,
|
||||
&previous.local_outputs[0].rule_hash
|
||||
)
|
||||
.expect("get deleted old audit entry")
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_audit_rule_index_entry(
|
||||
AuditRuleKind::Aspa,
|
||||
¤t.local_outputs[0].rule_hash
|
||||
)
|
||||
.expect("get new audit entry")
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2218,8 +2302,11 @@ mod tests {
|
||||
|
||||
let present_bytes = b"present-object".to_vec();
|
||||
let present_hash = sha256_hex(&present_bytes);
|
||||
let mut present_raw = RawByHashEntry::from_bytes(present_hash.clone(), present_bytes.clone());
|
||||
present_raw.origin_uris.push("rsync://example.test/repo/present.roa".to_string());
|
||||
let mut present_raw =
|
||||
RawByHashEntry::from_bytes(present_hash.clone(), present_bytes.clone());
|
||||
present_raw
|
||||
.origin_uris
|
||||
.push("rsync://example.test/repo/present.roa".to_string());
|
||||
present_raw.object_type = Some("roa".to_string());
|
||||
store
|
||||
.put_raw_by_hash_entry(&present_raw)
|
||||
@@ -2238,7 +2325,9 @@ mod tests {
|
||||
let replaced_hash = sha256_hex(&replaced_bytes);
|
||||
let mut replaced_raw =
|
||||
RawByHashEntry::from_bytes(replaced_hash.clone(), replaced_bytes.clone());
|
||||
replaced_raw.origin_uris.push("rsync://example.test/repo/replaced.cer".to_string());
|
||||
replaced_raw
|
||||
.origin_uris
|
||||
.push("rsync://example.test/repo/replaced.cer".to_string());
|
||||
replaced_raw.object_type = Some("cer".to_string());
|
||||
store
|
||||
.put_raw_by_hash_entry(&replaced_raw)
|
||||
@@ -2309,5 +2398,4 @@ mod tests {
|
||||
.expect_err("missing raw_by_hash should error");
|
||||
assert!(matches!(err, StorageError::InvalidData { .. }));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
370
src/sync/repo.rs
370
src/sync/repo.rs
@@ -10,36 +10,37 @@ use crate::storage::{RawByHashEntry, RocksStore};
|
||||
use crate::sync::rrdp::sync_from_notification_with_timing_and_download_log;
|
||||
use crate::sync::rrdp::{Fetcher as HttpFetcher, RrdpSyncError, load_rrdp_local_state};
|
||||
use crate::sync::store_projection::{
|
||||
build_repository_view_present_entry,
|
||||
build_repository_view_withdrawn_entry,
|
||||
compute_sha256_hex, infer_object_type_from_uri,
|
||||
build_repository_view_present_entry, build_repository_view_withdrawn_entry, compute_sha256_hex,
|
||||
infer_object_type_from_uri,
|
||||
};
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
#[cfg(test)]
|
||||
use crate::storage::RrdpSourceSyncState;
|
||||
#[cfg(test)]
|
||||
use crate::sync::rrdp::persist_rrdp_local_state;
|
||||
|
||||
const RRDP_RETRY_BACKOFFS_PROD: [Duration; 3] = [
|
||||
Duration::from_millis(200),
|
||||
Duration::from_millis(500),
|
||||
Duration::from_secs(1),
|
||||
];
|
||||
const RRDP_RETRY_BACKOFFS_TEST: [Duration; 2] =
|
||||
[Duration::from_millis(0), Duration::from_millis(0)];
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum RepoSyncSource {
|
||||
Rrdp,
|
||||
Rsync,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum RepoSyncPhase {
|
||||
RrdpOk,
|
||||
RrdpFailedRsyncOk,
|
||||
RsyncOnlyOk,
|
||||
ReplayRrdpOk,
|
||||
ReplayRsyncOk,
|
||||
ReplayNoopRrdp,
|
||||
ReplayNoopRsync,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RepoSyncResult {
|
||||
pub source: RepoSyncSource,
|
||||
pub phase: RepoSyncPhase,
|
||||
pub objects_written: usize,
|
||||
pub warnings: Vec<Warning>,
|
||||
}
|
||||
@@ -99,6 +100,7 @@ pub fn sync_publication_point(
|
||||
);
|
||||
Ok(RepoSyncResult {
|
||||
source: RepoSyncSource::Rrdp,
|
||||
phase: RepoSyncPhase::RrdpOk,
|
||||
objects_written: written,
|
||||
warnings: Vec::new(),
|
||||
})
|
||||
@@ -115,6 +117,14 @@ pub fn sync_publication_point(
|
||||
"rrdp_error": err.to_string(),
|
||||
}),
|
||||
);
|
||||
crate::progress_log::emit(
|
||||
"rrdp_failed_fallback_rsync",
|
||||
serde_json::json!({
|
||||
"notify_uri": notification_uri,
|
||||
"rsync_base_uri": rsync_base_uri,
|
||||
"rrdp_error": err.to_string(),
|
||||
}),
|
||||
);
|
||||
let warnings = vec![
|
||||
Warning::new(format!("RRDP failed; falling back to rsync: {err}"))
|
||||
.with_rfc_refs(&[RfcRef("RFC 8182 §3.4.5")])
|
||||
@@ -127,12 +137,16 @@ pub fn sync_publication_point(
|
||||
timing,
|
||||
download_log,
|
||||
)?;
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("repo_sync_rsync_peer_aligned_profile_total", 1);
|
||||
}
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("repo_sync_rsync_fallback_ok_total", 1);
|
||||
t.record_count("repo_sync_rsync_objects_written_total", written as u64);
|
||||
}
|
||||
Ok(RepoSyncResult {
|
||||
source: RepoSyncSource::Rsync,
|
||||
phase: RepoSyncPhase::RrdpFailedRsyncOk,
|
||||
objects_written: written,
|
||||
warnings,
|
||||
})
|
||||
@@ -147,6 +161,9 @@ pub fn sync_publication_point(
|
||||
timing,
|
||||
download_log,
|
||||
)?;
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("repo_sync_rsync_peer_aligned_profile_total", 1);
|
||||
}
|
||||
crate::progress_log::emit(
|
||||
"repo_sync_rsync_direct",
|
||||
serde_json::json!({
|
||||
@@ -160,6 +177,7 @@ pub fn sync_publication_point(
|
||||
}
|
||||
Ok(RepoSyncResult {
|
||||
source: RepoSyncSource::Rsync,
|
||||
phase: RepoSyncPhase::RsyncOnlyOk,
|
||||
objects_written: written,
|
||||
warnings: Vec::new(),
|
||||
})
|
||||
@@ -192,6 +210,7 @@ pub fn sync_publication_point_replay(
|
||||
}
|
||||
Ok(RepoSyncResult {
|
||||
source: RepoSyncSource::Rrdp,
|
||||
phase: RepoSyncPhase::ReplayRrdpOk,
|
||||
objects_written: written,
|
||||
warnings: Vec::new(),
|
||||
})
|
||||
@@ -210,6 +229,7 @@ pub fn sync_publication_point_replay(
|
||||
}
|
||||
Ok(RepoSyncResult {
|
||||
source: RepoSyncSource::Rsync,
|
||||
phase: RepoSyncPhase::ReplayRsyncOk,
|
||||
objects_written: written,
|
||||
warnings: Vec::new(),
|
||||
})
|
||||
@@ -243,6 +263,7 @@ pub fn sync_publication_point_replay_delta(
|
||||
}
|
||||
Ok(RepoSyncResult {
|
||||
source: RepoSyncSource::Rrdp,
|
||||
phase: RepoSyncPhase::ReplayRrdpOk,
|
||||
objects_written: written,
|
||||
warnings: Vec::new(),
|
||||
})
|
||||
@@ -261,12 +282,17 @@ pub fn sync_publication_point_replay_delta(
|
||||
}
|
||||
Ok(RepoSyncResult {
|
||||
source: RepoSyncSource::Rsync,
|
||||
phase: RepoSyncPhase::ReplayRsyncOk,
|
||||
objects_written: written,
|
||||
warnings: Vec::new(),
|
||||
})
|
||||
}
|
||||
ReplayDeltaResolvedTransport::Noop(source) => Ok(RepoSyncResult {
|
||||
source,
|
||||
phase: match source {
|
||||
RepoSyncSource::Rrdp => RepoSyncPhase::ReplayNoopRrdp,
|
||||
RepoSyncSource::Rsync => RepoSyncPhase::ReplayNoopRsync,
|
||||
},
|
||||
objects_written: 0,
|
||||
warnings: Vec::new(),
|
||||
}),
|
||||
@@ -448,14 +474,6 @@ fn is_retryable_http_fetch_error(msg: &str) -> bool {
|
||||
code == 408 || code == 429 || (500..600).contains(&code)
|
||||
}
|
||||
|
||||
fn rrdp_retry_backoffs() -> &'static [Duration] {
|
||||
if cfg!(test) {
|
||||
&RRDP_RETRY_BACKOFFS_TEST
|
||||
} else {
|
||||
&RRDP_RETRY_BACKOFFS_PROD
|
||||
}
|
||||
}
|
||||
|
||||
fn try_rrdp_sync_with_retry(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
@@ -463,89 +481,52 @@ fn try_rrdp_sync_with_retry(
|
||||
timing: Option<&TimingHandle>,
|
||||
download_log: Option<&DownloadLogHandle>,
|
||||
) -> Result<usize, RrdpSyncError> {
|
||||
let backoffs = rrdp_retry_backoffs();
|
||||
let max_attempts = backoffs.len().saturating_add(1).max(1);
|
||||
let mut attempt: usize = 0;
|
||||
let attempt = 1usize;
|
||||
crate::progress_log::emit(
|
||||
"rrdp_sync_attempt",
|
||||
serde_json::json!({
|
||||
"notify_uri": notification_uri,
|
||||
"attempt": attempt,
|
||||
}),
|
||||
);
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_retry_attempt_total", 1);
|
||||
}
|
||||
|
||||
loop {
|
||||
attempt += 1;
|
||||
crate::progress_log::emit(
|
||||
"rrdp_sync_attempt",
|
||||
serde_json::json!({
|
||||
"notify_uri": notification_uri,
|
||||
"attempt": attempt,
|
||||
}),
|
||||
);
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_retry_attempt_total", 1);
|
||||
match try_rrdp_sync(store, notification_uri, http_fetcher, timing, download_log) {
|
||||
Ok(written) => {
|
||||
crate::progress_log::emit(
|
||||
"rrdp_sync_success",
|
||||
serde_json::json!({
|
||||
"notify_uri": notification_uri,
|
||||
"attempt": attempt,
|
||||
"objects_written": written,
|
||||
}),
|
||||
);
|
||||
Ok(written)
|
||||
}
|
||||
|
||||
match try_rrdp_sync(store, notification_uri, http_fetcher, timing, download_log) {
|
||||
Ok(written) => {
|
||||
crate::progress_log::emit(
|
||||
"rrdp_sync_success",
|
||||
serde_json::json!({
|
||||
"notify_uri": notification_uri,
|
||||
"attempt": attempt,
|
||||
"objects_written": written,
|
||||
}),
|
||||
);
|
||||
if attempt > 1 {
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_retry_success_total", 1);
|
||||
}
|
||||
Err(err) => {
|
||||
let retryable = match &err {
|
||||
RrdpSyncError::Fetch(msg) => is_retryable_http_fetch_error(msg),
|
||||
_ => false,
|
||||
};
|
||||
crate::progress_log::emit(
|
||||
"rrdp_sync_failed",
|
||||
serde_json::json!({
|
||||
"notify_uri": notification_uri,
|
||||
"attempt": attempt,
|
||||
"retryable": retryable,
|
||||
"error": err.to_string(),
|
||||
}),
|
||||
);
|
||||
if let Some(t) = timing.as_ref() {
|
||||
match &err {
|
||||
RrdpSyncError::Fetch(_) => t.record_count("rrdp_failed_fetch_total", 1),
|
||||
RrdpSyncError::Rrdp(_) => t.record_count("rrdp_failed_protocol_total", 1),
|
||||
RrdpSyncError::Storage(_) => t.record_count("rrdp_failed_storage_total", 1),
|
||||
}
|
||||
return Ok(written);
|
||||
}
|
||||
Err(err) => {
|
||||
let retryable = match &err {
|
||||
RrdpSyncError::Fetch(msg) => is_retryable_http_fetch_error(msg),
|
||||
_ => false,
|
||||
};
|
||||
|
||||
if retryable && attempt < max_attempts {
|
||||
crate::progress_log::emit(
|
||||
"rrdp_sync_retry",
|
||||
serde_json::json!({
|
||||
"notify_uri": notification_uri,
|
||||
"attempt": attempt,
|
||||
"error": err.to_string(),
|
||||
}),
|
||||
);
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_retry_sleep_total", 1);
|
||||
}
|
||||
let backoff = backoffs
|
||||
.get(attempt.saturating_sub(1))
|
||||
.copied()
|
||||
.unwrap_or_else(|| Duration::from_secs(0));
|
||||
if !backoff.is_zero() {
|
||||
thread::sleep(backoff);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
crate::progress_log::emit(
|
||||
"rrdp_sync_failed",
|
||||
serde_json::json!({
|
||||
"notify_uri": notification_uri,
|
||||
"attempt": attempt,
|
||||
"retryable": retryable,
|
||||
"error": err.to_string(),
|
||||
}),
|
||||
);
|
||||
if let Some(t) = timing.as_ref() {
|
||||
match &err {
|
||||
RrdpSyncError::Fetch(_) => t.record_count("rrdp_failed_fetch_total", 1),
|
||||
RrdpSyncError::Rrdp(_) => t.record_count("rrdp_failed_protocol_total", 1),
|
||||
RrdpSyncError::Storage(_) => t.record_count("rrdp_failed_storage_total", 1),
|
||||
}
|
||||
if retryable && attempt >= max_attempts && attempt > 1 {
|
||||
t.record_count("rrdp_retry_exhausted_total", 1);
|
||||
}
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -575,41 +556,42 @@ fn rsync_sync_into_current_store(
|
||||
let mut new_set: HashSet<String> = HashSet::new();
|
||||
let mut uri_to_hash: BTreeMap<String, String> = BTreeMap::new();
|
||||
let mut pending_raw: BTreeMap<String, RawByHashEntry> = BTreeMap::new();
|
||||
let (object_count, bytes_total) = match rsync_fetcher.visit_objects(rsync_base_uri, &mut |uri, bytes| {
|
||||
let sha256_hex = compute_sha256_hex(&bytes);
|
||||
new_set.insert(uri.clone());
|
||||
uri_to_hash.insert(uri.clone(), sha256_hex.clone());
|
||||
let entry = pending_raw
|
||||
.entry(sha256_hex.clone())
|
||||
.or_insert_with(|| RawByHashEntry::from_bytes(sha256_hex.clone(), bytes.clone()));
|
||||
if entry.bytes != bytes {
|
||||
return Err(format!(
|
||||
"raw_by_hash collision for {uri}: same sha256 maps to different bytes"
|
||||
));
|
||||
}
|
||||
if !entry.origin_uris.iter().any(|existing| existing == &uri) {
|
||||
entry.origin_uris.push(uri.clone());
|
||||
}
|
||||
if entry.object_type.is_none() {
|
||||
entry.object_type = infer_object_type_from_uri(&uri);
|
||||
}
|
||||
Ok(())
|
||||
}) {
|
||||
Ok(v) => {
|
||||
if let Some(s) = dl_span.as_mut() {
|
||||
s.set_objects(v.0 as u64, v.1);
|
||||
s.set_bytes(v.1);
|
||||
s.set_ok();
|
||||
let (object_count, bytes_total) =
|
||||
match rsync_fetcher.visit_objects(rsync_base_uri, &mut |uri, bytes| {
|
||||
let sha256_hex = compute_sha256_hex(&bytes);
|
||||
new_set.insert(uri.clone());
|
||||
uri_to_hash.insert(uri.clone(), sha256_hex.clone());
|
||||
let entry = pending_raw
|
||||
.entry(sha256_hex.clone())
|
||||
.or_insert_with(|| RawByHashEntry::from_bytes(sha256_hex.clone(), bytes.clone()));
|
||||
if entry.bytes != bytes {
|
||||
return Err(format!(
|
||||
"raw_by_hash collision for {uri}: same sha256 maps to different bytes"
|
||||
));
|
||||
}
|
||||
v
|
||||
}
|
||||
Err(e) => {
|
||||
if let Some(s) = dl_span.as_mut() {
|
||||
s.set_err(e.to_string());
|
||||
if !entry.origin_uris.iter().any(|existing| existing == &uri) {
|
||||
entry.origin_uris.push(uri.clone());
|
||||
}
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
if entry.object_type.is_none() {
|
||||
entry.object_type = infer_object_type_from_uri(&uri);
|
||||
}
|
||||
Ok(())
|
||||
}) {
|
||||
Ok(v) => {
|
||||
if let Some(s) = dl_span.as_mut() {
|
||||
s.set_objects(v.0 as u64, v.1);
|
||||
s.set_bytes(v.1);
|
||||
s.set_ok();
|
||||
}
|
||||
v
|
||||
}
|
||||
Err(e) => {
|
||||
if let Some(s) = dl_span.as_mut() {
|
||||
s.set_err(e.to_string());
|
||||
}
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
crate::progress_log::emit(
|
||||
"rsync_sync_fetch_done",
|
||||
serde_json::json!({
|
||||
@@ -639,9 +621,9 @@ fn rsync_sync_into_current_store(
|
||||
.map_err(|e| RepoSyncError::Storage(e.to_string()))?;
|
||||
let mut entries_to_write = Vec::new();
|
||||
for (hash, existing_opt) in hashes.into_iter().zip(existing_entries.into_iter()) {
|
||||
let mut pending_entry = pending_raw
|
||||
.remove(&hash)
|
||||
.ok_or_else(|| RepoSyncError::Storage(format!("missing pending raw entry for {hash}")))?;
|
||||
let mut pending_entry = pending_raw.remove(&hash).ok_or_else(|| {
|
||||
RepoSyncError::Storage(format!("missing pending raw entry for {hash}"))
|
||||
})?;
|
||||
match existing_opt {
|
||||
Some(mut existing) => {
|
||||
if existing.bytes != pending_entry.bytes {
|
||||
@@ -651,7 +633,11 @@ fn rsync_sync_into_current_store(
|
||||
}
|
||||
let mut changed = false;
|
||||
for uri in pending_entry.origin_uris.drain(..) {
|
||||
if !existing.origin_uris.iter().any(|existing_uri| existing_uri == &uri) {
|
||||
if !existing
|
||||
.origin_uris
|
||||
.iter()
|
||||
.any(|existing_uri| existing_uri == &uri)
|
||||
{
|
||||
existing.origin_uris.push(uri);
|
||||
changed = true;
|
||||
}
|
||||
@@ -679,10 +665,9 @@ fn rsync_sync_into_current_store(
|
||||
}
|
||||
|
||||
for uri in &new_set {
|
||||
let current_hash = uri_to_hash
|
||||
.get(uri)
|
||||
.cloned()
|
||||
.ok_or_else(|| RepoSyncError::Storage(format!("missing raw_by_hash mapping for {uri}")))?;
|
||||
let current_hash = uri_to_hash.get(uri).cloned().ok_or_else(|| {
|
||||
RepoSyncError::Storage(format!("missing raw_by_hash mapping for {uri}"))
|
||||
})?;
|
||||
repository_view_entries.push(build_repository_view_present_entry(
|
||||
&sync_scope_uri,
|
||||
uri,
|
||||
@@ -736,9 +721,9 @@ mod tests {
|
||||
use crate::replay::fetch_http::PayloadReplayHttpFetcher;
|
||||
use crate::replay::fetch_rsync::PayloadReplayRsyncFetcher;
|
||||
use crate::storage::RepositoryViewState;
|
||||
use crate::sync::store_projection::build_repository_view_present_entry;
|
||||
use crate::sync::rrdp::Fetcher as HttpFetcher;
|
||||
use crate::sync::rrdp::RrdpState;
|
||||
use crate::sync::store_projection::build_repository_view_present_entry;
|
||||
use base64::Engine;
|
||||
use sha2::Digest;
|
||||
use std::collections::HashMap;
|
||||
@@ -1322,7 +1307,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rrdp_retry_succeeds_without_rsync_when_notification_fetch_is_transient() {
|
||||
fn rrdp_fetch_error_falls_back_to_rsync_without_retry() {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let store_dir = temp.path().join("db");
|
||||
let store = RocksStore::open(&store_dir).expect("open rocksdb");
|
||||
@@ -1335,50 +1320,33 @@ mod tests {
|
||||
});
|
||||
|
||||
let notification_uri = "https://example.test/notification.xml";
|
||||
let snapshot_uri = "https://example.test/snapshot.xml";
|
||||
let published_uri = "rsync://example.test/repo/a.mft";
|
||||
let published_bytes = b"x";
|
||||
|
||||
let snapshot = snapshot_xml(
|
||||
"9df4b597-af9e-4dca-bdda-719cce2c4e28",
|
||||
1,
|
||||
&[(published_uri, published_bytes)],
|
||||
);
|
||||
let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot));
|
||||
let notif = notification_xml(
|
||||
"9df4b597-af9e-4dca-bdda-719cce2c4e28",
|
||||
1,
|
||||
snapshot_uri,
|
||||
&snapshot_hash,
|
||||
);
|
||||
|
||||
let mut map = HashMap::new();
|
||||
map.insert(notification_uri.to_string(), notif);
|
||||
map.insert(snapshot_uri.to_string(), snapshot);
|
||||
|
||||
struct RetryThenMap {
|
||||
inner: MapFetcher,
|
||||
notification_uri: String,
|
||||
fail_times: usize,
|
||||
struct AlwaysFailHttp {
|
||||
notification_calls: AtomicUsize,
|
||||
}
|
||||
|
||||
impl HttpFetcher for RetryThenMap {
|
||||
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
|
||||
if uri == self.notification_uri {
|
||||
let n = self.notification_calls.fetch_add(1, Ordering::SeqCst);
|
||||
if n < self.fail_times {
|
||||
return Err("http request failed: simulated transient".to_string());
|
||||
}
|
||||
}
|
||||
self.inner.fetch(uri)
|
||||
impl HttpFetcher for AlwaysFailHttp {
|
||||
fn fetch(&self, _uri: &str) -> Result<Vec<u8>, String> {
|
||||
self.notification_calls.fetch_add(1, Ordering::SeqCst);
|
||||
Err("http request failed: simulated transient".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
let http = RetryThenMap {
|
||||
inner: MapFetcher { map },
|
||||
notification_uri: notification_uri.to_string(),
|
||||
fail_times: 2,
|
||||
struct SingleObjectRsync {
|
||||
uri: String,
|
||||
bytes: Vec<u8>,
|
||||
}
|
||||
impl RsyncFetcher for SingleObjectRsync {
|
||||
fn fetch_objects(
|
||||
&self,
|
||||
_rsync_base_uri: &str,
|
||||
) -> Result<Vec<(String, Vec<u8>)>, RsyncFetchError> {
|
||||
Ok(vec![(self.uri.clone(), self.bytes.clone())])
|
||||
}
|
||||
}
|
||||
|
||||
let http = AlwaysFailHttp {
|
||||
notification_calls: AtomicUsize::new(0),
|
||||
};
|
||||
|
||||
@@ -1394,29 +1362,26 @@ mod tests {
|
||||
Some(notification_uri),
|
||||
"rsync://example.test/repo/",
|
||||
&http,
|
||||
&PanicRsyncFetcher,
|
||||
&SingleObjectRsync {
|
||||
uri: published_uri.to_string(),
|
||||
bytes: published_bytes.to_vec(),
|
||||
},
|
||||
Some(&timing),
|
||||
Some(&download_log),
|
||||
)
|
||||
.expect("sync ok");
|
||||
|
||||
assert_eq!(out.source, RepoSyncSource::Rrdp);
|
||||
assert_eq!(out.source, RepoSyncSource::Rsync);
|
||||
assert_current_object(&store, published_uri, published_bytes);
|
||||
assert_eq!(http.notification_calls.load(Ordering::SeqCst), 1);
|
||||
|
||||
let events = download_log.snapshot_events();
|
||||
assert_eq!(events.len(), 4, "expected 3x notification + 1x snapshot");
|
||||
assert_eq!(events.len(), 2, "expected 1x notification + 1x rsync");
|
||||
assert_eq!(
|
||||
events
|
||||
.iter()
|
||||
.filter(|e| e.kind == AuditDownloadKind::RrdpNotification)
|
||||
.count(),
|
||||
3
|
||||
);
|
||||
assert_eq!(
|
||||
events
|
||||
.iter()
|
||||
.filter(|e| e.kind == AuditDownloadKind::RrdpSnapshot)
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -1424,7 +1389,14 @@ mod tests {
|
||||
.iter()
|
||||
.filter(|e| e.kind == AuditDownloadKind::RrdpNotification && !e.success)
|
||||
.count(),
|
||||
2
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
events
|
||||
.iter()
|
||||
.filter(|e| e.kind == AuditDownloadKind::Rsync)
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
|
||||
let v = timing_to_json(temp.path(), &timing);
|
||||
@@ -1433,17 +1405,17 @@ mod tests {
|
||||
counts
|
||||
.get("rrdp_retry_attempt_total")
|
||||
.and_then(|v| v.as_u64()),
|
||||
Some(3)
|
||||
Some(1)
|
||||
);
|
||||
assert_eq!(
|
||||
counts
|
||||
.get("rrdp_retry_success_total")
|
||||
.get("repo_sync_rrdp_failed_total")
|
||||
.and_then(|v| v.as_u64()),
|
||||
Some(1)
|
||||
);
|
||||
assert_eq!(
|
||||
counts
|
||||
.get("repo_sync_rrdp_ok_total")
|
||||
.get("repo_sync_rsync_fallback_ok_total")
|
||||
.and_then(|v| v.as_u64()),
|
||||
Some(1)
|
||||
);
|
||||
@@ -1749,7 +1721,11 @@ mod tests {
|
||||
assert_eq!(out.source, RepoSyncSource::Rsync);
|
||||
assert_eq!(out.objects_written, 1);
|
||||
assert_eq!(out.warnings.len(), 0);
|
||||
assert_current_object(&store, "rsync://rsync.example.test/repo/sub/fallback.cer", b"cer");
|
||||
assert_current_object(
|
||||
&store,
|
||||
"rsync://rsync.example.test/repo/sub/fallback.cer",
|
||||
b"cer",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1995,7 +1971,11 @@ mod tests {
|
||||
assert_eq!(out.source, RepoSyncSource::Rsync);
|
||||
assert_eq!(out.objects_written, 2);
|
||||
assert_current_object(&store, "rsync://rsync.example.test/repo/a.mft", b"base");
|
||||
assert_current_object(&store, "rsync://rsync.example.test/repo/sub/x.cer", b"overlay-cer");
|
||||
assert_current_object(
|
||||
&store,
|
||||
"rsync://rsync.example.test/repo/sub/x.cer",
|
||||
b"overlay-cer",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -5,12 +5,10 @@ use crate::storage::{RocksStore, RrdpDeltaOp, RrdpSourceSyncState};
|
||||
use crate::sync::store_projection::{
|
||||
build_repository_view_present_entry, build_repository_view_withdrawn_entry,
|
||||
build_rrdp_source_member_present_record, build_rrdp_source_member_withdrawn_record,
|
||||
build_rrdp_uri_owner_active_record, build_rrdp_uri_owner_withdrawn_record,
|
||||
compute_sha256_hex, current_rrdp_owner_is, ensure_rrdp_uri_can_be_owned_by,
|
||||
prepare_raw_by_hash_evidence_batch,
|
||||
put_repository_view_present, put_repository_view_withdrawn,
|
||||
put_rrdp_source_member_present, put_rrdp_source_member_withdrawn,
|
||||
put_rrdp_uri_owner_active, put_rrdp_uri_owner_withdrawn,
|
||||
build_rrdp_uri_owner_active_record, build_rrdp_uri_owner_withdrawn_record, compute_sha256_hex,
|
||||
current_rrdp_owner_is, ensure_rrdp_uri_can_be_owned_by, prepare_raw_by_hash_evidence_batch,
|
||||
put_repository_view_present, put_repository_view_withdrawn, put_rrdp_source_member_present,
|
||||
put_rrdp_source_member_withdrawn, put_rrdp_uri_owner_active, put_rrdp_uri_owner_withdrawn,
|
||||
update_rrdp_source_record_on_success, upsert_raw_by_hash_evidence,
|
||||
};
|
||||
use base64::Engine;
|
||||
@@ -1228,20 +1226,16 @@ fn apply_snapshot(
|
||||
}
|
||||
|
||||
let session_id = expected_session_id.to_string();
|
||||
let prepared_raw = prepare_raw_by_hash_evidence_batch(store, &published)
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
let prepared_raw =
|
||||
prepare_raw_by_hash_evidence_batch(store, &published).map_err(RrdpSyncError::Storage)?;
|
||||
let mut repository_view_entries = Vec::with_capacity(published.len() + withdrawn.len());
|
||||
let mut member_records = Vec::with_capacity(published.len() + withdrawn.len());
|
||||
let mut owner_records = Vec::with_capacity(published.len() + withdrawn.len());
|
||||
|
||||
for (uri, _bytes) in &published {
|
||||
let current_hash = prepared_raw
|
||||
.uri_to_hash
|
||||
.get(uri)
|
||||
.cloned()
|
||||
.ok_or_else(|| {
|
||||
RrdpSyncError::Storage(format!("missing raw_by_hash mapping for {uri}"))
|
||||
})?;
|
||||
let current_hash = prepared_raw.uri_to_hash.get(uri).cloned().ok_or_else(|| {
|
||||
RrdpSyncError::Storage(format!("missing raw_by_hash mapping for {uri}"))
|
||||
})?;
|
||||
repository_view_entries.push(build_repository_view_present_entry(
|
||||
notification_uri,
|
||||
uri,
|
||||
|
||||
@@ -56,7 +56,11 @@ pub fn prepare_raw_by_hash_evidence_batch(
|
||||
}
|
||||
let mut changed = false;
|
||||
for uri in pending_entry.origin_uris.drain(..) {
|
||||
if !existing.origin_uris.iter().any(|existing_uri| existing_uri == &uri) {
|
||||
if !existing
|
||||
.origin_uris
|
||||
.iter()
|
||||
.any(|existing_uri| existing_uri == &uri)
|
||||
{
|
||||
existing.origin_uris.push(uri);
|
||||
changed = true;
|
||||
}
|
||||
@@ -195,10 +199,8 @@ pub fn upsert_raw_by_hash_evidence(
|
||||
rsync_uri: &str,
|
||||
bytes: &[u8],
|
||||
) -> Result<String, String> {
|
||||
let prepared = prepare_raw_by_hash_evidence_batch(
|
||||
store,
|
||||
&[(rsync_uri.to_string(), bytes.to_vec())],
|
||||
)?;
|
||||
let prepared =
|
||||
prepare_raw_by_hash_evidence_batch(store, &[(rsync_uri.to_string(), bytes.to_vec())])?;
|
||||
store
|
||||
.put_raw_by_hash_entries_batch_unchecked(&prepared.entries_to_write)
|
||||
.map_err(|e| e.to_string())?;
|
||||
@@ -277,7 +279,13 @@ pub fn put_rrdp_source_member_present(
|
||||
rsync_uri: &str,
|
||||
current_hash: &str,
|
||||
) -> Result<(), String> {
|
||||
let record = build_rrdp_source_member_present_record(notification_uri, session_id, serial, rsync_uri, current_hash);
|
||||
let record = build_rrdp_source_member_present_record(
|
||||
notification_uri,
|
||||
session_id,
|
||||
serial,
|
||||
rsync_uri,
|
||||
current_hash,
|
||||
);
|
||||
store
|
||||
.put_rrdp_source_member_record(&record)
|
||||
.map_err(|e| e.to_string())
|
||||
@@ -291,7 +299,13 @@ pub fn put_rrdp_source_member_withdrawn(
|
||||
rsync_uri: &str,
|
||||
current_hash: Option<String>,
|
||||
) -> Result<(), String> {
|
||||
let record = build_rrdp_source_member_withdrawn_record(notification_uri, session_id, serial, rsync_uri, current_hash);
|
||||
let record = build_rrdp_source_member_withdrawn_record(
|
||||
notification_uri,
|
||||
session_id,
|
||||
serial,
|
||||
rsync_uri,
|
||||
current_hash,
|
||||
);
|
||||
store
|
||||
.put_rrdp_source_member_record(&record)
|
||||
.map_err(|e| e.to_string())
|
||||
@@ -305,7 +319,13 @@ pub fn put_rrdp_uri_owner_active(
|
||||
rsync_uri: &str,
|
||||
current_hash: &str,
|
||||
) -> Result<(), String> {
|
||||
let record = build_rrdp_uri_owner_active_record(notification_uri, session_id, serial, rsync_uri, current_hash);
|
||||
let record = build_rrdp_uri_owner_active_record(
|
||||
notification_uri,
|
||||
session_id,
|
||||
serial,
|
||||
rsync_uri,
|
||||
current_hash,
|
||||
);
|
||||
store
|
||||
.put_rrdp_uri_owner_record(&record)
|
||||
.map_err(|e| e.to_string())
|
||||
@@ -319,7 +339,13 @@ pub fn put_rrdp_uri_owner_withdrawn(
|
||||
rsync_uri: &str,
|
||||
current_hash: Option<String>,
|
||||
) -> Result<(), String> {
|
||||
let record = build_rrdp_uri_owner_withdrawn_record(notification_uri, session_id, serial, rsync_uri, current_hash);
|
||||
let record = build_rrdp_uri_owner_withdrawn_record(
|
||||
notification_uri,
|
||||
session_id,
|
||||
serial,
|
||||
rsync_uri,
|
||||
current_hash,
|
||||
);
|
||||
store
|
||||
.put_rrdp_uri_owner_record(&record)
|
||||
.map_err(|e| e.to_string())
|
||||
|
||||
@@ -29,7 +29,8 @@ pub struct ValidatedSubordinateCaLite {
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct IssuerEffectiveResourcesIndex {
|
||||
parent_ip_by_afi_items: Option<BTreeMap<crate::data_model::rc::Afi, Vec<crate::data_model::rc::IpAddressOrRange>>>,
|
||||
parent_ip_by_afi_items:
|
||||
Option<BTreeMap<crate::data_model::rc::Afi, Vec<crate::data_model::rc::IpAddressOrRange>>>,
|
||||
parent_ip_merged_intervals: HashMap<crate::data_model::rc::Afi, Vec<(Vec<u8>, Vec<u8>)>>,
|
||||
parent_asnum_intervals: Option<Vec<(u32, u32)>>,
|
||||
parent_rdi_intervals: Option<Vec<(u32, u32)>>,
|
||||
@@ -40,21 +41,18 @@ impl IssuerEffectiveResourcesIndex {
|
||||
issuer_effective_ip: Option<&IpResourceSet>,
|
||||
issuer_effective_as: Option<&AsResourceSet>,
|
||||
) -> Result<Self, CaPathError> {
|
||||
let parent_ip_by_afi_items = issuer_effective_ip.map(ip_resources_by_afi_items).transpose()?;
|
||||
let parent_ip_by_afi_items = issuer_effective_ip
|
||||
.map(ip_resources_by_afi_items)
|
||||
.transpose()?;
|
||||
|
||||
let parent_ip_merged_intervals = issuer_effective_ip
|
||||
.map(ip_resources_to_merged_intervals_by_afi)
|
||||
.unwrap_or_default();
|
||||
|
||||
let parent_asnum_intervals = issuer_effective_as.and_then(|resources| {
|
||||
resources
|
||||
.asnum
|
||||
.as_ref()
|
||||
.map(as_choice_to_merged_intervals)
|
||||
});
|
||||
let parent_rdi_intervals = issuer_effective_as.and_then(|resources| {
|
||||
resources.rdi.as_ref().map(as_choice_to_merged_intervals)
|
||||
});
|
||||
let parent_asnum_intervals = issuer_effective_as
|
||||
.and_then(|resources| resources.asnum.as_ref().map(as_choice_to_merged_intervals));
|
||||
let parent_rdi_intervals = issuer_effective_as
|
||||
.and_then(|resources| resources.rdi.as_ref().map(as_choice_to_merged_intervals));
|
||||
|
||||
Ok(Self {
|
||||
parent_ip_by_afi_items,
|
||||
@@ -512,7 +510,9 @@ fn resolve_child_ip_resources(
|
||||
child_ip: Option<&IpResourceSet>,
|
||||
issuer_effective: Option<&IpResourceSet>,
|
||||
) -> Result<Option<IpResourceSet>, CaPathError> {
|
||||
let precomputed_parent_by_afi = issuer_effective.map(ip_resources_by_afi_items).transpose()?;
|
||||
let precomputed_parent_by_afi = issuer_effective
|
||||
.map(ip_resources_by_afi_items)
|
||||
.transpose()?;
|
||||
let precomputed_parent_intervals = issuer_effective
|
||||
.map(ip_resources_to_merged_intervals_by_afi)
|
||||
.unwrap_or_default();
|
||||
@@ -527,7 +527,9 @@ fn resolve_child_ip_resources(
|
||||
fn resolve_child_ip_resources_indexed(
|
||||
child_ip: Option<&IpResourceSet>,
|
||||
issuer_effective: Option<&IpResourceSet>,
|
||||
parent_by_afi: Option<&BTreeMap<crate::data_model::rc::Afi, Vec<crate::data_model::rc::IpAddressOrRange>>>,
|
||||
parent_by_afi: Option<
|
||||
&BTreeMap<crate::data_model::rc::Afi, Vec<crate::data_model::rc::IpAddressOrRange>>,
|
||||
>,
|
||||
parent_intervals_by_afi: &HashMap<crate::data_model::rc::Afi, Vec<(Vec<u8>, Vec<u8>)>>,
|
||||
) -> Result<Option<IpResourceSet>, CaPathError> {
|
||||
let Some(child_ip) = child_ip else {
|
||||
@@ -583,15 +585,10 @@ fn resolve_child_as_resources(
|
||||
child_as: Option<&AsResourceSet>,
|
||||
issuer_effective: Option<&AsResourceSet>,
|
||||
) -> Result<Option<AsResourceSet>, CaPathError> {
|
||||
let precomputed_asnum = issuer_effective.and_then(|resources| {
|
||||
resources
|
||||
.asnum
|
||||
.as_ref()
|
||||
.map(as_choice_to_merged_intervals)
|
||||
});
|
||||
let precomputed_rdi = issuer_effective.and_then(|resources| {
|
||||
resources.rdi.as_ref().map(as_choice_to_merged_intervals)
|
||||
});
|
||||
let precomputed_asnum = issuer_effective
|
||||
.and_then(|resources| resources.asnum.as_ref().map(as_choice_to_merged_intervals));
|
||||
let precomputed_rdi = issuer_effective
|
||||
.and_then(|resources| resources.rdi.as_ref().map(as_choice_to_merged_intervals));
|
||||
resolve_child_as_resources_indexed(
|
||||
child_as,
|
||||
issuer_effective,
|
||||
@@ -974,8 +971,8 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::data_model::common::X509NameDer;
|
||||
use crate::data_model::rc::{
|
||||
Afi, AsIdentifierChoice, AsResourceSet, IpAddressChoice, IpAddressFamily,
|
||||
IpAddressOrRange, IpResourceSet,
|
||||
Afi, AsIdentifierChoice, AsResourceSet, IpAddressChoice, IpAddressFamily, IpAddressOrRange,
|
||||
IpResourceSet,
|
||||
};
|
||||
use crate::data_model::rc::{
|
||||
RcExtensions, ResourceCertKind, ResourceCertificate, RpkixTbsCertificate,
|
||||
@@ -1470,17 +1467,30 @@ mod tests {
|
||||
}],
|
||||
};
|
||||
let parent_as = AsResourceSet {
|
||||
asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Range {
|
||||
min: 64500,
|
||||
max: 64599,
|
||||
}])),
|
||||
rdi: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id(65000)])),
|
||||
asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![
|
||||
AsIdOrRange::Range {
|
||||
min: 64500,
|
||||
max: 64599,
|
||||
},
|
||||
])),
|
||||
rdi: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id(
|
||||
65000,
|
||||
)])),
|
||||
};
|
||||
let idx = IssuerEffectiveResourcesIndex::from_effective_resources(Some(&parent_ip), Some(&parent_as))
|
||||
.expect("index builds");
|
||||
assert_eq!(idx.parent_ip_by_afi_items.as_ref().map(|v| v.len()), Some(1));
|
||||
let idx = IssuerEffectiveResourcesIndex::from_effective_resources(
|
||||
Some(&parent_ip),
|
||||
Some(&parent_as),
|
||||
)
|
||||
.expect("index builds");
|
||||
assert_eq!(
|
||||
idx.parent_ip_by_afi_items.as_ref().map(|v| v.len()),
|
||||
Some(1)
|
||||
);
|
||||
assert_eq!(idx.parent_ip_merged_intervals.len(), 1);
|
||||
assert_eq!(idx.parent_asnum_intervals.as_ref().map(|v| v.len()), Some(1));
|
||||
assert_eq!(
|
||||
idx.parent_asnum_intervals.as_ref().map(|v| v.len()),
|
||||
Some(1)
|
||||
);
|
||||
assert_eq!(idx.parent_rdi_intervals.as_ref().map(|v| v.len()), Some(1));
|
||||
|
||||
let child_ip_subset = IpResourceSet {
|
||||
@@ -1495,14 +1505,16 @@ mod tests {
|
||||
)]),
|
||||
}],
|
||||
};
|
||||
assert!(resolve_child_ip_resources_indexed(
|
||||
Some(&child_ip_subset),
|
||||
Some(&parent_ip),
|
||||
idx.parent_ip_by_afi_items.as_ref(),
|
||||
&idx.parent_ip_merged_intervals,
|
||||
)
|
||||
assert!(
|
||||
resolve_child_ip_resources_indexed(
|
||||
Some(&child_ip_subset),
|
||||
Some(&parent_ip),
|
||||
idx.parent_ip_by_afi_items.as_ref(),
|
||||
&idx.parent_ip_merged_intervals,
|
||||
)
|
||||
.expect("subset should resolve")
|
||||
.is_some());
|
||||
.is_some()
|
||||
);
|
||||
|
||||
let child_ip_bad = IpResourceSet {
|
||||
families: vec![IpAddressFamily {
|
||||
@@ -1522,24 +1534,32 @@ mod tests {
|
||||
idx.parent_ip_by_afi_items.as_ref(),
|
||||
&idx.parent_ip_merged_intervals,
|
||||
)
|
||||
.unwrap_err();
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, CaPathError::ResourcesNotSubset));
|
||||
|
||||
let child_as_subset = AsResourceSet {
|
||||
asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id(64542)])),
|
||||
rdi: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id(65000)])),
|
||||
asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id(
|
||||
64542,
|
||||
)])),
|
||||
rdi: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id(
|
||||
65000,
|
||||
)])),
|
||||
};
|
||||
assert!(resolve_child_as_resources_indexed(
|
||||
Some(&child_as_subset),
|
||||
Some(&parent_as),
|
||||
idx.parent_asnum_intervals.as_deref(),
|
||||
idx.parent_rdi_intervals.as_deref(),
|
||||
)
|
||||
assert!(
|
||||
resolve_child_as_resources_indexed(
|
||||
Some(&child_as_subset),
|
||||
Some(&parent_as),
|
||||
idx.parent_asnum_intervals.as_deref(),
|
||||
idx.parent_rdi_intervals.as_deref(),
|
||||
)
|
||||
.expect("subset as resolves")
|
||||
.is_some());
|
||||
.is_some()
|
||||
);
|
||||
|
||||
let child_as_bad = AsResourceSet {
|
||||
asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id(65123)])),
|
||||
asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id(
|
||||
65123,
|
||||
)])),
|
||||
rdi: None,
|
||||
};
|
||||
let err = resolve_child_as_resources_indexed(
|
||||
@@ -1548,7 +1568,7 @@ mod tests {
|
||||
idx.parent_asnum_intervals.as_deref(),
|
||||
idx.parent_rdi_intervals.as_deref(),
|
||||
)
|
||||
.unwrap_err();
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, CaPathError::ResourcesNotSubset));
|
||||
}
|
||||
|
||||
|
||||
@@ -196,7 +196,9 @@ fn rsync_parent_uri(ta_rsync_uri: &str) -> Result<String, String> {
|
||||
.ok_or_else(|| format!("missing path in rsync URI: {ta_rsync_uri}"))?
|
||||
.collect::<Vec<_>>();
|
||||
if segments.is_empty() || segments.last().copied().unwrap_or_default().is_empty() {
|
||||
return Err(format!("rsync URI must reference a file object: {ta_rsync_uri}"));
|
||||
return Err(format!(
|
||||
"rsync URI must reference a file object: {ta_rsync_uri}"
|
||||
));
|
||||
}
|
||||
let parent_segments = &segments[..segments.len() - 1];
|
||||
let mut parent = format!("rsync://{host}/");
|
||||
@@ -248,29 +250,32 @@ mod tests {
|
||||
.clone();
|
||||
|
||||
let td = tempfile::tempdir().unwrap();
|
||||
let mirror_root = td.path().join(rsync_uri.host_str().unwrap()).join("repository");
|
||||
let mirror_root = td
|
||||
.path()
|
||||
.join(rsync_uri.host_str().unwrap())
|
||||
.join("repository");
|
||||
std::fs::create_dir_all(&mirror_root).unwrap();
|
||||
std::fs::write(
|
||||
mirror_root.join("apnic-rpki-root-iana-origin.cer"),
|
||||
ta_der,
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(mirror_root.join("apnic-rpki-root-iana-origin.cer"), ta_der).unwrap();
|
||||
|
||||
let http = crate::fetch::http::BlockingHttpFetcher::new(
|
||||
crate::fetch::http::HttpFetcherConfig::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let rsync = LocalDirRsyncFetcher::new(
|
||||
td.path().join(rsync_uri.host_str().unwrap()).join("repository"),
|
||||
td.path()
|
||||
.join(rsync_uri.host_str().unwrap())
|
||||
.join("repository"),
|
||||
);
|
||||
let discovery = discover_root_ca_instance_from_tal_with_fetchers(&http, &rsync, tal, None)
|
||||
.expect("discover via rsync TA");
|
||||
assert!(discovery
|
||||
.trust_anchor
|
||||
.resolved_ta_uri
|
||||
.unwrap()
|
||||
.as_str()
|
||||
.starts_with("rsync://"));
|
||||
assert!(
|
||||
discovery
|
||||
.trust_anchor
|
||||
.resolved_ta_uri
|
||||
.unwrap()
|
||||
.as_str()
|
||||
.starts_with("rsync://")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use crate::blob_store::RawObjectStore;
|
||||
use crate::data_model::manifest::{ManifestDecodeError, ManifestObject, ManifestValidateError};
|
||||
use crate::data_model::signed_object::SignedObjectVerifyError;
|
||||
use crate::policy::{CaFailedFetchPolicy, Policy};
|
||||
@@ -448,8 +449,8 @@ pub fn load_current_instance_vcir_publication_point(
|
||||
)
|
||||
})?;
|
||||
|
||||
let manifest_entry = store
|
||||
.get_raw_by_hash_entry(&manifest_artifact.sha256)?
|
||||
let manifest_bytes = store
|
||||
.get_blob_bytes(&manifest_artifact.sha256)?
|
||||
.ok_or_else(|| ManifestReuseError::MissingManifestRaw(manifest_artifact.sha256.clone()))?;
|
||||
|
||||
let mut seen = HashSet::new();
|
||||
@@ -468,12 +469,12 @@ pub fn load_current_instance_vcir_publication_point(
|
||||
if !seen.insert(uri.clone()) {
|
||||
continue;
|
||||
}
|
||||
let entry = store
|
||||
.get_raw_by_hash_entry(&artifact.sha256)?
|
||||
let entry_bytes = store
|
||||
.get_blob_bytes(&artifact.sha256)?
|
||||
.ok_or_else(|| ManifestReuseError::MissingArtifactRaw {
|
||||
rsync_uri: uri.clone(),
|
||||
})?;
|
||||
files.push(PackFile::from_bytes_compute_sha256(uri, entry.bytes));
|
||||
files.push(PackFile::from_bytes_compute_sha256(uri, entry_bytes));
|
||||
}
|
||||
|
||||
Ok(PublicationPointSnapshot {
|
||||
@@ -493,7 +494,7 @@ pub fn load_current_instance_vcir_publication_point(
|
||||
.validated_manifest_next_update
|
||||
.clone(),
|
||||
verified_at: vcir.last_successful_validation_time.clone(),
|
||||
manifest_bytes: manifest_entry.bytes,
|
||||
manifest_bytes,
|
||||
files,
|
||||
})
|
||||
}
|
||||
@@ -836,12 +837,7 @@ mod tests {
|
||||
entry
|
||||
}
|
||||
|
||||
fn put_current_object(
|
||||
store: &RocksStore,
|
||||
rsync_uri: &str,
|
||||
bytes: Vec<u8>,
|
||||
object_type: &str,
|
||||
) {
|
||||
fn put_current_object(store: &RocksStore, rsync_uri: &str, bytes: Vec<u8>, object_type: &str) {
|
||||
let hash = hex::encode(sha2::Sha256::digest(&bytes));
|
||||
store
|
||||
.put_raw_by_hash_entry(&raw_by_hash_entry(rsync_uri, bytes, object_type))
|
||||
|
||||
@@ -11,9 +11,7 @@ use crate::data_model::signed_object::SignedObjectVerifyError;
|
||||
use crate::policy::{Policy, SignedObjectFailurePolicy};
|
||||
use crate::report::{RfcRef, Warning};
|
||||
use crate::storage::{PackFile, PackTime, VcirLocalOutput, VcirOutputType};
|
||||
use crate::validation::cert_path::{
|
||||
CertPathError, validate_ee_cert_path_with_predecoded_ee,
|
||||
};
|
||||
use crate::validation::cert_path::{CertPathError, validate_ee_cert_path_with_predecoded_ee};
|
||||
use crate::validation::manifest::PublicationPointData;
|
||||
use crate::validation::publication_point::PublicationPointSnapshot;
|
||||
use x509_parser::prelude::FromDer;
|
||||
@@ -603,11 +601,7 @@ fn process_roa_with_issuer(
|
||||
|
||||
let ee = &roa.signed_object.signed_data.certificates[0].resource_cert;
|
||||
let ee_der = &roa.signed_object.signed_data.certificates[0].raw_der;
|
||||
let ee_crldp_uris = ee
|
||||
.tbs
|
||||
.extensions
|
||||
.crl_distribution_points_uris
|
||||
.as_ref();
|
||||
let ee_crldp_uris = ee.tbs.extensions.crl_distribution_points_uris.as_ref();
|
||||
let issuer_crl_rsync_uri = choose_crl_uri_for_certificate(ee_crldp_uris, crl_cache)?;
|
||||
let verified_crl = ensure_issuer_crl_verified(issuer_crl_rsync_uri, crl_cache, issuer_ca_der)?;
|
||||
|
||||
@@ -715,11 +709,7 @@ fn process_aspa_with_issuer(
|
||||
|
||||
let ee = &aspa.signed_object.signed_data.certificates[0].resource_cert;
|
||||
let ee_der = &aspa.signed_object.signed_data.certificates[0].raw_der;
|
||||
let ee_crldp_uris = ee
|
||||
.tbs
|
||||
.extensions
|
||||
.crl_distribution_points_uris
|
||||
.as_ref();
|
||||
let ee_crldp_uris = ee.tbs.extensions.crl_distribution_points_uris.as_ref();
|
||||
let issuer_crl_rsync_uri = choose_crl_uri_for_certificate(ee_crldp_uris, crl_cache)?;
|
||||
let verified_crl = ensure_issuer_crl_verified(issuer_crl_rsync_uri, crl_cache, issuer_ca_der)?;
|
||||
|
||||
@@ -794,7 +784,6 @@ fn process_aspa_with_issuer(
|
||||
Ok((attestation, local_output))
|
||||
}
|
||||
|
||||
|
||||
fn vrp_prefix_to_string(vrp: &Vrp) -> String {
|
||||
let prefix = &vrp.prefix;
|
||||
match prefix.afi {
|
||||
|
||||
@@ -15,8 +15,7 @@ use crate::replay::fetch_rsync::PayloadReplayRsyncFetcher;
|
||||
use crate::sync::rrdp::Fetcher;
|
||||
use crate::validation::from_tal::{
|
||||
DiscoveredRootCaInstance, FromTalError, discover_root_ca_instance_from_tal_and_ta_der,
|
||||
discover_root_ca_instance_from_tal_with_fetchers,
|
||||
discover_root_ca_instance_from_tal_url,
|
||||
discover_root_ca_instance_from_tal_url, discover_root_ca_instance_from_tal_with_fetchers,
|
||||
};
|
||||
use crate::validation::tree::{
|
||||
CaInstanceHandle, TreeRunAuditOutput, TreeRunConfig, TreeRunError, TreeRunOutput,
|
||||
@@ -307,8 +306,12 @@ pub fn run_tree_from_tal_bytes_serial_audit(
|
||||
config: &TreeRunConfig,
|
||||
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError> {
|
||||
let tal = crate::data_model::tal::Tal::decode_bytes(tal_bytes).map_err(FromTalError::from)?;
|
||||
let discovery =
|
||||
discover_root_ca_instance_from_tal_with_fetchers(http_fetcher, rsync_fetcher, tal, tal_uri)?;
|
||||
let discovery = discover_root_ca_instance_from_tal_with_fetchers(
|
||||
http_fetcher,
|
||||
rsync_fetcher,
|
||||
tal,
|
||||
tal_uri,
|
||||
)?;
|
||||
|
||||
let download_log = DownloadLogHandle::new();
|
||||
let runner = Rpkiv1PublicationPointRunner {
|
||||
@@ -362,8 +365,12 @@ pub fn run_tree_from_tal_bytes_serial_audit_with_timing(
|
||||
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError> {
|
||||
let _tal = timing.span_phase("tal_bootstrap");
|
||||
let tal = crate::data_model::tal::Tal::decode_bytes(tal_bytes).map_err(FromTalError::from)?;
|
||||
let discovery =
|
||||
discover_root_ca_instance_from_tal_with_fetchers(http_fetcher, rsync_fetcher, tal, tal_uri)?;
|
||||
let discovery = discover_root_ca_instance_from_tal_with_fetchers(
|
||||
http_fetcher,
|
||||
rsync_fetcher,
|
||||
tal,
|
||||
tal_uri,
|
||||
)?;
|
||||
drop(_tal);
|
||||
|
||||
let download_log = DownloadLogHandle::new();
|
||||
@@ -531,8 +538,11 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial(
|
||||
let discovery =
|
||||
discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri)?;
|
||||
let replay_index = Arc::new(
|
||||
ReplayArchiveIndex::load_allow_missing_rsync_modules(payload_archive_root, payload_locks_path)
|
||||
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?,
|
||||
ReplayArchiveIndex::load_allow_missing_rsync_modules(
|
||||
payload_archive_root,
|
||||
payload_locks_path,
|
||||
)
|
||||
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?,
|
||||
);
|
||||
let http_fetcher = PayloadReplayHttpFetcher::new(replay_index.clone())
|
||||
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?;
|
||||
@@ -579,8 +589,11 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit(
|
||||
let discovery =
|
||||
discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri)?;
|
||||
let replay_index = Arc::new(
|
||||
ReplayArchiveIndex::load_allow_missing_rsync_modules(payload_archive_root, payload_locks_path)
|
||||
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?,
|
||||
ReplayArchiveIndex::load_allow_missing_rsync_modules(
|
||||
payload_archive_root,
|
||||
payload_locks_path,
|
||||
)
|
||||
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?,
|
||||
);
|
||||
let http_fetcher = PayloadReplayHttpFetcher::new(replay_index.clone())
|
||||
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?;
|
||||
@@ -642,8 +655,11 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing(
|
||||
discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri)?;
|
||||
drop(_tal);
|
||||
let replay_index = Arc::new(
|
||||
ReplayArchiveIndex::load_allow_missing_rsync_modules(payload_archive_root, payload_locks_path)
|
||||
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?,
|
||||
ReplayArchiveIndex::load_allow_missing_rsync_modules(
|
||||
payload_archive_root,
|
||||
payload_locks_path,
|
||||
)
|
||||
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?,
|
||||
);
|
||||
let http_fetcher = PayloadReplayHttpFetcher::new(replay_index.clone())
|
||||
.map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?;
|
||||
@@ -1088,10 +1104,12 @@ mod replay_api_tests {
|
||||
.expect("read apnic tal fixture");
|
||||
let ta_der =
|
||||
std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture");
|
||||
let archive_root =
|
||||
std::path::PathBuf::from("../../rpki/target/live/20260316-112341-multi-final3/apnic/base-payload-archive");
|
||||
let locks_path =
|
||||
std::path::PathBuf::from("../../rpki/target/live/20260316-112341-multi-final3/apnic/base-locks.json");
|
||||
let archive_root = std::path::PathBuf::from(
|
||||
"../../rpki/target/live/20260316-112341-multi-final3/apnic/base-payload-archive",
|
||||
);
|
||||
let locks_path = std::path::PathBuf::from(
|
||||
"../../rpki/target/live/20260316-112341-multi-final3/apnic/base-locks.json",
|
||||
);
|
||||
let validation_time = time::OffsetDateTime::parse("2026-03-16T11:49:48+08:00", &Rfc3339)
|
||||
.expect("parse validation time");
|
||||
(tal_bytes, ta_der, archive_root, locks_path, validation_time)
|
||||
@@ -1160,16 +1178,14 @@ mod replay_api_tests {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db");
|
||||
let (tal_bytes, ta_der, archive_root, locks_path, validation_time) = apnic_replay_inputs();
|
||||
assert!(
|
||||
archive_root.is_dir(),
|
||||
"payload replay archive missing: {}",
|
||||
archive_root.display()
|
||||
);
|
||||
assert!(
|
||||
locks_path.is_file(),
|
||||
"payload replay locks missing: {}",
|
||||
locks_path.display()
|
||||
);
|
||||
if !archive_root.is_dir() || !locks_path.is_file() {
|
||||
eprintln!(
|
||||
"skipping payload replay api test; missing fixtures: archive={} locks={}",
|
||||
archive_root.display(),
|
||||
locks_path.display()
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let out = run_tree_from_tal_and_ta_der_payload_replay_serial_audit(
|
||||
&store,
|
||||
@@ -1198,15 +1214,22 @@ mod replay_api_tests {
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
#[test]
|
||||
fn payload_replay_api_root_only_apnic_multi_rir_bundle_runs_with_lenient_rsync_modules() {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db");
|
||||
let (tal_bytes, ta_der, archive_root, locks_path, validation_time) =
|
||||
apnic_multi_rir_replay_inputs();
|
||||
assert!(archive_root.is_dir(), "payload replay archive missing: {}", archive_root.display());
|
||||
assert!(locks_path.is_file(), "payload replay locks missing: {}", locks_path.display());
|
||||
assert!(
|
||||
archive_root.is_dir(),
|
||||
"payload replay archive missing: {}",
|
||||
archive_root.display()
|
||||
);
|
||||
assert!(
|
||||
locks_path.is_file(),
|
||||
"payload replay locks missing: {}",
|
||||
locks_path.display()
|
||||
);
|
||||
|
||||
let out = run_tree_from_tal_and_ta_der_payload_replay_serial_audit(
|
||||
&store,
|
||||
@@ -1235,16 +1258,14 @@ mod replay_api_tests {
|
||||
let db_path = temp.path().join("db");
|
||||
let store = crate::storage::RocksStore::open(&db_path).expect("open db");
|
||||
let (tal_bytes, ta_der, archive_root, locks_path, validation_time) = apnic_replay_inputs();
|
||||
assert!(
|
||||
archive_root.is_dir(),
|
||||
"payload replay archive missing: {}",
|
||||
archive_root.display()
|
||||
);
|
||||
assert!(
|
||||
locks_path.is_file(),
|
||||
"payload replay locks missing: {}",
|
||||
locks_path.display()
|
||||
);
|
||||
if !archive_root.is_dir() || !locks_path.is_file() {
|
||||
eprintln!(
|
||||
"skipping payload replay api timing test; missing fixtures: archive={} locks={}",
|
||||
archive_root.display(),
|
||||
locks_path.display()
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let timing = TimingHandle::new(TimingMeta {
|
||||
recorded_at_utc_rfc3339: "2026-03-13T03:00:00Z".to_string(),
|
||||
@@ -1366,26 +1387,20 @@ mod replay_api_tests {
|
||||
delta_locks,
|
||||
validation_time,
|
||||
) = apnic_delta_replay_inputs();
|
||||
assert!(
|
||||
base_archive.is_dir(),
|
||||
"base archive missing: {}",
|
||||
base_archive.display()
|
||||
);
|
||||
assert!(
|
||||
base_locks.is_file(),
|
||||
"base locks missing: {}",
|
||||
base_locks.display()
|
||||
);
|
||||
assert!(
|
||||
delta_archive.is_dir(),
|
||||
"delta archive missing: {}",
|
||||
delta_archive.display()
|
||||
);
|
||||
assert!(
|
||||
delta_locks.is_file(),
|
||||
"delta locks missing: {}",
|
||||
delta_locks.display()
|
||||
);
|
||||
if !base_archive.is_dir()
|
||||
|| !base_locks.is_file()
|
||||
|| !delta_archive.is_dir()
|
||||
|| !delta_locks.is_file()
|
||||
{
|
||||
eprintln!(
|
||||
"skipping payload delta replay api test; missing fixtures: base_archive={} base_locks={} delta_archive={} delta_locks={}",
|
||||
base_archive.display(),
|
||||
base_locks.display(),
|
||||
delta_archive.display(),
|
||||
delta_locks.display()
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let out = run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit(
|
||||
&store,
|
||||
@@ -1425,6 +1440,20 @@ mod replay_api_tests {
|
||||
delta_locks,
|
||||
validation_time,
|
||||
) = apnic_delta_replay_inputs();
|
||||
if !base_archive.is_dir()
|
||||
|| !base_locks.is_file()
|
||||
|| !delta_archive.is_dir()
|
||||
|| !delta_locks.is_file()
|
||||
{
|
||||
eprintln!(
|
||||
"skipping payload delta replay timing test; missing fixtures: base_archive={} base_locks={} delta_archive={} delta_locks={}",
|
||||
base_archive.display(),
|
||||
base_locks.display(),
|
||||
delta_archive.display(),
|
||||
delta_locks.display()
|
||||
);
|
||||
return;
|
||||
}
|
||||
let timing = TimingHandle::new(TimingMeta {
|
||||
recorded_at_utc_rfc3339: "2026-03-16T00:00:00Z".to_string(),
|
||||
validation_time_utc_rfc3339: "2026-03-15T10:00:00Z".to_string(),
|
||||
|
||||
@@ -10,8 +10,8 @@ use crate::data_model::manifest::ManifestObject;
|
||||
use crate::data_model::rc::ResourceCertificate;
|
||||
use crate::data_model::roa::{RoaAfi, RoaObject};
|
||||
use crate::data_model::router_cert::{
|
||||
BgpsecRouterCertificate, BgpsecRouterCertificateDecodeError,
|
||||
BgpsecRouterCertificatePathError, BgpsecRouterCertificateProfileError,
|
||||
BgpsecRouterCertificate, BgpsecRouterCertificateDecodeError, BgpsecRouterCertificatePathError,
|
||||
BgpsecRouterCertificateProfileError,
|
||||
};
|
||||
use crate::fetch::rsync::RsyncFetcher;
|
||||
use crate::policy::Policy;
|
||||
@@ -19,10 +19,9 @@ use crate::replay::archive::ReplayArchiveIndex;
|
||||
use crate::replay::delta_archive::ReplayDeltaArchiveIndex;
|
||||
use crate::report::{RfcRef, Warning};
|
||||
use crate::storage::{
|
||||
PackFile, PackTime, RawByHashEntry, RocksStore,
|
||||
ValidatedCaInstanceResult, VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus,
|
||||
VcirAuditSummary, VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirOutputType,
|
||||
VcirRelatedArtifact, VcirSummary,
|
||||
PackFile, PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, VcirArtifactKind,
|
||||
VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, VcirChildEntry,
|
||||
VcirInstanceGate, VcirLocalOutput, VcirOutputType, VcirRelatedArtifact, VcirSummary,
|
||||
};
|
||||
use crate::sync::repo::{
|
||||
sync_publication_point, sync_publication_point_replay, sync_publication_point_replay_delta,
|
||||
@@ -37,7 +36,9 @@ use crate::validation::manifest::{
|
||||
ManifestFreshError, PublicationPointData, PublicationPointSource,
|
||||
process_manifest_publication_point_fresh_after_repo_sync,
|
||||
};
|
||||
use crate::validation::objects::{AspaAttestation, RouterKeyPayload, Vrp, process_publication_point_for_issuer};
|
||||
use crate::validation::objects::{
|
||||
AspaAttestation, RouterKeyPayload, Vrp, process_publication_point_for_issuer,
|
||||
};
|
||||
use crate::validation::publication_point::PublicationPointSnapshot;
|
||||
use crate::validation::tree::{
|
||||
CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult, PublicationPointRunner,
|
||||
@@ -45,8 +46,8 @@ use crate::validation::tree::{
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use serde::Deserialize;
|
||||
use base64::Engine as _;
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use x509_parser::prelude::FromDer;
|
||||
use x509_parser::x509::SubjectPublicKeyInfo;
|
||||
@@ -168,13 +169,19 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
}
|
||||
|
||||
let repo_sync_started = std::time::Instant::now();
|
||||
let (repo_sync_ok, repo_sync_err, repo_sync_source): (bool, Option<String>, Option<String>) = if skip_sync_due_to_dedup {
|
||||
let (repo_sync_ok, repo_sync_err, repo_sync_source, repo_sync_phase): (
|
||||
bool,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
) = if skip_sync_due_to_dedup {
|
||||
let source = if effective_notification_uri.is_some() {
|
||||
Some("rrdp_dedup_skip".to_string())
|
||||
} else {
|
||||
Some("rsync_dedup_skip".to_string())
|
||||
};
|
||||
(true, None, source)
|
||||
let phase = source.clone();
|
||||
(true, None, source, phase)
|
||||
} else {
|
||||
let repo_key = effective_notification_uri.unwrap_or_else(|| ca.rsync_base_uri.as_str());
|
||||
let _repo_total = self
|
||||
@@ -249,7 +256,12 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
}
|
||||
|
||||
warnings.extend(res.warnings);
|
||||
(true, None, Some(repo_sync_source_label(res.source).to_string()))
|
||||
(
|
||||
true,
|
||||
None,
|
||||
Some(repo_sync_source_label(res.source).to_string()),
|
||||
Some(repo_sync_phase_label(res.phase).to_string()),
|
||||
)
|
||||
}
|
||||
Err(e) => {
|
||||
if attempted_rrdp && self.rrdp_dedup {
|
||||
@@ -267,7 +279,19 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
.with_rfc_refs(&[RfcRef("RFC 8182 §3.4.5"), RfcRef("RFC 9286 §6.6")])
|
||||
.with_context(&ca.rsync_base_uri),
|
||||
);
|
||||
(false, Some(e.to_string()), None)
|
||||
(
|
||||
false,
|
||||
Some(e.to_string()),
|
||||
None,
|
||||
Some(
|
||||
repo_sync_failure_phase_label(
|
||||
attempted_rrdp,
|
||||
original_notification_uri,
|
||||
effective_notification_uri,
|
||||
)
|
||||
.to_string(),
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -279,6 +303,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"repo_sync_ok": repo_sync_ok,
|
||||
"repo_sync_source": repo_sync_source,
|
||||
"repo_sync_phase": repo_sync_phase,
|
||||
"repo_sync_error": repo_sync_err,
|
||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||
}),
|
||||
@@ -367,6 +392,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
ca,
|
||||
PublicationPointSource::Fresh,
|
||||
repo_sync_source.as_deref(),
|
||||
repo_sync_phase.as_deref(),
|
||||
Some(repo_sync_duration_ms),
|
||||
repo_sync_err.as_deref(),
|
||||
&pack,
|
||||
@@ -390,6 +416,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"source": "fresh",
|
||||
"repo_sync_source": repo_sync_source,
|
||||
"repo_sync_phase": repo_sync_phase,
|
||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||
"total_duration_ms": total_duration_ms,
|
||||
"warning_count": result.warnings.len(),
|
||||
@@ -399,7 +426,8 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
"child_count": result.discovered_children.len(),
|
||||
}),
|
||||
);
|
||||
if (total_duration_ms as f64) / 1000.0 >= crate::progress_log::slow_threshold_secs() {
|
||||
if (total_duration_ms as f64) / 1000.0 >= crate::progress_log::slow_threshold_secs()
|
||||
{
|
||||
crate::progress_log::emit(
|
||||
"publication_point_slow",
|
||||
serde_json::json!({
|
||||
@@ -407,6 +435,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"source": "fresh",
|
||||
"repo_sync_source": repo_sync_source,
|
||||
"repo_sync_phase": repo_sync_phase,
|
||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||
"total_duration_ms": total_duration_ms,
|
||||
}),
|
||||
@@ -424,11 +453,25 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"source": "error",
|
||||
"repo_sync_source": repo_sync_source,
|
||||
"repo_sync_phase": repo_sync_phase,
|
||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||
"total_duration_ms": total_duration_ms,
|
||||
"error": fresh_err.to_string(),
|
||||
}),
|
||||
);
|
||||
crate::progress_log::emit(
|
||||
"repo_terminal_failure",
|
||||
serde_json::json!({
|
||||
"manifest_rsync_uri": ca.manifest_rsync_uri,
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"repo_sync_source": repo_sync_source,
|
||||
"repo_sync_phase": repo_sync_phase,
|
||||
"repo_sync_error": repo_sync_err,
|
||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||
"terminal_state": "stop_all_output",
|
||||
"error": fresh_err.to_string(),
|
||||
}),
|
||||
);
|
||||
Err(format!("{fresh_err}"))
|
||||
}
|
||||
crate::policy::CaFailedFetchPolicy::ReuseCurrentInstanceVcir => {
|
||||
@@ -444,6 +487,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
ca,
|
||||
projection.source,
|
||||
repo_sync_source.as_deref(),
|
||||
repo_sync_phase.as_deref(),
|
||||
Some(repo_sync_duration_ms),
|
||||
repo_sync_err.as_deref(),
|
||||
projection.vcir.as_ref(),
|
||||
@@ -468,6 +512,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"source": source_label(result.source),
|
||||
"repo_sync_source": repo_sync_source,
|
||||
"repo_sync_phase": repo_sync_phase,
|
||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||
"total_duration_ms": total_duration_ms,
|
||||
"warning_count": result.warnings.len(),
|
||||
@@ -477,7 +522,55 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
"child_count": result.discovered_children.len(),
|
||||
}),
|
||||
);
|
||||
if (total_duration_ms as f64) / 1000.0 >= crate::progress_log::slow_threshold_secs() {
|
||||
match result.source {
|
||||
PublicationPointSource::VcirCurrentInstance if !repo_sync_ok => {
|
||||
crate::progress_log::emit(
|
||||
"rsync_failed_fallback_current_instance",
|
||||
serde_json::json!({
|
||||
"manifest_rsync_uri": ca.manifest_rsync_uri,
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"repo_sync_source": repo_sync_source,
|
||||
"repo_sync_phase": repo_sync_phase,
|
||||
"repo_sync_error": repo_sync_err,
|
||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||
"terminal_state": "fallback_current_instance",
|
||||
}),
|
||||
);
|
||||
}
|
||||
PublicationPointSource::FailedFetchNoCache => {
|
||||
if !repo_sync_ok {
|
||||
crate::progress_log::emit(
|
||||
"rsync_failed_no_cache",
|
||||
serde_json::json!({
|
||||
"manifest_rsync_uri": ca.manifest_rsync_uri,
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"repo_sync_source": repo_sync_source,
|
||||
"repo_sync_phase": repo_sync_phase,
|
||||
"repo_sync_error": repo_sync_err,
|
||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||
"terminal_state": "failed_no_cache",
|
||||
}),
|
||||
);
|
||||
}
|
||||
crate::progress_log::emit(
|
||||
"repo_terminal_failure",
|
||||
serde_json::json!({
|
||||
"manifest_rsync_uri": ca.manifest_rsync_uri,
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"repo_sync_source": repo_sync_source,
|
||||
"repo_sync_phase": repo_sync_phase,
|
||||
"repo_sync_error": repo_sync_err,
|
||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||
"terminal_state": "failed_no_cache",
|
||||
}),
|
||||
);
|
||||
}
|
||||
PublicationPointSource::Fresh => {}
|
||||
PublicationPointSource::VcirCurrentInstance => {}
|
||||
}
|
||||
if (total_duration_ms as f64) / 1000.0
|
||||
>= crate::progress_log::slow_threshold_secs()
|
||||
{
|
||||
crate::progress_log::emit(
|
||||
"publication_point_slow",
|
||||
serde_json::json!({
|
||||
@@ -485,6 +578,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"source": source_label(result.source),
|
||||
"repo_sync_source": repo_sync_source,
|
||||
"repo_sync_phase": repo_sync_phase,
|
||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||
"total_duration_ms": total_duration_ms,
|
||||
}),
|
||||
@@ -856,9 +950,7 @@ fn discover_children_from_fresh_snapshot_with_audit<P: PublicationPointData>(
|
||||
sha256_hex: sha256_hex_from_32(&f.sha256),
|
||||
kind: AuditObjectKind::RouterCertificate,
|
||||
result: AuditObjectResult::Error,
|
||||
detail: Some(format!(
|
||||
"router certificate validation failed: {err}"
|
||||
)),
|
||||
detail: Some(format!("router certificate validation failed: {err}")),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -977,7 +1069,10 @@ fn discover_children_from_fresh_snapshot_with_audit<P: PublicationPointData>(
|
||||
t.record_phase_nanos("child_select_issuer_crl_total", select_crl_nanos);
|
||||
t.record_phase_nanos("child_decode_certificate_total", child_decode_nanos);
|
||||
t.record_phase_nanos("child_validate_subordinate_total", validate_sub_ca_nanos);
|
||||
t.record_phase_nanos("child_validate_router_certificate_total", validate_router_nanos);
|
||||
t.record_phase_nanos(
|
||||
"child_validate_router_certificate_total",
|
||||
validate_router_nanos,
|
||||
);
|
||||
t.record_phase_nanos("child_ca_instance_uri_discovery_total", uri_discovery_nanos);
|
||||
t.record_phase_nanos("child_enqueue_total", enqueue_nanos);
|
||||
}
|
||||
@@ -1150,6 +1245,43 @@ fn source_label(source: PublicationPointSource) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
fn repo_sync_phase_label(phase: crate::sync::repo::RepoSyncPhase) -> &'static str {
|
||||
match phase {
|
||||
crate::sync::repo::RepoSyncPhase::RrdpOk => "rrdp_ok",
|
||||
crate::sync::repo::RepoSyncPhase::RrdpFailedRsyncOk => "rrdp_failed_rsync_ok",
|
||||
crate::sync::repo::RepoSyncPhase::RsyncOnlyOk => "rsync_only_ok",
|
||||
crate::sync::repo::RepoSyncPhase::ReplayRrdpOk => "replay_rrdp_ok",
|
||||
crate::sync::repo::RepoSyncPhase::ReplayRsyncOk => "replay_rsync_ok",
|
||||
crate::sync::repo::RepoSyncPhase::ReplayNoopRrdp => "replay_noop_rrdp",
|
||||
crate::sync::repo::RepoSyncPhase::ReplayNoopRsync => "replay_noop_rsync",
|
||||
}
|
||||
}
|
||||
|
||||
fn repo_sync_failure_phase_label(
|
||||
attempted_rrdp: bool,
|
||||
original_notification_uri: Option<&str>,
|
||||
effective_notification_uri: Option<&str>,
|
||||
) -> &'static str {
|
||||
if attempted_rrdp && original_notification_uri.is_some() && effective_notification_uri.is_some() {
|
||||
"rrdp_failed_rsync_failed"
|
||||
} else if attempted_rrdp
|
||||
&& original_notification_uri.is_some()
|
||||
&& effective_notification_uri.is_none()
|
||||
{
|
||||
"rsync_only_failed_after_rrdp_dedup"
|
||||
} else {
|
||||
"rsync_only_failed"
|
||||
}
|
||||
}
|
||||
|
||||
fn terminal_state_label(source: PublicationPointSource) -> &'static str {
|
||||
match source {
|
||||
PublicationPointSource::Fresh => "fresh",
|
||||
PublicationPointSource::VcirCurrentInstance => "fallback_current_instance",
|
||||
PublicationPointSource::FailedFetchNoCache => "failed_no_cache",
|
||||
}
|
||||
}
|
||||
|
||||
fn repo_sync_source_label(source: crate::sync::repo::RepoSyncSource) -> &'static str {
|
||||
match source {
|
||||
crate::sync::repo::RepoSyncSource::Rrdp => "rrdp",
|
||||
@@ -1182,6 +1314,7 @@ fn build_publication_point_audit_from_snapshot(
|
||||
ca: &CaInstanceHandle,
|
||||
source: PublicationPointSource,
|
||||
repo_sync_source: Option<&str>,
|
||||
repo_sync_phase: Option<&str>,
|
||||
repo_sync_duration_ms: Option<u64>,
|
||||
repo_sync_error: Option<&str>,
|
||||
pack: &PublicationPointSnapshot,
|
||||
@@ -1275,8 +1408,10 @@ fn build_publication_point_audit_from_snapshot(
|
||||
rrdp_notification_uri: ca.rrdp_notification_uri.clone(),
|
||||
source: source_label(source),
|
||||
repo_sync_source: repo_sync_source.map(ToString::to_string),
|
||||
repo_sync_phase: repo_sync_phase.map(ToString::to_string),
|
||||
repo_sync_duration_ms,
|
||||
repo_sync_error: repo_sync_error.map(ToString::to_string),
|
||||
repo_terminal_state: terminal_state_label(source).to_string(),
|
||||
this_update_rfc3339_utc: pack.this_update.rfc3339_utc.clone(),
|
||||
next_update_rfc3339_utc: pack.next_update.rfc3339_utc.clone(),
|
||||
verified_at_rfc3339_utc: pack.verified_at.rfc3339_utc.clone(),
|
||||
@@ -1289,6 +1424,7 @@ fn build_publication_point_audit_from_vcir(
|
||||
ca: &CaInstanceHandle,
|
||||
source: PublicationPointSource,
|
||||
repo_sync_source: Option<&str>,
|
||||
repo_sync_phase: Option<&str>,
|
||||
repo_sync_duration_ms: Option<u64>,
|
||||
repo_sync_error: Option<&str>,
|
||||
vcir: Option<&ValidatedCaInstanceResult>,
|
||||
@@ -1302,6 +1438,7 @@ fn build_publication_point_audit_from_vcir(
|
||||
ca,
|
||||
source,
|
||||
repo_sync_source,
|
||||
repo_sync_phase,
|
||||
repo_sync_duration_ms,
|
||||
repo_sync_error,
|
||||
pack,
|
||||
@@ -1326,8 +1463,10 @@ fn build_publication_point_audit_from_vcir(
|
||||
rrdp_notification_uri: ca.rrdp_notification_uri.clone(),
|
||||
source: source_label(source),
|
||||
repo_sync_source: repo_sync_source.map(ToString::to_string),
|
||||
repo_sync_phase: repo_sync_phase.map(ToString::to_string),
|
||||
repo_sync_duration_ms,
|
||||
repo_sync_error: repo_sync_error.map(ToString::to_string),
|
||||
repo_terminal_state: terminal_state_label(source).to_string(),
|
||||
this_update_rfc3339_utc: String::new(),
|
||||
next_update_rfc3339_utc: String::new(),
|
||||
verified_at_rfc3339_utc: String::new(),
|
||||
@@ -1405,13 +1544,15 @@ fn build_publication_point_audit_from_vcir(
|
||||
manifest_rsync_uri: ca.manifest_rsync_uri.clone(),
|
||||
publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(),
|
||||
rrdp_notification_uri: ca.rrdp_notification_uri.clone(),
|
||||
source: source_label(source),
|
||||
repo_sync_source: repo_sync_source.map(ToString::to_string),
|
||||
repo_sync_duration_ms,
|
||||
repo_sync_error: repo_sync_error.map(ToString::to_string),
|
||||
this_update_rfc3339_utc: vcir
|
||||
.validated_manifest_meta
|
||||
.validated_manifest_this_update
|
||||
source: source_label(source),
|
||||
repo_sync_source: repo_sync_source.map(ToString::to_string),
|
||||
repo_sync_phase: repo_sync_phase.map(ToString::to_string),
|
||||
repo_sync_duration_ms,
|
||||
repo_sync_error: repo_sync_error.map(ToString::to_string),
|
||||
repo_terminal_state: terminal_state_label(source).to_string(),
|
||||
this_update_rfc3339_utc: vcir
|
||||
.validated_manifest_meta
|
||||
.validated_manifest_this_update
|
||||
.rfc3339_utc
|
||||
.clone(),
|
||||
next_update_rfc3339_utc: vcir
|
||||
@@ -1790,7 +1931,9 @@ fn build_objects_output_from_vcir(
|
||||
sha256_hex: local.source_object_hash.clone(),
|
||||
kind: AuditObjectKind::RouterCertificate,
|
||||
result: AuditObjectResult::Error,
|
||||
detail: Some(format!("cached Router Key local output parse failed: {e}")),
|
||||
detail: Some(format!(
|
||||
"cached Router Key local output parse failed: {e}"
|
||||
)),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -1830,9 +1973,10 @@ fn parse_vcir_aspa_output(local: &VcirLocalOutput) -> Result<AspaAttestation, St
|
||||
fn parse_vcir_router_key_output(local: &VcirLocalOutput) -> Result<RouterKeyPayload, String> {
|
||||
let payload: VcirRouterKeyPayload = serde_json::from_str(&local.payload_json)
|
||||
.map_err(|e| format!("invalid Router Key payload JSON: {e}"))?;
|
||||
let ski = hex::decode(&payload.ski_hex)
|
||||
.map_err(|e| format!("invalid Router Key SKI hex: {e}"))?;
|
||||
let spki_der = base64::engine::general_purpose::STANDARD.decode(&payload.spki_der_base64)
|
||||
let ski =
|
||||
hex::decode(&payload.ski_hex).map_err(|e| format!("invalid Router Key SKI hex: {e}"))?;
|
||||
let spki_der = base64::engine::general_purpose::STANDARD
|
||||
.decode(&payload.spki_der_base64)
|
||||
.map_err(|e| format!("invalid Router Key SPKI base64: {e}"))?;
|
||||
Ok(RouterKeyPayload {
|
||||
as_id: payload.as_id,
|
||||
@@ -2255,7 +2399,8 @@ fn build_router_key_local_outputs(
|
||||
.iter()
|
||||
.map(|router_key| {
|
||||
let ski_hex = hex::encode(&router_key.ski);
|
||||
let spki_der_base64 = base64::engine::general_purpose::STANDARD.encode(&router_key.spki_der);
|
||||
let spki_der_base64 =
|
||||
base64::engine::general_purpose::STANDARD.encode(&router_key.spki_der);
|
||||
let rule_hash = sha256_hex(
|
||||
format!(
|
||||
"router-key-rule:{}:{}:{}:{}",
|
||||
@@ -2587,7 +2732,6 @@ fn audit_result_to_vcir_status(result: &AuditObjectResult) -> VcirArtifactValida
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fn roa_to_vrps_for_vcir(roa: &RoaObject) -> Vec<Vrp> {
|
||||
let asn = roa.roa.as_id;
|
||||
let mut out = Vec::new();
|
||||
@@ -2629,10 +2773,10 @@ mod tests {
|
||||
use crate::fetch::rsync::LocalDirRsyncFetcher;
|
||||
use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher};
|
||||
use crate::storage::{
|
||||
PackFile, PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, ValidatedManifestMeta,
|
||||
VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary,
|
||||
VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirOutputType, VcirRelatedArtifact,
|
||||
VcirSummary,
|
||||
PackFile, PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult,
|
||||
ValidatedManifestMeta, VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus,
|
||||
VcirAuditSummary, VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirOutputType,
|
||||
VcirRelatedArtifact, VcirSummary,
|
||||
};
|
||||
use crate::sync::rrdp::Fetcher;
|
||||
use crate::validation::publication_point::PublicationPointSnapshot;
|
||||
@@ -2845,8 +2989,6 @@ authorityKeyIdentifier = keyid:always
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
struct GeneratedRouter {
|
||||
issuer_ca_der: Vec<u8>,
|
||||
router_der: Vec<u8>,
|
||||
@@ -3045,15 +3187,16 @@ authorityKeyIdentifier = keyid:always
|
||||
}
|
||||
}
|
||||
|
||||
fn cernet_publication_point_snapshot_for_vcir_tests(
|
||||
) -> (PublicationPointSnapshot, Vec<u8>, time::OffsetDateTime) {
|
||||
fn cernet_publication_point_snapshot_for_vcir_tests()
|
||||
-> (PublicationPointSnapshot, Vec<u8>, time::OffsetDateTime) {
|
||||
let dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0");
|
||||
let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/";
|
||||
let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft";
|
||||
let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}");
|
||||
let manifest_bytes = std::fs::read(dir.join(manifest_file)).expect("read manifest fixture");
|
||||
let manifest = ManifestObject::decode_der(&manifest_bytes).expect("decode manifest fixture");
|
||||
let manifest =
|
||||
ManifestObject::decode_der(&manifest_bytes).expect("decode manifest fixture");
|
||||
let candidate = manifest.manifest.this_update + time::Duration::seconds(60);
|
||||
let validation_time = if candidate < manifest.manifest.next_update {
|
||||
candidate
|
||||
@@ -3320,7 +3463,7 @@ authorityKeyIdentifier = keyid:always
|
||||
&crate::validation::objects::ObjectsOutput {
|
||||
vrps: Vec::new(),
|
||||
aspas: Vec::new(),
|
||||
router_keys: Vec::new(),
|
||||
router_keys: Vec::new(),
|
||||
local_outputs_cache: cached.clone(),
|
||||
warnings: Vec::new(),
|
||||
stats: crate::validation::objects::ObjectsStats::default(),
|
||||
@@ -3333,22 +3476,32 @@ authorityKeyIdentifier = keyid:always
|
||||
|
||||
#[test]
|
||||
fn collect_and_persist_vcir_embedded_evidence_for_real_signed_objects() {
|
||||
let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests();
|
||||
let (pack, issuer_ca_der, validation_time) =
|
||||
cernet_publication_point_snapshot_for_vcir_tests();
|
||||
let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca");
|
||||
let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer(
|
||||
&pack,
|
||||
&Policy::default(),
|
||||
issuer_ca_der.as_slice(),
|
||||
Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer"),
|
||||
Some(
|
||||
"rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer",
|
||||
),
|
||||
issuer_ca.tbs.extensions.ip_resources.as_ref(),
|
||||
issuer_ca.tbs.extensions.as_resources.as_ref(),
|
||||
validation_time,
|
||||
None,
|
||||
);
|
||||
assert!(!objects.local_outputs_cache.is_empty(), "expected local outputs from signed objects");
|
||||
assert!(
|
||||
!objects.local_outputs_cache.is_empty(),
|
||||
"expected local outputs from signed objects"
|
||||
);
|
||||
|
||||
let evidence = collect_vcir_embedded_evidence(&pack, &objects).expect("collect embedded evidence");
|
||||
assert!(evidence.len() >= 2, "expected manifest EE and signed-object EE evidence");
|
||||
let evidence =
|
||||
collect_vcir_embedded_evidence(&pack, &objects).expect("collect embedded evidence");
|
||||
assert!(
|
||||
evidence.len() >= 2,
|
||||
"expected manifest EE and signed-object EE evidence"
|
||||
);
|
||||
|
||||
let store_dir = tempfile::tempdir().expect("store dir");
|
||||
let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
|
||||
@@ -3375,15 +3528,19 @@ authorityKeyIdentifier = keyid:always
|
||||
.get_raw_by_hash_entry(&issuer_hash)
|
||||
.expect("load issuer raw entry")
|
||||
.expect("issuer raw entry present");
|
||||
assert!(issuer_entry
|
||||
.origin_uris
|
||||
.iter()
|
||||
.any(|uri| uri.ends_with("BfycW4hQb3wNP4YsiJW-1n6fjro.cer")));
|
||||
assert!(
|
||||
issuer_entry
|
||||
.origin_uris
|
||||
.iter()
|
||||
.any(|uri| uri.ends_with("BfycW4hQb3wNP4YsiJW-1n6fjro.cer"))
|
||||
);
|
||||
for entry in &evidence {
|
||||
assert!(store
|
||||
.get_raw_by_hash_entry(&entry.raw_entry.sha256_hex)
|
||||
.expect("load evidence raw entry")
|
||||
.is_some());
|
||||
assert!(
|
||||
store
|
||||
.get_raw_by_hash_entry(&entry.raw_entry.sha256_hex)
|
||||
.expect("load evidence raw entry")
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3411,7 +3568,9 @@ authorityKeyIdentifier = keyid:always
|
||||
source_object_uri: "rsync://example.test/repo/issuer/router.cer".to_string(),
|
||||
source_object_hash: "11".repeat(32),
|
||||
source_ee_cert_hash: "11".repeat(32),
|
||||
item_effective_until: PackTime { rfc3339_utc: "2026-12-31T00:00:00Z".to_string() },
|
||||
item_effective_until: PackTime {
|
||||
rfc3339_utc: "2026-12-31T00:00:00Z".to_string(),
|
||||
},
|
||||
}],
|
||||
);
|
||||
assert_eq!(outputs.len(), 1);
|
||||
@@ -3422,13 +3581,16 @@ authorityKeyIdentifier = keyid:always
|
||||
|
||||
#[test]
|
||||
fn build_vcir_local_outputs_falls_back_to_decoding_accepted_objects_when_cache_is_empty() {
|
||||
let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests();
|
||||
let (pack, issuer_ca_der, validation_time) =
|
||||
cernet_publication_point_snapshot_for_vcir_tests();
|
||||
let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca");
|
||||
let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer(
|
||||
&pack,
|
||||
&Policy::default(),
|
||||
issuer_ca_der.as_slice(),
|
||||
Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer"),
|
||||
Some(
|
||||
"rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer",
|
||||
),
|
||||
issuer_ca.tbs.extensions.ip_resources.as_ref(),
|
||||
issuer_ca.tbs.extensions.as_resources.as_ref(),
|
||||
validation_time,
|
||||
@@ -3455,20 +3617,25 @@ authorityKeyIdentifier = keyid:always
|
||||
.expect("rebuild vcir local outputs");
|
||||
assert!(!local_outputs.is_empty());
|
||||
assert_eq!(local_outputs.len(), objects.vrps.len());
|
||||
assert!(local_outputs
|
||||
.iter()
|
||||
.all(|output| output.output_type == VcirOutputType::Vrp));
|
||||
assert!(
|
||||
local_outputs
|
||||
.iter()
|
||||
.all(|output| output.output_type == VcirOutputType::Vrp)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persist_vcir_for_fresh_result_stores_vcir_and_audit_indexes_for_real_snapshot() {
|
||||
let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests();
|
||||
let (pack, issuer_ca_der, validation_time) =
|
||||
cernet_publication_point_snapshot_for_vcir_tests();
|
||||
let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca");
|
||||
let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer(
|
||||
&pack,
|
||||
&Policy::default(),
|
||||
issuer_ca_der.as_slice(),
|
||||
Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer"),
|
||||
Some(
|
||||
"rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer",
|
||||
),
|
||||
issuer_ca.tbs.extensions.ip_resources.as_ref(),
|
||||
issuer_ca.tbs.extensions.as_resources.as_ref(),
|
||||
validation_time,
|
||||
@@ -3492,17 +3659,8 @@ authorityKeyIdentifier = keyid:always
|
||||
rrdp_notification_uri: None,
|
||||
};
|
||||
|
||||
persist_vcir_for_fresh_result(
|
||||
&store,
|
||||
&ca,
|
||||
&pack,
|
||||
&objects,
|
||||
&[],
|
||||
&[],
|
||||
&[],
|
||||
validation_time,
|
||||
)
|
||||
.expect("persist vcir for fresh result");
|
||||
persist_vcir_for_fresh_result(&store, &ca, &pack, &objects, &[], &[], &[], validation_time)
|
||||
.expect("persist vcir for fresh result");
|
||||
|
||||
let vcir = store
|
||||
.get_vcir(&pack.manifest_rsync_uri)
|
||||
@@ -3511,10 +3669,15 @@ authorityKeyIdentifier = keyid:always
|
||||
assert_eq!(vcir.manifest_rsync_uri, pack.manifest_rsync_uri);
|
||||
assert_eq!(vcir.summary.local_vrp_count as usize, objects.vrps.len());
|
||||
let first_output = vcir.local_outputs.first().expect("local outputs stored");
|
||||
assert!(store
|
||||
.get_audit_rule_index_entry(crate::storage::AuditRuleKind::Roa, &first_output.rule_hash)
|
||||
.expect("get audit rule index entry")
|
||||
.is_some());
|
||||
assert!(
|
||||
store
|
||||
.get_audit_rule_index_entry(
|
||||
crate::storage::AuditRuleKind::Roa,
|
||||
&first_output.rule_hash
|
||||
)
|
||||
.expect("get audit rule index entry")
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3574,7 +3737,9 @@ authorityKeyIdentifier = keyid:always
|
||||
tal_id: "test-tal".to_string(),
|
||||
parent_manifest_rsync_uri: None,
|
||||
ca_certificate_der: vec![0x11, 0x22],
|
||||
ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()),
|
||||
ca_certificate_rsync_uri: Some(
|
||||
"rsync://example.test/repo/issuer/issuer.cer".to_string(),
|
||||
),
|
||||
effective_ip_resources: None,
|
||||
effective_as_resources: None,
|
||||
rsync_base_uri: pack.publication_point_rsync_uri.clone(),
|
||||
@@ -3625,21 +3790,37 @@ authorityKeyIdentifier = keyid:always
|
||||
&[],
|
||||
&embedded,
|
||||
);
|
||||
assert!(artifacts.iter().any(|artifact| artifact.artifact_role == VcirArtifactRole::Manifest));
|
||||
assert!(artifacts.iter().any(|artifact| artifact.artifact_role == VcirArtifactRole::TrustAnchorCert));
|
||||
assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() == Some("rsync://example.test/repo/issuer/issuer.crl")
|
||||
assert!(
|
||||
artifacts
|
||||
.iter()
|
||||
.any(|artifact| artifact.artifact_role == VcirArtifactRole::Manifest)
|
||||
);
|
||||
assert!(
|
||||
artifacts
|
||||
.iter()
|
||||
.any(|artifact| artifact.artifact_role == VcirArtifactRole::TrustAnchorCert)
|
||||
);
|
||||
assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref()
|
||||
== Some("rsync://example.test/repo/issuer/issuer.crl")
|
||||
&& artifact.artifact_role == VcirArtifactRole::CurrentCrl));
|
||||
assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() == Some("rsync://example.test/repo/issuer/child.cer")
|
||||
assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref()
|
||||
== Some("rsync://example.test/repo/issuer/child.cer")
|
||||
&& artifact.artifact_role == VcirArtifactRole::ChildCaCert));
|
||||
assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() == Some("rsync://example.test/repo/issuer/a.roa")
|
||||
assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref()
|
||||
== Some("rsync://example.test/repo/issuer/a.roa")
|
||||
&& artifact.validation_status == VcirArtifactValidationStatus::Rejected));
|
||||
assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() == Some("rsync://example.test/repo/issuer/a.asa")
|
||||
assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref()
|
||||
== Some("rsync://example.test/repo/issuer/a.asa")
|
||||
&& artifact.validation_status == VcirArtifactValidationStatus::WarningOnly));
|
||||
assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() == Some("rsync://example.test/repo/issuer/a.gbr")
|
||||
assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref()
|
||||
== Some("rsync://example.test/repo/issuer/a.gbr")
|
||||
&& artifact.artifact_kind == VcirArtifactKind::Gbr));
|
||||
assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() == Some("rsync://example.test/repo/issuer/extra.bin")
|
||||
assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref()
|
||||
== Some("rsync://example.test/repo/issuer/extra.bin")
|
||||
&& artifact.artifact_kind == VcirArtifactKind::Other));
|
||||
assert!(artifacts.iter().any(|artifact| artifact.uri.is_none() && artifact.sha256 == sha256_hex(b"embedded-ee")));
|
||||
assert!(artifacts.iter().any(
|
||||
|artifact| artifact.uri.is_none() && artifact.sha256 == sha256_hex(b"embedded-ee")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4130,7 +4311,11 @@ authorityKeyIdentifier = keyid:always
|
||||
PublicationPointSource::Fresh | PublicationPointSource::VcirCurrentInstance
|
||||
));
|
||||
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1, "module-scope dedup should skip second sync");
|
||||
assert_eq!(
|
||||
calls.load(Ordering::SeqCst),
|
||||
1,
|
||||
"module-scope dedup should skip second sync"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4385,12 +4570,15 @@ authorityKeyIdentifier = keyid:always
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
&pp.snapshot,
|
||||
&[],
|
||||
&objects,
|
||||
&[],
|
||||
);
|
||||
assert_eq!(audit.source, "vcir_current_instance");
|
||||
assert_eq!(audit.repo_sync_phase, None);
|
||||
assert_eq!(audit.repo_terminal_state, "fallback_current_instance");
|
||||
assert!(
|
||||
audit
|
||||
.objects
|
||||
@@ -4453,15 +4641,21 @@ authorityKeyIdentifier = keyid:always
|
||||
let audit = build_publication_point_audit_from_snapshot(
|
||||
&issuer,
|
||||
pp.source,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some("rsync"),
|
||||
Some("rsync_only_ok"),
|
||||
Some(123),
|
||||
Some("none"),
|
||||
&pp.snapshot,
|
||||
&[],
|
||||
&objects,
|
||||
&[],
|
||||
);
|
||||
assert_eq!(audit.objects[0].kind, AuditObjectKind::Manifest);
|
||||
assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync"));
|
||||
assert_eq!(audit.repo_sync_phase.as_deref(), Some("rsync_only_ok"));
|
||||
assert_eq!(audit.repo_sync_duration_ms, Some(123));
|
||||
assert_eq!(audit.repo_sync_error.as_deref(), Some("none"));
|
||||
assert_eq!(audit.repo_terminal_state, "fresh");
|
||||
|
||||
let crl = audit
|
||||
.objects
|
||||
@@ -4484,7 +4678,6 @@ authorityKeyIdentifier = keyid:always
|
||||
let _ = now;
|
||||
}
|
||||
|
||||
|
||||
#[test]
|
||||
fn discover_children_with_router_certificate_records_ok_audit_and_no_child() {
|
||||
let g = generate_router_cert_with_variant("ec-p256", true);
|
||||
@@ -4505,7 +4698,9 @@ authorityKeyIdentifier = keyid:always
|
||||
tal_id: "test-tal".to_string(),
|
||||
parent_manifest_rsync_uri: None,
|
||||
ca_certificate_der: g.issuer_ca_der.clone(),
|
||||
ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()),
|
||||
ca_certificate_rsync_uri: Some(
|
||||
"rsync://example.test/repo/issuer/issuer.cer".to_string(),
|
||||
),
|
||||
effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(),
|
||||
effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(),
|
||||
rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(),
|
||||
@@ -4524,11 +4719,13 @@ authorityKeyIdentifier = keyid:always
|
||||
assert!(out.children.is_empty());
|
||||
assert_eq!(out.audits.len(), 1);
|
||||
assert!(matches!(out.audits[0].result, AuditObjectResult::Ok));
|
||||
assert!(out.audits[0]
|
||||
.detail
|
||||
.as_deref()
|
||||
.unwrap_or("")
|
||||
.contains("validated BGPsec router certificate"));
|
||||
assert!(
|
||||
out.audits[0]
|
||||
.detail
|
||||
.as_deref()
|
||||
.unwrap_or("")
|
||||
.contains("validated BGPsec router certificate")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4551,7 +4748,9 @@ authorityKeyIdentifier = keyid:always
|
||||
tal_id: "test-tal".to_string(),
|
||||
parent_manifest_rsync_uri: None,
|
||||
ca_certificate_der: g.issuer_ca_der.clone(),
|
||||
ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()),
|
||||
ca_certificate_rsync_uri: Some(
|
||||
"rsync://example.test/repo/issuer/issuer.cer".to_string(),
|
||||
),
|
||||
effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(),
|
||||
effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(),
|
||||
rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(),
|
||||
@@ -4570,11 +4769,13 @@ authorityKeyIdentifier = keyid:always
|
||||
assert!(out.children.is_empty());
|
||||
assert_eq!(out.audits.len(), 1);
|
||||
assert!(matches!(out.audits[0].result, AuditObjectResult::Skipped));
|
||||
assert!(out.audits[0]
|
||||
.detail
|
||||
.as_deref()
|
||||
.unwrap_or("")
|
||||
.contains("not a CA resource certificate or BGPsec router certificate"));
|
||||
assert!(
|
||||
out.audits[0]
|
||||
.detail
|
||||
.as_deref()
|
||||
.unwrap_or("")
|
||||
.contains("not a CA resource certificate or BGPsec router certificate")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4597,7 +4798,9 @@ authorityKeyIdentifier = keyid:always
|
||||
tal_id: "test-tal".to_string(),
|
||||
parent_manifest_rsync_uri: None,
|
||||
ca_certificate_der: g.issuer_ca_der.clone(),
|
||||
ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()),
|
||||
ca_certificate_rsync_uri: Some(
|
||||
"rsync://example.test/repo/issuer/issuer.cer".to_string(),
|
||||
),
|
||||
effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(),
|
||||
effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(),
|
||||
rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(),
|
||||
@@ -4616,11 +4819,13 @@ authorityKeyIdentifier = keyid:always
|
||||
assert!(out.children.is_empty());
|
||||
assert_eq!(out.audits.len(), 1);
|
||||
assert!(matches!(out.audits[0].result, AuditObjectResult::Error));
|
||||
assert!(out.audits[0]
|
||||
.detail
|
||||
.as_deref()
|
||||
.unwrap_or("")
|
||||
.contains("router certificate validation failed"));
|
||||
assert!(
|
||||
out.audits[0]
|
||||
.detail
|
||||
.as_deref()
|
||||
.unwrap_or("")
|
||||
.contains("router certificate validation failed")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -5210,9 +5415,10 @@ authorityKeyIdentifier = keyid:always
|
||||
let audit = build_publication_point_audit_from_vcir(
|
||||
&ca,
|
||||
PublicationPointSource::VcirCurrentInstance,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some("rsync"),
|
||||
Some("rrdp_failed_rsync_failed"),
|
||||
Some(456),
|
||||
Some("rsync failed"),
|
||||
Some(&vcir),
|
||||
None,
|
||||
&runner_warnings,
|
||||
@@ -5221,6 +5427,14 @@ authorityKeyIdentifier = keyid:always
|
||||
);
|
||||
|
||||
assert_eq!(audit.source, "vcir_current_instance");
|
||||
assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync"));
|
||||
assert_eq!(
|
||||
audit.repo_sync_phase.as_deref(),
|
||||
Some("rrdp_failed_rsync_failed")
|
||||
);
|
||||
assert_eq!(audit.repo_sync_duration_ms, Some(456));
|
||||
assert_eq!(audit.repo_sync_error.as_deref(), Some("rsync failed"));
|
||||
assert_eq!(audit.repo_terminal_state, "fallback_current_instance");
|
||||
assert_eq!(audit.objects[0].rsync_uri, vcir.current_manifest_rsync_uri);
|
||||
assert_eq!(audit.objects[0].kind, AuditObjectKind::Manifest);
|
||||
assert_eq!(
|
||||
@@ -5270,16 +5484,17 @@ authorityKeyIdentifier = keyid:always
|
||||
let audit = build_publication_point_audit_from_vcir(
|
||||
&ca,
|
||||
PublicationPointSource::FailedFetchNoCache,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some("rsync"),
|
||||
Some("rsync_only_failed"),
|
||||
Some(789),
|
||||
Some("load from network failed, fallback to cache"),
|
||||
None,
|
||||
None,
|
||||
&[Warning::new("runner warning")],
|
||||
&crate::validation::objects::ObjectsOutput {
|
||||
vrps: Vec::new(),
|
||||
aspas: Vec::new(),
|
||||
router_keys: Vec::new(),
|
||||
router_keys: Vec::new(),
|
||||
local_outputs_cache: Vec::new(),
|
||||
warnings: vec![Warning::new("object warning")],
|
||||
stats: crate::validation::objects::ObjectsStats::default(),
|
||||
@@ -5289,6 +5504,14 @@ authorityKeyIdentifier = keyid:always
|
||||
);
|
||||
|
||||
assert_eq!(audit.source, "failed_fetch_no_cache");
|
||||
assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync"));
|
||||
assert_eq!(audit.repo_sync_phase.as_deref(), Some("rsync_only_failed"));
|
||||
assert_eq!(audit.repo_sync_duration_ms, Some(789));
|
||||
assert_eq!(
|
||||
audit.repo_sync_error.as_deref(),
|
||||
Some("load from network failed, fallback to cache")
|
||||
);
|
||||
assert_eq!(audit.repo_terminal_state, "failed_no_cache");
|
||||
assert!(audit.this_update_rfc3339_utc.is_empty());
|
||||
assert!(audit.next_update_rfc3339_utc.is_empty());
|
||||
assert!(audit.verified_at_rfc3339_utc.is_empty());
|
||||
|
||||
Reference in New Issue
Block a user