NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260410 完成五个rir 基于cir的三方回放，raw by hash 独立db，发现内存占用大，连续大rir 录制发生oom

Browse Source
This commit is contained in:
yuyr
2026-04-11 11:24:32 +08:00
parent e083fe4daa
commit 77fc2f1a41
96 changed files with 5159 additions and 1720 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/cir/decode.rs
View File

@@ -14,7 +14,10 @@ pub enum CirDecodeError {
UnexpectedVersion { expected: u32, actual: u32 },
#[error("unexpected digest algorithm OID: expected {expected}, got {actual}")]
UnexpectedDigestAlgorithm { expected: &'static str, actual: String },
UnexpectedDigestAlgorithm {
expected: &'static str,
actual: String,
},
#[error("CIR model validation failed after decode: {0}")]
Validate(String),
@@ -85,12 +88,17 @@ fn decode_object(der: &[u8]) -> Result<CirObject, CirDecodeError> {
let mut top = DerReader::new(der);
let mut seq = top.take_sequence().map_err(CirDecodeError::Parse)?;
if !top.is_empty() {
return Err(CirDecodeError::Parse("trailing bytes after CirObject".into()));
return Err(CirDecodeError::Parse(
"trailing bytes after CirObject".into(),
));
}
let rsync_uri = std::str::from_utf8(seq.take_tag(0x16).map_err(CirDecodeError::Parse)?)
.map_err(|e| CirDecodeError::Parse(e.to_string()))?
.to_string();
let sha256 = seq.take_octet_string().map_err(CirDecodeError::Parse)?.to_vec();
let sha256 = seq
.take_octet_string()
.map_err(CirDecodeError::Parse)?
.to_vec();
if !seq.is_empty() {
return Err(CirDecodeError::Parse("trailing fields in CirObject".into()));
}
@@ -106,7 +114,10 @@ fn decode_tal(der: &[u8]) -> Result<CirTal, CirDecodeError> {
let tal_uri = std::str::from_utf8(seq.take_tag(0x16).map_err(CirDecodeError::Parse)?)
.map_err(|e| CirDecodeError::Parse(e.to_string()))?
.to_string();
let tal_bytes = seq.take_octet_string().map_err(CirDecodeError::Parse)?.to_vec();
let tal_bytes = seq
.take_octet_string()
.map_err(CirDecodeError::Parse)?
.to_vec();
if !seq.is_empty() {
return Err(CirDecodeError::Parse("trailing fields in CirTal".into()));
}
@@ -150,12 +161,10 @@ fn parse_generalized_time(bytes: &[u8]) -> Result<time::OffsetDateTime, CirDecod
let hour = parse(8..10)? as u8;
let minute = parse(10..12)? as u8;
let second = parse(12..14)? as u8;
let month = time::Month::try_from(month)
.map_err(|e| CirDecodeError::Parse(e.to_string()))?;
let month = time::Month::try_from(month).map_err(|e| CirDecodeError::Parse(e.to_string()))?;
let date = time::Date::from_calendar_date(year, month, day)
.map_err(|e| CirDecodeError::Parse(e.to_string()))?;
let timev = time::Time::from_hms(hour, minute, second)
.map_err(|e| CirDecodeError::Parse(e.to_string()))?;
Ok(time::PrimitiveDateTime::new(date, timev).assume_utc())
}

10
src/cir/encode.rs
View File

@@ -9,9 +9,7 @@ pub enum CirEncodeError {
Validate(String),
}
pub fn encode_cir(
cir: &CanonicalInputRepresentation,
) -> Result<Vec<u8>, CirEncodeError> {
pub fn encode_cir(cir: &CanonicalInputRepresentation) -> Result<Vec<u8>, CirEncodeError> {
cir.validate().map_err(CirEncodeError::Validate)?;
Ok(encode_sequence(&[
encode_integer_u32(cir.version),
@@ -134,7 +132,10 @@ fn encode_len_into(len: usize, out: &mut Vec<u8>) {
return;
}
let bytes = len.to_be_bytes();
let first_non_zero = bytes.iter().position(|&b| b != 0).unwrap_or(bytes.len() - 1);
let first_non_zero = bytes
.iter()
.position(|&b| b != 0)
.unwrap_or(bytes.len() - 1);
let len_bytes = &bytes[first_non_zero..];
out.push(0x80 | (len_bytes.len() as u8));
out.extend_from_slice(len_bytes);
@@ -144,4 +145,3 @@ fn encode_len_into(len: usize, out: &mut Vec<u8>) {
const _: () = {
let _ = CIR_VERSION_V1;
};

213
src/cir/export.rs
View File

@@ -3,13 +3,14 @@ use std::collections::BTreeSet;
use std::path::Path;
use crate::audit::{AuditObjectResult, PublicationPointAudit};
use crate::blob_store::RawObjectStore;
use crate::cir::encode::{CirEncodeError, encode_cir};
use crate::cir::model::{
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal,
};
use crate::cir::static_pool::{
CirStaticPoolError, CirStaticPoolExportSummary, write_bytes_to_static_pool,
export_hashes_from_store,
CirStaticPoolError, CirStaticPoolExportSummary, export_hashes_from_store,
write_bytes_to_static_pool,
};
use crate::data_model::ta::TrustAnchor;
use crate::storage::{RepositoryViewState, RocksStore};
@@ -46,11 +47,24 @@ pub enum CirExportError {
Write(String, String),
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum CirObjectPoolExportSummary {
Static(CirStaticPoolExportSummary),
RawStore(CirRawStoreExportSummary),
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CirRawStoreExportSummary {
pub unique_hashes: usize,
pub written_entries: usize,
pub reused_entries: usize,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CirExportSummary {
pub object_count: usize,
pub tal_count: usize,
pub static_pool: CirStaticPoolExportSummary,
pub object_pool: CirObjectPoolExportSummary,
pub timing: CirExportTiming,
}
@@ -132,7 +146,10 @@ pub fn build_cir_from_run(
Ok(cir)
}
pub fn write_cir_file(path: &Path, cir: &CanonicalInputRepresentation) -> Result<(), CirExportError> {
pub fn write_cir_file(
path: &Path,
cir: &CanonicalInputRepresentation,
) -> Result<(), CirExportError> {
let der = encode_cir(cir)?;
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)
@@ -175,6 +192,58 @@ pub fn export_cir_static_pool(
Ok(summary)
}
pub fn export_cir_raw_store(
store: &RocksStore,
raw_store_path: &Path,
cir: &CanonicalInputRepresentation,
trust_anchor: &TrustAnchor,
) -> Result<CirRawStoreExportSummary, CirExportError> {
let ta_hash = ta_sha256_hex(&trust_anchor.ta_certificate.raw_der);
let unique: BTreeSet<String> = cir
.objects
.iter()
.map(|item| hex::encode(&item.sha256))
.collect();
let mut written_entries = 0usize;
let mut reused_entries = 0usize;
for sha256_hex in &unique {
if store
.get_raw_entry(sha256_hex)
.map_err(|e| CirExportError::Write(raw_store_path.display().to_string(), e.to_string()))?
.is_some()
{
reused_entries += 1;
continue;
}
if sha256_hex == &ta_hash {
let mut entry =
crate::storage::RawByHashEntry::from_bytes(ta_hash.clone(), trust_anchor.ta_certificate.raw_der.clone());
entry.object_type = Some("cer".to_string());
for object in &cir.objects {
if hex::encode(&object.sha256) == ta_hash {
entry.origin_uris.push(object.rsync_uri.clone());
}
}
store
.put_raw_by_hash_entry(&entry)
.map_err(|e| CirExportError::Write(raw_store_path.display().to_string(), e.to_string()))?;
written_entries += 1;
continue;
}
return Err(CirExportError::Write(
raw_store_path.display().to_string(),
format!("raw store missing object for sha256={sha256_hex}"),
));
}
Ok(CirRawStoreExportSummary {
unique_hashes: unique.len(),
written_entries,
reused_entries,
})
}
pub fn export_cir_from_run(
store: &RocksStore,
trust_anchor: &TrustAnchor,
@@ -182,9 +251,19 @@ pub fn export_cir_from_run(
validation_time: time::OffsetDateTime,
publication_points: &[PublicationPointAudit],
cir_out: &Path,
static_root: &Path,
static_root: Option<&Path>,
raw_store_path: Option<&Path>,
capture_date_utc: time::Date,
) -> Result<CirExportSummary, CirExportError> {
let backend_count = static_root.is_some() as u8 + raw_store_path.is_some() as u8;
match backend_count {
1 => {}
_ => {
return Err(CirExportError::Validate(
"must specify exactly one CIR object pool backend".to_string(),
));
}
}
let total_started = std::time::Instant::now();
let started = std::time::Instant::now();
@@ -198,7 +277,22 @@ pub fn export_cir_from_run(
let build_cir_ms = started.elapsed().as_millis() as u64;
let started = std::time::Instant::now();
let static_pool = export_cir_static_pool(store, static_root, capture_date_utc, &cir, trust_anchor)?;
let object_pool = if let Some(static_root) = static_root {
CirObjectPoolExportSummary::Static(export_cir_static_pool(
store,
static_root,
capture_date_utc,
&cir,
trust_anchor,
)?)
} else {
CirObjectPoolExportSummary::RawStore(export_cir_raw_store(
store,
raw_store_path.expect("validated"),
&cir,
trust_anchor,
)?)
};
let static_pool_ms = started.elapsed().as_millis() as u64;
let started = std::time::Instant::now();
@@ -208,7 +302,7 @@ pub fn export_cir_from_run(
Ok(CirExportSummary {
object_count: cir.objects.len(),
tal_count: cir.tals.len(),
static_pool,
object_pool,
timing: CirExportTiming {
build_cir_ms,
static_pool_ms,
@@ -246,7 +340,8 @@ mod tests {
fn sample_trust_anchor() -> TrustAnchor {
let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let tal_bytes = std::fs::read(base.join("tests/fixtures/tal/apnic-rfc7730-https.tal")).unwrap();
let tal_bytes =
std::fs::read(base.join("tests/fixtures/tal/apnic-rfc7730-https.tal")).unwrap();
let ta_der = std::fs::read(base.join("tests/fixtures/ta/apnic-ta.cer")).unwrap();
let tal = Tal::decode_bytes(&tal_bytes).unwrap();
TrustAnchor::bind_der(tal, &ta_der, None).unwrap()
@@ -264,7 +359,8 @@ mod tests {
let bytes = b"object-a".to_vec();
let hash = sha256_hex(&bytes);
let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone());
raw.origin_uris.push("rsync://example.test/repo/a.cer".into());
raw.origin_uris
.push("rsync://example.test/repo/a.cer".into());
store.put_raw_by_hash_entry(&raw).unwrap();
store
.put_repository_view_entry(&RepositoryViewEntry {
@@ -284,18 +380,20 @@ mod tests {
sample_time(),
&[],
)
.expect("build cir");
.expect("build cir");
assert_eq!(cir.version, CIR_VERSION_V1);
assert_eq!(cir.tals.len(), 1);
assert_eq!(cir.tals[0].tal_uri, "https://example.test/root.tal");
assert!(cir
.objects
.iter()
.any(|item| item.rsync_uri == "rsync://example.test/repo/a.cer"));
assert!(cir
.objects
.iter()
.any(|item| item.rsync_uri.contains("apnic-rpki-root-iana-origin.cer")));
assert!(
cir.objects
.iter()
.any(|item| item.rsync_uri == "rsync://example.test/repo/a.cer")
);
assert!(
cir.objects
.iter()
.any(|item| item.rsync_uri.contains("apnic-rpki-root-iana-origin.cer"))
);
}
#[test]
@@ -309,7 +407,8 @@ mod tests {
let bytes = b"object-b".to_vec();
let hash = sha256_hex(&bytes);
let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone());
raw.origin_uris.push("rsync://example.test/repo/b.roa".into());
raw.origin_uris
.push("rsync://example.test/repo/b.roa".into());
store.put_raw_by_hash_entry(&raw).unwrap();
store
.put_repository_view_entry(&RepositoryViewEntry {
@@ -330,12 +429,17 @@ mod tests {
sample_time(),
&[],
&cir_path,
&static_root,
Some(&static_root),
None,
sample_date(),
)
.expect("export cir");
assert_eq!(summary.tal_count, 1);
assert!(summary.object_count >= 2);
match summary.object_pool {
CirObjectPoolExportSummary::Static(_) => {}
other => panic!("unexpected backend: {other:?}"),
}
let der = std::fs::read(&cir_path).unwrap();
let cir = decode_cir(&der).unwrap();
@@ -345,6 +449,51 @@ mod tests {
assert_eq!(std::fs::read(object_path).unwrap(), bytes);
}
#[test]
fn export_cir_from_run_uses_raw_store_backend_without_pool_export() {
let td = tempfile::tempdir().unwrap();
let store_dir = td.path().join("db");
let raw_store = td.path().join("raw-store.db");
let out_dir = td.path().join("out");
let store = RocksStore::open_with_external_raw_store(&store_dir, &raw_store).unwrap();
let bytes = b"object-d".to_vec();
let hash = sha256_hex(&bytes);
let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone());
raw.origin_uris
.push("rsync://example.test/repo/d.roa".into());
store.put_raw_by_hash_entry(&raw).unwrap();
store
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/d.roa".to_string(),
current_hash: Some(hash.clone()),
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
})
.unwrap();
let ta = sample_trust_anchor();
let cir_path = out_dir.join("example.cir");
let summary = export_cir_from_run(
&store,
&ta,
"https://example.test/root.tal",
sample_time(),
&[],
&cir_path,
None,
Some(&raw_store),
sample_date(),
)
.expect("export cir to raw store");
match summary.object_pool {
CirObjectPoolExportSummary::RawStore(ref s) => assert!(s.unique_hashes >= 2),
other => panic!("unexpected backend: {other:?}"),
}
assert!(raw_store.exists());
}
#[test]
fn build_cir_from_run_includes_vcir_current_instance_objects_from_audit() {
let td = tempfile::tempdir().unwrap();
@@ -357,14 +506,16 @@ mod tests {
};
pp.objects.push(crate::audit::ObjectAuditEntry {
rsync_uri: "rsync://example.test/repo/fallback.mft".to_string(),
sha256_hex: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string(),
sha256_hex: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
.to_string(),
kind: crate::audit::AuditObjectKind::Manifest,
result: crate::audit::AuditObjectResult::Ok,
detail: None,
});
pp.objects.push(crate::audit::ObjectAuditEntry {
rsync_uri: "rsync://example.test/repo/fallback.roa".to_string(),
sha256_hex: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(),
sha256_hex: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
.to_string(),
kind: crate::audit::AuditObjectKind::Roa,
result: crate::audit::AuditObjectResult::Ok,
detail: None,
@@ -379,13 +530,15 @@ mod tests {
)
.expect("build cir");
assert!(cir
.objects
.iter()
.any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.mft"));
assert!(cir
.objects
.iter()
.any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.roa"));
assert!(
cir.objects
.iter()
.any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.mft")
);
assert!(
cir.objects
.iter()
.any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.roa")
);
}
}

370
src/cir/materialize.rs
View File

@@ -1,6 +1,7 @@
use std::fs;
use std::path::{Path, PathBuf};
use crate::blob_store::{ExternalRawStoreDb, RawObjectStore};
use crate::cir::model::CanonicalInputRepresentation;
#[derive(Debug, thiserror::Error)]
@@ -27,13 +28,30 @@ pub enum CirMaterializeError {
MissingStaticObject { sha256_hex: String },
#[error("link target failed: {src} -> {dst}: {detail}")]
Link { src: String, dst: String, detail: String },
Link {
src: String,
dst: String,
detail: String,
},
#[error("copy target failed: {src} -> {dst}: {detail}")]
Copy { src: String, dst: String, detail: String },
Copy {
src: String,
dst: String,
detail: String,
},
#[error("mirror tree mismatch after materialize: {0}")]
TreeMismatch(String),
#[error("open raw store failed: {path}: {detail}")]
OpenRawStore { path: String, detail: String },
#[error("raw object not found for sha256={sha256_hex}")]
MissingRawStoreObject { sha256_hex: String },
#[error("read raw store failed for sha256={sha256_hex}: {detail}")]
ReadRawStore { sha256_hex: String, detail: String },
}
#[derive(Clone, Debug, PartialEq, Eq)]
@@ -49,8 +67,7 @@ pub fn materialize_cir(
mirror_root: &Path,
clean_rebuild: bool,
) -> Result<CirMaterializeSummary, CirMaterializeError> {
cir.validate()
.map_err(CirMaterializeError::TreeMismatch)?;
cir.validate().map_err(CirMaterializeError::TreeMismatch)?;
if clean_rebuild && mirror_root.exists() {
fs::remove_dir_all(mirror_root).map_err(|e| CirMaterializeError::RemoveMirrorRoot {
@@ -120,6 +137,89 @@ pub fn materialize_cir(
})
}
pub fn materialize_cir_from_raw_store(
cir: &CanonicalInputRepresentation,
raw_store_db: &Path,
mirror_root: &Path,
clean_rebuild: bool,
) -> Result<CirMaterializeSummary, CirMaterializeError> {
cir.validate().map_err(CirMaterializeError::TreeMismatch)?;
if clean_rebuild && mirror_root.exists() {
fs::remove_dir_all(mirror_root).map_err(|e| CirMaterializeError::RemoveMirrorRoot {
path: mirror_root.display().to_string(),
detail: e.to_string(),
})?;
}
fs::create_dir_all(mirror_root).map_err(|e| CirMaterializeError::CreateMirrorRoot {
path: mirror_root.display().to_string(),
detail: e.to_string(),
})?;
let raw_store =
ExternalRawStoreDb::open(raw_store_db).map_err(|e| CirMaterializeError::OpenRawStore {
path: raw_store_db.display().to_string(),
detail: e.to_string(),
})?;
let mut copied_files = 0usize;
for object in &cir.objects {
let sha256_hex = hex::encode(&object.sha256);
let entry = raw_store
.get_raw_entry(&sha256_hex)
.map_err(|e| CirMaterializeError::ReadRawStore {
sha256_hex: sha256_hex.clone(),
detail: e.to_string(),
})?
.ok_or_else(|| CirMaterializeError::MissingRawStoreObject {
sha256_hex: sha256_hex.clone(),
})?;
let relative = mirror_relative_path_for_rsync_uri(&object.rsync_uri)?;
let target = mirror_root.join(&relative);
if let Some(parent) = target.parent() {
fs::create_dir_all(parent).map_err(|e| CirMaterializeError::CreateParent {
path: parent.display().to_string(),
detail: e.to_string(),
})?;
}
if target.exists() {
fs::remove_file(&target).map_err(|e| CirMaterializeError::RemoveExistingTarget {
path: target.display().to_string(),
detail: e.to_string(),
})?;
}
fs::write(&target, &entry.bytes).map_err(|e| CirMaterializeError::Copy {
src: raw_store_db.display().to_string(),
dst: target.display().to_string(),
detail: e.to_string(),
})?;
copied_files += 1;
}
let actual = collect_materialized_uris(mirror_root)?;
let expected = cir
.objects
.iter()
.map(|item| item.rsync_uri.clone())
.collect::<std::collections::BTreeSet<_>>();
if actual != expected {
return Err(CirMaterializeError::TreeMismatch(format!(
"expected {} files, got {} files",
expected.len(),
actual.len()
)));
}
Ok(CirMaterializeSummary {
object_count: cir.objects.len(),
linked_files: 0,
copied_files,
})
}
pub fn mirror_relative_path_for_rsync_uri(rsync_uri: &str) -> Result<PathBuf, CirMaterializeError> {
let url = url::Url::parse(rsync_uri)
.map_err(|_| CirMaterializeError::InvalidRsyncUri(rsync_uri.to_string()))?;
@@ -160,8 +260,8 @@ pub fn resolve_static_pool_file(
let prefix1 = &sha256_hex[0..2];
let prefix2 = &sha256_hex[2..4];
let entries = fs::read_dir(static_root)
.map_err(|_| CirMaterializeError::MissingStaticObject {
let entries =
fs::read_dir(static_root).map_err(|_| CirMaterializeError::MissingStaticObject {
sha256_hex: sha256_hex.to_string(),
})?;
let mut dates = entries
@@ -215,10 +315,15 @@ fn collect_materialized_uris(
#[cfg(test)]
mod tests {
use super::{CirMaterializeError, materialize_cir, mirror_relative_path_for_rsync_uri, resolve_static_pool_file};
use super::{
CirMaterializeError, materialize_cir, materialize_cir_from_raw_store,
mirror_relative_path_for_rsync_uri, resolve_static_pool_file,
};
use crate::blob_store::ExternalRawStoreDb;
use crate::cir::model::{
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal,
};
use sha2::Digest;
use std::path::{Path, PathBuf};
fn sample_time() -> time::OffsetDateTime {
@@ -257,21 +362,45 @@ mod tests {
}
}
fn cir_with_real_hashes(a: &[u8], b: &[u8]) -> CanonicalInputRepresentation {
CanonicalInputRepresentation {
version: CIR_VERSION_V1,
hash_alg: CirHashAlgorithm::Sha256,
validation_time: sample_time(),
objects: vec![
CirObject {
rsync_uri: "rsync://example.net/repo/a.cer".to_string(),
sha256: sha2::Sha256::digest(a).to_vec(),
},
CirObject {
rsync_uri: "rsync://example.net/repo/nested/b.roa".to_string(),
sha256: sha2::Sha256::digest(b).to_vec(),
},
],
tals: vec![CirTal {
tal_uri: "https://tal.example.net/root.tal".to_string(),
tal_bytes: b"x".to_vec(),
}],
}
}
#[test]
fn mirror_relative_path_for_rsync_uri_maps_host_and_path() {
let path =
mirror_relative_path_for_rsync_uri("rsync://example.net/repo/nested/b.roa").unwrap();
assert_eq!(path, PathBuf::from("example.net").join("repo").join("nested").join("b.roa"));
assert_eq!(
path,
PathBuf::from("example.net")
.join("repo")
.join("nested")
.join("b.roa")
);
}
#[test]
fn resolve_static_pool_file_finds_hash_across_dates() {
let td = tempfile::tempdir().unwrap();
let path = td
.path()
.join("20260407")
.join("11")
.join("11");
let path = td.path().join("20260407").join("11").join("11");
std::fs::create_dir_all(&path).unwrap();
let file = path.join("1111111111111111111111111111111111111111111111111111111111111111");
std::fs::write(&file, b"x").unwrap();
@@ -289,14 +418,20 @@ mod tests {
let td = tempfile::tempdir().unwrap();
let err = resolve_static_pool_file(td.path(), "not-a-hash")
.expect_err("invalid hash should fail");
assert!(matches!(err, CirMaterializeError::MissingStaticObject { .. }));
assert!(matches!(
err,
CirMaterializeError::MissingStaticObject { .. }
));
let err = resolve_static_pool_file(
td.path(),
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
)
.expect_err("missing hash should fail");
assert!(matches!(err, CirMaterializeError::MissingStaticObject { .. }));
assert!(matches!(
err,
CirMaterializeError::MissingStaticObject { .. }
));
}
#[test]
@@ -333,7 +468,10 @@ mod tests {
let summary = materialize_cir(&sample_cir(), &static_root, &mirror_root, true).unwrap();
assert_eq!(summary.object_count, 2);
assert_eq!(std::fs::read(mirror_root.join("example.net/repo/a.cer")).unwrap(), b"a");
assert_eq!(
std::fs::read(mirror_root.join("example.net/repo/a.cer")).unwrap(),
b"a"
);
assert_eq!(
std::fs::read(mirror_root.join("example.net/repo/nested/b.roa")).unwrap(),
b"b"
@@ -344,14 +482,12 @@ mod tests {
#[test]
fn materialize_fails_when_static_object_missing() {
let td = tempfile::tempdir().unwrap();
let err = materialize_cir(
&sample_cir(),
td.path(),
&td.path().join("mirror"),
true,
)
.expect_err("missing static object must fail");
assert!(matches!(err, CirMaterializeError::MissingStaticObject { .. }));
let err = materialize_cir(&sample_cir(), td.path(), &td.path().join("mirror"), true)
.expect_err("missing static object must fail");
assert!(matches!(
err,
CirMaterializeError::MissingStaticObject { .. }
));
}
#[test]
@@ -380,6 +516,192 @@ mod tests {
assert!(matches!(err, CirMaterializeError::TreeMismatch(_)));
}
#[test]
fn materialize_from_raw_store_creates_expected_tree() {
let td = tempfile::tempdir().unwrap();
let raw_store_path = td.path().join("raw-store.db");
let mirror_root = td.path().join("mirror");
let a = b"a".to_vec();
let b = b"b".to_vec();
let cir = CanonicalInputRepresentation {
version: CIR_VERSION_V1,
hash_alg: CirHashAlgorithm::Sha256,
validation_time: sample_time(),
objects: vec![
CirObject {
rsync_uri: "rsync://example.net/repo/a.cer".to_string(),
sha256: sha2::Sha256::digest(&a).to_vec(),
},
CirObject {
rsync_uri: "rsync://example.net/repo/nested/b.roa".to_string(),
sha256: sha2::Sha256::digest(&b).to_vec(),
},
],
tals: vec![CirTal {
tal_uri: "https://tal.example.net/root.tal".to_string(),
tal_bytes: b"x".to_vec(),
}],
};
{
let raw_store = ExternalRawStoreDb::open(&raw_store_path).unwrap();
let mut entry_a =
crate::storage::RawByHashEntry::from_bytes(hex::encode(&cir.objects[0].sha256), a);
entry_a.origin_uris.push(cir.objects[0].rsync_uri.clone());
raw_store.put_raw_entry(&entry_a).unwrap();
let mut entry_b =
crate::storage::RawByHashEntry::from_bytes(hex::encode(&cir.objects[1].sha256), b);
entry_b.origin_uris.push(cir.objects[1].rsync_uri.clone());
raw_store.put_raw_entry(&entry_b).unwrap();
}
let summary =
materialize_cir_from_raw_store(&cir, &raw_store_path, &mirror_root, true).unwrap();
assert_eq!(summary.object_count, 2);
assert_eq!(summary.linked_files, 0);
assert_eq!(summary.copied_files, 2);
assert_eq!(
std::fs::read(mirror_root.join("example.net/repo/a.cer")).unwrap(),
b"a"
);
assert_eq!(
std::fs::read(mirror_root.join("example.net/repo/nested/b.roa")).unwrap(),
b"b"
);
}
#[test]
fn materialize_from_raw_store_fails_when_object_missing() {
let td = tempfile::tempdir().unwrap();
let raw_store_path = td.path().join("raw-store.db");
let mirror_root = td.path().join("mirror");
let cir = cir_with_real_hashes(b"a", b"b");
{
let raw_store = ExternalRawStoreDb::open(&raw_store_path).unwrap();
let only = crate::storage::RawByHashEntry::from_bytes(
hex::encode(&cir.objects[0].sha256),
b"a".to_vec(),
);
raw_store.put_raw_entry(&only).unwrap();
}
let err = materialize_cir_from_raw_store(&cir, &raw_store_path, &mirror_root, true)
.expect_err("missing second object should fail");
assert!(matches!(
err,
CirMaterializeError::MissingRawStoreObject { .. }
));
}
#[test]
fn materialize_from_raw_store_detects_stale_tree_when_not_clean_rebuild() {
let td = tempfile::tempdir().unwrap();
let raw_store_path = td.path().join("raw-store.db");
let mirror_root = td.path().join("mirror");
let cir = cir_with_real_hashes(b"a", b"b");
{
let raw_store = ExternalRawStoreDb::open(&raw_store_path).unwrap();
let mut entry_a = crate::storage::RawByHashEntry::from_bytes(
hex::encode(&cir.objects[0].sha256),
b"a".to_vec(),
);
entry_a.origin_uris.push(cir.objects[0].rsync_uri.clone());
raw_store.put_raw_entry(&entry_a).unwrap();
let mut entry_b = crate::storage::RawByHashEntry::from_bytes(
hex::encode(&cir.objects[1].sha256),
b"b".to_vec(),
);
entry_b.origin_uris.push(cir.objects[1].rsync_uri.clone());
raw_store.put_raw_entry(&entry_b).unwrap();
}
std::fs::create_dir_all(mirror_root.join("extra")).unwrap();
std::fs::write(mirror_root.join("extra/stale.txt"), b"stale").unwrap();
let err =
materialize_cir_from_raw_store(&cir, &raw_store_path, &mirror_root, false)
.expect_err("stale file should fail exact tree check");
assert!(matches!(err, CirMaterializeError::TreeMismatch(_)));
}
#[test]
fn materialize_from_raw_store_overwrites_existing_targets() {
let td = tempfile::tempdir().unwrap();
let raw_store_path = td.path().join("raw-store.db");
let mirror_root = td.path().join("mirror");
let a = b"new-a".to_vec();
let b = b"new-b".to_vec();
let cir = cir_with_real_hashes(&a, &b);
{
let raw_store = ExternalRawStoreDb::open(&raw_store_path).unwrap();
let mut entry_a = crate::storage::RawByHashEntry::from_bytes(
hex::encode(&cir.objects[0].sha256),
a.clone(),
);
entry_a.origin_uris.push(cir.objects[0].rsync_uri.clone());
raw_store.put_raw_entry(&entry_a).unwrap();
let mut entry_b = crate::storage::RawByHashEntry::from_bytes(
hex::encode(&cir.objects[1].sha256),
b.clone(),
);
entry_b.origin_uris.push(cir.objects[1].rsync_uri.clone());
raw_store.put_raw_entry(&entry_b).unwrap();
}
let target = mirror_root.join("example.net/repo/a.cer");
std::fs::create_dir_all(target.parent().unwrap()).unwrap();
std::fs::write(&target, b"old").unwrap();
let summary =
materialize_cir_from_raw_store(&cir, &raw_store_path, &mirror_root, false).unwrap();
assert_eq!(summary.copied_files, 2);
assert_eq!(std::fs::read(&target).unwrap(), a);
}
#[test]
fn materialize_from_raw_store_reports_codec_errors() {
let td = tempfile::tempdir().unwrap();
let raw_store_path = td.path().join("raw-store.db");
let mirror_root = td.path().join("mirror");
{
let _raw_store = ExternalRawStoreDb::open(&raw_store_path).unwrap();
}
{
let db = rocksdb::DB::open_default(&raw_store_path).unwrap();
db.put(
b"rawbyhash:1111111111111111111111111111111111111111111111111111111111111111",
b"bad-cbor",
)
.unwrap();
}
let err = materialize_cir_from_raw_store(
&CanonicalInputRepresentation {
version: CIR_VERSION_V1,
hash_alg: CirHashAlgorithm::Sha256,
validation_time: sample_time(),
objects: vec![CirObject {
rsync_uri: "rsync://example.net/repo/a.cer".to_string(),
sha256: hex::decode(
"1111111111111111111111111111111111111111111111111111111111111111",
)
.unwrap(),
}],
tals: vec![CirTal {
tal_uri: "https://tal.example.net/root.tal".to_string(),
tal_bytes: b"x".to_vec(),
}],
},
&raw_store_path,
&mirror_root,
true,
)
.expect_err("corrupt raw-store object should fail");
assert!(matches!(
err,
CirMaterializeError::ReadRawStore { .. } | CirMaterializeError::MissingRawStoreObject { .. }
));
}
fn write_static(root: &Path, date: &str, hash: &str, bytes: &[u8]) {
let path = root.join(date).join(&hash[0..2]).join(&hash[2..4]);
std::fs::create_dir_all(&path).unwrap();

45
src/cir/mod.rs
View File

@@ -1,26 +1,29 @@
pub mod decode;
pub mod encode;
#[cfg(feature = "full")]
pub mod export;
pub mod materialize;
pub mod model;
pub mod sequence;
#[cfg(feature = "full")]
pub mod export;
#[cfg(feature = "full")]
pub mod static_pool;
pub use decode::{CirDecodeError, decode_cir};
pub use encode::{CirEncodeError, encode_cir};
#[cfg(feature = "full")]
pub use export::{
CirExportError, CirExportSummary, build_cir_from_run, export_cir_from_run, write_cir_file,
};
pub use materialize::{
CirMaterializeError, CirMaterializeSummary, materialize_cir, mirror_relative_path_for_rsync_uri,
resolve_static_pool_file,
CirMaterializeError, CirMaterializeSummary, materialize_cir,
materialize_cir_from_raw_store,
mirror_relative_path_for_rsync_uri, resolve_static_pool_file,
};
pub use model::{
CIR_VERSION_V1, CirHashAlgorithm, CirObject, CirTal, CanonicalInputRepresentation,
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal,
};
pub use sequence::{CirSequenceManifest, CirSequenceStep, CirSequenceStepKind};
#[cfg(feature = "full")]
pub use export::{CirExportError, CirExportSummary, build_cir_from_run, export_cir_from_run, write_cir_file};
#[cfg(feature = "full")]
pub use static_pool::{
CirStaticPoolError, CirStaticPoolExportSummary, CirStaticPoolWriteResult,
export_hashes_from_store, static_pool_path, static_pool_relative_path,
@@ -59,9 +62,8 @@ mod tests {
],
tals: vec![CirTal {
tal_uri: "https://tal.example.net/root.tal".to_string(),
tal_bytes:
b"https://tal.example.net/ta.cer\nrsync://example.net/repo/ta.cer\nMIIB"
.to_vec(),
tal_bytes: b"https://tal.example.net/ta.cer\nrsync://example.net/repo/ta.cer\nMIIB"
.to_vec(),
}],
}
}
@@ -74,7 +76,10 @@ mod tests {
} else {
let len = value.len();
let bytes = len.to_be_bytes();
let first_non_zero = bytes.iter().position(|&b| b != 0).unwrap_or(bytes.len() - 1);
let first_non_zero = bytes
.iter()
.position(|&b| b != 0)
.unwrap_or(bytes.len() - 1);
let len_bytes = &bytes[first_non_zero..];
out.push(0x80 | len_bytes.len() as u8);
out.extend_from_slice(len_bytes);
@@ -178,8 +183,7 @@ mod tests {
der[idx + sha256_bytes.len() - 1] ^= 0x01;
let err = decode_cir(&der).expect_err("wrong oid must fail");
assert!(
err.to_string()
.contains(crate::data_model::oid::OID_SHA256),
err.to_string().contains(crate::data_model::oid::OID_SHA256),
"{err}"
);
}
@@ -222,7 +226,10 @@ mod tests {
tals: Vec::new(),
};
let err = encode_cir(&no_tals).expect_err("empty tals must fail");
assert!(err.to_string().contains("CIR.tals must be non-empty"), "{err}");
assert!(
err.to_string().contains("CIR.tals must be non-empty"),
"{err}"
);
}
#[test]
@@ -276,7 +283,10 @@ mod tests {
let mut der = encode_cir(&cir).expect("encode cir");
der.push(0);
let err = decode_cir(&der).expect_err("trailing bytes after cir must fail");
assert!(err.to_string().contains("trailing bytes after CIR"), "{err}");
assert!(
err.to_string().contains("trailing bytes after CIR"),
"{err}"
);
let object = test_encode_tlv(
0x30,
@@ -307,7 +317,10 @@ mod tests {
.concat(),
);
let err = decode_cir(&bad).expect_err("trailing field in object must fail");
assert!(err.to_string().contains("trailing fields in CirObject"), "{err}");
assert!(
err.to_string().contains("trailing fields in CirObject"),
"{err}"
);
}
#[test]

4
src/cir/model.rs
View File

@@ -111,7 +111,9 @@ fn validate_sorted_unique_strings<'a>(
) -> Result<(), String> {
let mut prev: Option<&'a str> = None;
for key in items {
if let Some(prev_key) = prev && key <= prev_key {
if let Some(prev_key) = prev
&& key <= prev_key
{
return Err(message.into());
}
prev = Some(key);

72
src/cir/sequence.rs
View File

@@ -15,13 +15,24 @@ pub struct CirSequenceStep {
pub cir_path: String,
pub ccr_path: String,
pub report_path: String,
#[serde(default)]
pub timing_path: Option<String>,
#[serde(default)]
pub stdout_log_path: Option<String>,
#[serde(default)]
pub stderr_log_path: Option<String>,
#[serde(default)]
pub artifact_prefix: Option<String>,
pub previous_step_id: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CirSequenceManifest {
pub version: u32,
pub static_root: String,
#[serde(default)]
pub static_root: Option<String>,
#[serde(default)]
pub raw_store_db_path: Option<String>,
pub steps: Vec<CirSequenceStep>,
}
@@ -30,8 +41,16 @@ impl CirSequenceManifest {
if self.version == 0 {
return Err("sequence.version must be positive".to_string());
}
if self.static_root.trim().is_empty() {
return Err("sequence.static_root must not be empty".to_string());
let backend_count = self.static_root.is_some() as u8 + self.raw_store_db_path.is_some() as u8;
if backend_count != 1 {
return Err("sequence must set exactly one of static_root or raw_store_db_path".to_string());
}
match (self.static_root.as_ref(), self.raw_store_db_path.as_ref()) {
(Some(static_root), None) if !static_root.trim().is_empty() => {}
(None, Some(raw_store_db_path)) if !raw_store_db_path.trim().is_empty() => {}
_ => {
return Err("sequence backend path must not be empty".to_string());
}
}
if self.steps.is_empty() {
return Err("sequence.steps must not be empty".to_string());
@@ -49,6 +68,34 @@ impl CirSequenceManifest {
"sequence.steps[{idx}].validation_time must not be empty"
));
}
if let Some(timing_path) = &step.timing_path
&& timing_path.trim().is_empty()
{
return Err(format!(
"sequence.steps[{idx}].timing_path must not be empty when set"
));
}
if let Some(stdout_log_path) = &step.stdout_log_path
&& stdout_log_path.trim().is_empty()
{
return Err(format!(
"sequence.steps[{idx}].stdout_log_path must not be empty when set"
));
}
if let Some(stderr_log_path) = &step.stderr_log_path
&& stderr_log_path.trim().is_empty()
{
return Err(format!(
"sequence.steps[{idx}].stderr_log_path must not be empty when set"
));
}
if let Some(artifact_prefix) = &step.artifact_prefix
&& artifact_prefix.trim().is_empty()
{
return Err(format!(
"sequence.steps[{idx}].artifact_prefix must not be empty when set"
));
}
if step.cir_path.trim().is_empty()
|| step.ccr_path.trim().is_empty()
|| step.report_path.trim().is_empty()
@@ -92,7 +139,8 @@ mod tests {
fn sample_manifest() -> CirSequenceManifest {
CirSequenceManifest {
version: 1,
static_root: "static".to_string(),
static_root: Some("static".to_string()),
raw_store_db_path: None,
steps: vec![
CirSequenceStep {
step_id: "full".to_string(),
@@ -101,6 +149,10 @@ mod tests {
cir_path: "full/input.cir".to_string(),
ccr_path: "full/result.ccr".to_string(),
report_path: "full/report.json".to_string(),
timing_path: Some("full/timing.json".to_string()),
stdout_log_path: Some("full/stdout.log".to_string()),
stderr_log_path: Some("full/stderr.log".to_string()),
artifact_prefix: Some("2026-04-09T00:00:00Z-test".to_string()),
previous_step_id: None,
},
CirSequenceStep {
@@ -110,6 +162,10 @@ mod tests {
cir_path: "delta-001/input.cir".to_string(),
ccr_path: "delta-001/result.ccr".to_string(),
report_path: "delta-001/report.json".to_string(),
timing_path: Some("delta-001/timing.json".to_string()),
stdout_log_path: Some("delta-001/stdout.log".to_string()),
stderr_log_path: Some("delta-001/stderr.log".to_string()),
artifact_prefix: Some("2026-04-09T00:10:00Z-test".to_string()),
previous_step_id: Some("full".to_string()),
},
],
@@ -144,4 +200,12 @@ mod tests {
let err = bad.validate().expect_err("missing previous must fail");
assert!(err.contains("previous_step_id"));
}
#[test]
fn sequence_manifest_validate_accepts_raw_store_backend() {
let mut manifest = sample_manifest();
manifest.static_root = None;
manifest.raw_store_db_path = Some("raw-store.db".to_string());
manifest.validate().expect("raw store sequence");
}
}

39
src/cir/static_pool.rs
View File

@@ -3,6 +3,7 @@ use std::fs::{self, OpenOptions};
use std::io::Write;
use std::path::{Path, PathBuf};
use crate::blob_store::RawObjectStore;
use crate::storage::{RawByHashEntry, RocksStore};
#[derive(Debug, thiserror::Error)]
@@ -75,10 +76,7 @@ pub fn static_pool_path(
capture_date_utc: time::Date,
sha256_hex: &str,
) -> Result<PathBuf, CirStaticPoolError> {
Ok(static_root.join(static_pool_relative_path(
capture_date_utc,
sha256_hex,
)?))
Ok(static_root.join(static_pool_relative_path(capture_date_utc, sha256_hex)?))
}
pub fn write_bytes_to_static_pool(
@@ -191,12 +189,13 @@ pub fn export_hashes_from_store(
let mut written_files = 0usize;
let mut reused_files = 0usize;
for sha256_hex in &unique {
let entry = store
.get_raw_by_hash_entry(sha256_hex)
let bytes = store
.get_blob_bytes(sha256_hex)
.map_err(|e| CirStaticPoolError::Storage(e.to_string()))?
.ok_or_else(|| CirStaticPoolError::MissingRawByHash {
sha256_hex: sha256_hex.clone(),
})?;
let entry = RawByHashEntry::from_bytes(sha256_hex.clone(), bytes);
let result = write_raw_entry_to_static_pool(static_root, capture_date_utc, &entry)?;
if result.written {
written_files += 1;
@@ -223,9 +222,7 @@ fn format_utc_date(date: time::Date) -> String {
fn validate_sha256_hex(sha256_hex: &str) -> Result<(), CirStaticPoolError> {
if sha256_hex.len() != 64 || !sha256_hex.as_bytes().iter().all(u8::is_ascii_hexdigit) {
return Err(CirStaticPoolError::InvalidSha256Hex(
sha256_hex.to_string(),
));
return Err(CirStaticPoolError::InvalidSha256Hex(sha256_hex.to_string()));
}
Ok(())
}
@@ -238,8 +235,8 @@ fn compute_sha256_hex(bytes: &[u8]) -> String {
#[cfg(test)]
mod tests {
use super::{
CirStaticPoolError, compute_sha256_hex, export_hashes_from_store, static_pool_relative_path,
write_bytes_to_static_pool,
CirStaticPoolError, compute_sha256_hex, export_hashes_from_store,
static_pool_relative_path, write_bytes_to_static_pool,
};
use crate::storage::{RawByHashEntry, RepositoryViewEntry, RepositoryViewState, RocksStore};
use std::fs;
@@ -270,8 +267,8 @@ mod tests {
let bytes = b"static-pool-object";
let sha = compute_sha256_hex(bytes);
let first = write_bytes_to_static_pool(td.path(), sample_date(), &sha, bytes)
.expect("first write");
let first =
write_bytes_to_static_pool(td.path(), sample_date(), &sha, bytes).expect("first write");
let second = write_bytes_to_static_pool(td.path(), sample_date(), &sha, bytes)
.expect("second write");
@@ -281,11 +278,13 @@ mod tests {
let all_files: Vec<_> = walk_files(td.path());
assert_eq!(all_files.len(), 1);
assert!(!all_files[0]
.file_name()
.and_then(|name| name.to_str())
.unwrap_or_default()
.contains(".tmp."));
assert!(
!all_files[0]
.file_name()
.and_then(|name| name.to_str())
.unwrap_or_default()
.contains(".tmp.")
);
}
#[test]
@@ -315,7 +314,9 @@ mod tests {
let bytes = b"store-object".to_vec();
let sha = compute_sha256_hex(&bytes);
let mut entry = RawByHashEntry::from_bytes(sha.clone(), bytes.clone());
entry.origin_uris.push("rsync://example.test/repo/object.cer".to_string());
entry
.origin_uris
.push("rsync://example.test/repo/object.cer".to_string());
store.put_raw_by_hash_entry(&entry).expect("put raw entry");
store
.put_repository_view_entry(&RepositoryViewEntry {