20260407 & 20260408 基于cir 三方replay对齐,并且materialize 使用hard link优化

This commit is contained in:
2026-04-08 16:27:46 +08:00
parent 34fb9657f1
commit c9ef5aaf4c
36 changed files with 8743 additions and 31 deletions

161
src/cir/decode.rs Normal file
View File

@@ -0,0 +1,161 @@
use crate::cir::model::{
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal,
};
use crate::data_model::common::DerReader;
use crate::data_model::oid::{OID_SHA256, OID_SHA256_RAW};
use der_parser::der::parse_der_oid;
#[derive(Debug, thiserror::Error)]
pub enum CirDecodeError {
#[error("DER parse error: {0}")]
Parse(String),
#[error("unexpected CIR version: expected {expected}, got {actual}")]
UnexpectedVersion { expected: u32, actual: u32 },
#[error("unexpected digest algorithm OID: expected {expected}, got {actual}")]
UnexpectedDigestAlgorithm { expected: &'static str, actual: String },
#[error("CIR model validation failed after decode: {0}")]
Validate(String),
}
pub fn decode_cir(der: &[u8]) -> Result<CanonicalInputRepresentation, CirDecodeError> {
let mut top = DerReader::new(der);
let mut seq = top.take_sequence().map_err(CirDecodeError::Parse)?;
if !top.is_empty() {
return Err(CirDecodeError::Parse("trailing bytes after CIR".into()));
}
let version = seq.take_uint_u64().map_err(CirDecodeError::Parse)? as u32;
if version != CIR_VERSION_V1 {
return Err(CirDecodeError::UnexpectedVersion {
expected: CIR_VERSION_V1,
actual: version,
});
}
let hash_alg = decode_hash_alg(seq.take_tag(0x06).map_err(CirDecodeError::Parse)?)?;
let validation_time =
parse_generalized_time(seq.take_tag(0x18).map_err(CirDecodeError::Parse)?)?;
let objects_der = seq.take_tag(0x30).map_err(CirDecodeError::Parse)?;
let mut objects_reader = DerReader::new(objects_der);
let mut objects = Vec::new();
while !objects_reader.is_empty() {
let (_tag, full, _value) = objects_reader
.take_any_full()
.map_err(CirDecodeError::Parse)?;
objects.push(decode_object(full)?);
}
let tals_der = seq.take_tag(0x30).map_err(CirDecodeError::Parse)?;
let mut tals_reader = DerReader::new(tals_der);
let mut tals = Vec::new();
while !tals_reader.is_empty() {
let (_tag, full, _value) = tals_reader.take_any_full().map_err(CirDecodeError::Parse)?;
tals.push(decode_tal(full)?);
}
if !seq.is_empty() {
return Err(CirDecodeError::Parse("trailing fields in CIR".into()));
}
let cir = CanonicalInputRepresentation {
version,
hash_alg,
validation_time,
objects,
tals,
};
cir.validate().map_err(CirDecodeError::Validate)?;
Ok(cir)
}
fn decode_hash_alg(raw_body: &[u8]) -> Result<CirHashAlgorithm, CirDecodeError> {
if raw_body != OID_SHA256_RAW {
return Err(CirDecodeError::UnexpectedDigestAlgorithm {
expected: OID_SHA256,
actual: oid_string(raw_body)?,
});
}
Ok(CirHashAlgorithm::Sha256)
}
fn decode_object(der: &[u8]) -> Result<CirObject, CirDecodeError> {
let mut top = DerReader::new(der);
let mut seq = top.take_sequence().map_err(CirDecodeError::Parse)?;
if !top.is_empty() {
return Err(CirDecodeError::Parse("trailing bytes after CirObject".into()));
}
let rsync_uri = std::str::from_utf8(seq.take_tag(0x16).map_err(CirDecodeError::Parse)?)
.map_err(|e| CirDecodeError::Parse(e.to_string()))?
.to_string();
let sha256 = seq.take_octet_string().map_err(CirDecodeError::Parse)?.to_vec();
if !seq.is_empty() {
return Err(CirDecodeError::Parse("trailing fields in CirObject".into()));
}
Ok(CirObject { rsync_uri, sha256 })
}
fn decode_tal(der: &[u8]) -> Result<CirTal, CirDecodeError> {
let mut top = DerReader::new(der);
let mut seq = top.take_sequence().map_err(CirDecodeError::Parse)?;
if !top.is_empty() {
return Err(CirDecodeError::Parse("trailing bytes after CirTal".into()));
}
let tal_uri = std::str::from_utf8(seq.take_tag(0x16).map_err(CirDecodeError::Parse)?)
.map_err(|e| CirDecodeError::Parse(e.to_string()))?
.to_string();
let tal_bytes = seq.take_octet_string().map_err(CirDecodeError::Parse)?.to_vec();
if !seq.is_empty() {
return Err(CirDecodeError::Parse("trailing fields in CirTal".into()));
}
Ok(CirTal { tal_uri, tal_bytes })
}
fn oid_string(raw_body: &[u8]) -> Result<String, CirDecodeError> {
let der = {
let mut out = Vec::with_capacity(raw_body.len() + 2);
out.push(0x06);
if raw_body.len() < 0x80 {
out.push(raw_body.len() as u8);
} else {
return Err(CirDecodeError::Parse("OID too long".into()));
}
out.extend_from_slice(raw_body);
out
};
let (_rem, oid) = parse_der_oid(&der).map_err(|e| CirDecodeError::Parse(e.to_string()))?;
let oid = oid
.as_oid_val()
.map_err(|e| CirDecodeError::Parse(e.to_string()))?;
Ok(oid.to_string())
}
fn parse_generalized_time(bytes: &[u8]) -> Result<time::OffsetDateTime, CirDecodeError> {
let s = std::str::from_utf8(bytes).map_err(|e| CirDecodeError::Parse(e.to_string()))?;
if s.len() != 15 || !s.ends_with('Z') {
return Err(CirDecodeError::Parse(
"GeneralizedTime must be YYYYMMDDHHMMSSZ".into(),
));
}
let parse = |range: std::ops::Range<usize>| -> Result<u32, CirDecodeError> {
s[range]
.parse::<u32>()
.map_err(|e| CirDecodeError::Parse(e.to_string()))
};
let year = parse(0..4)? as i32;
let month = parse(4..6)? as u8;
let day = parse(6..8)? as u8;
let hour = parse(8..10)? as u8;
let minute = parse(10..12)? as u8;
let second = parse(12..14)? as u8;
let month = time::Month::try_from(month)
.map_err(|e| CirDecodeError::Parse(e.to_string()))?;
let date = time::Date::from_calendar_date(year, month, day)
.map_err(|e| CirDecodeError::Parse(e.to_string()))?;
let timev = time::Time::from_hms(hour, minute, second)
.map_err(|e| CirDecodeError::Parse(e.to_string()))?;
Ok(time::PrimitiveDateTime::new(date, timev).assume_utc())
}

147
src/cir/encode.rs Normal file
View File

@@ -0,0 +1,147 @@
use crate::cir::model::{
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal,
};
use crate::data_model::oid::OID_SHA256_RAW;
#[derive(Debug, thiserror::Error)]
pub enum CirEncodeError {
#[error("CIR model validation failed: {0}")]
Validate(String),
}
pub fn encode_cir(
cir: &CanonicalInputRepresentation,
) -> Result<Vec<u8>, CirEncodeError> {
cir.validate().map_err(CirEncodeError::Validate)?;
Ok(encode_sequence(&[
encode_integer_u32(cir.version),
encode_oid(match cir.hash_alg {
CirHashAlgorithm::Sha256 => OID_SHA256_RAW,
}),
encode_generalized_time(cir.validation_time),
encode_sequence(
&cir.objects
.iter()
.map(encode_object)
.collect::<Result<Vec<_>, _>>()?,
),
encode_sequence(
&cir.tals
.iter()
.map(encode_tal)
.collect::<Result<Vec<_>, _>>()?,
),
]))
}
fn encode_object(object: &CirObject) -> Result<Vec<u8>, CirEncodeError> {
object.validate().map_err(CirEncodeError::Validate)?;
Ok(encode_sequence(&[
encode_ia5_string(object.rsync_uri.as_bytes()),
encode_octet_string(&object.sha256),
]))
}
fn encode_tal(tal: &CirTal) -> Result<Vec<u8>, CirEncodeError> {
tal.validate().map_err(CirEncodeError::Validate)?;
Ok(encode_sequence(&[
encode_ia5_string(tal.tal_uri.as_bytes()),
encode_octet_string(&tal.tal_bytes),
]))
}
fn encode_generalized_time(t: time::OffsetDateTime) -> Vec<u8> {
let t = t.to_offset(time::UtcOffset::UTC);
let s = format!(
"{:04}{:02}{:02}{:02}{:02}{:02}Z",
t.year(),
u8::from(t.month()),
t.day(),
t.hour(),
t.minute(),
t.second()
);
encode_tlv(0x18, s.into_bytes())
}
fn encode_integer_u32(v: u32) -> Vec<u8> {
encode_integer_bytes(unsigned_integer_bytes(v as u64))
}
fn encode_integer_bytes(mut bytes: Vec<u8>) -> Vec<u8> {
if bytes.is_empty() {
bytes.push(0);
}
if bytes[0] & 0x80 != 0 {
bytes.insert(0, 0);
}
encode_tlv(0x02, bytes)
}
fn unsigned_integer_bytes(v: u64) -> Vec<u8> {
if v == 0 {
return vec![0];
}
let mut out = Vec::new();
let mut n = v;
while n > 0 {
out.push((n & 0xFF) as u8);
n >>= 8;
}
out.reverse();
out
}
fn encode_oid(raw_body: &[u8]) -> Vec<u8> {
encode_tlv(0x06, raw_body.to_vec())
}
fn encode_ia5_string(bytes: &[u8]) -> Vec<u8> {
encode_tlv(0x16, bytes.to_vec())
}
fn encode_octet_string(bytes: &[u8]) -> Vec<u8> {
encode_tlv(0x04, bytes.to_vec())
}
fn encode_sequence(elements: &[Vec<u8>]) -> Vec<u8> {
let mut body = Vec::new();
for element in elements {
body.extend_from_slice(element);
}
encode_tlv(0x30, body)
}
fn encode_tlv(tag: u8, value: Vec<u8>) -> Vec<u8> {
let mut out = Vec::with_capacity(1 + encoded_len_len(value.len()) + value.len());
out.push(tag);
encode_len_into(value.len(), &mut out);
out.extend_from_slice(&value);
out
}
fn encoded_len_len(len: usize) -> usize {
if len < 0x80 {
1
} else {
1 + len.to_be_bytes().iter().skip_while(|&&b| b == 0).count()
}
}
fn encode_len_into(len: usize, out: &mut Vec<u8>) {
if len < 0x80 {
out.push(len as u8);
return;
}
let bytes = len.to_be_bytes();
let first_non_zero = bytes.iter().position(|&b| b != 0).unwrap_or(bytes.len() - 1);
let len_bytes = &bytes[first_non_zero..];
out.push(0x80 | (len_bytes.len() as u8));
out.extend_from_slice(len_bytes);
}
#[allow(dead_code)]
const _: () = {
let _ = CIR_VERSION_V1;
};

285
src/cir/export.rs Normal file
View File

@@ -0,0 +1,285 @@
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::path::Path;
use crate::cir::encode::{CirEncodeError, encode_cir};
use crate::cir::model::{
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal,
};
use crate::cir::static_pool::{
CirStaticPoolError, CirStaticPoolExportSummary, write_bytes_to_static_pool,
export_hashes_from_store,
};
use crate::data_model::ta::TrustAnchor;
use crate::storage::{RepositoryViewState, RocksStore};
#[derive(Debug, thiserror::Error)]
pub enum CirExportError {
#[error("list repository_view entries failed: {0}")]
ListRepositoryView(String),
#[error("CIR TAL URI must be http(s), got: {0}")]
InvalidTalUri(String),
#[error("TAL does not contain any rsync TA URI; CIR replay scheme A requires one")]
MissingTaRsyncUri,
#[error("CIR model validation failed: {0}")]
Validate(String),
#[error("encode CIR failed: {0}")]
Encode(#[from] CirEncodeError),
#[error("static pool export failed: {0}")]
StaticPool(#[from] CirStaticPoolError),
#[error("write CIR file failed: {0}: {1}")]
Write(String, String),
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CirExportSummary {
pub object_count: usize,
pub tal_count: usize,
pub static_pool: CirStaticPoolExportSummary,
}
pub fn build_cir_from_run(
store: &RocksStore,
trust_anchor: &TrustAnchor,
tal_uri: &str,
validation_time: time::OffsetDateTime,
) -> Result<CanonicalInputRepresentation, CirExportError> {
if !(tal_uri.starts_with("https://") || tal_uri.starts_with("http://")) {
return Err(CirExportError::InvalidTalUri(tal_uri.to_string()));
}
let entries = store
.list_repository_view_entries_with_prefix("rsync://")
.map_err(|e| CirExportError::ListRepositoryView(e.to_string()))?;
let mut objects: BTreeMap<String, String> = BTreeMap::new();
for entry in entries {
if matches!(
entry.state,
RepositoryViewState::Present | RepositoryViewState::Replaced
) && let Some(hash) = entry.current_hash
{
objects.insert(entry.rsync_uri, hash.to_ascii_lowercase());
}
}
let ta_hash = ta_sha256_hex(&trust_anchor.ta_certificate.raw_der);
let mut saw_rsync_uri = false;
for uri in &trust_anchor.tal.ta_uris {
if uri.scheme() == "rsync" {
saw_rsync_uri = true;
objects.insert(uri.as_str().to_string(), ta_hash.clone());
}
}
if !saw_rsync_uri {
return Err(CirExportError::MissingTaRsyncUri);
}
let cir = CanonicalInputRepresentation {
version: CIR_VERSION_V1,
hash_alg: CirHashAlgorithm::Sha256,
validation_time: validation_time.to_offset(time::UtcOffset::UTC),
objects: objects
.into_iter()
.map(|(rsync_uri, sha256_hex)| CirObject {
rsync_uri,
sha256: hex::decode(sha256_hex).expect("validated hex"),
})
.collect(),
tals: vec![CirTal {
tal_uri: tal_uri.to_string(),
tal_bytes: trust_anchor.tal.raw.clone(),
}],
};
cir.validate().map_err(CirExportError::Validate)?;
Ok(cir)
}
pub fn write_cir_file(path: &Path, cir: &CanonicalInputRepresentation) -> Result<(), CirExportError> {
let der = encode_cir(cir)?;
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)
.map_err(|e| CirExportError::Write(path.display().to_string(), e.to_string()))?;
}
std::fs::write(path, der)
.map_err(|e| CirExportError::Write(path.display().to_string(), e.to_string()))
}
pub fn export_cir_static_pool(
store: &RocksStore,
static_root: &Path,
capture_date_utc: time::Date,
cir: &CanonicalInputRepresentation,
trust_anchor: &TrustAnchor,
) -> Result<CirStaticPoolExportSummary, CirExportError> {
let ta_hash = ta_sha256_hex(&trust_anchor.ta_certificate.raw_der);
let hashes = cir
.objects
.iter()
.map(|item| hex::encode(&item.sha256))
.filter(|hash| hash != &ta_hash)
.collect::<Vec<_>>();
let mut summary = export_hashes_from_store(store, static_root, capture_date_utc, &hashes)?;
let ta_result = write_bytes_to_static_pool(
static_root,
capture_date_utc,
&ta_hash,
&trust_anchor.ta_certificate.raw_der,
)?;
let mut unique = hashes.iter().cloned().collect::<BTreeSet<_>>();
unique.insert(ta_hash.clone());
summary.unique_hashes = unique.len();
if ta_result.written {
summary.written_files += 1;
} else {
summary.reused_files += 1;
}
Ok(summary)
}
pub fn export_cir_from_run(
store: &RocksStore,
trust_anchor: &TrustAnchor,
tal_uri: &str,
validation_time: time::OffsetDateTime,
cir_out: &Path,
static_root: &Path,
capture_date_utc: time::Date,
) -> Result<CirExportSummary, CirExportError> {
let cir = build_cir_from_run(store, trust_anchor, tal_uri, validation_time)?;
let static_pool = export_cir_static_pool(store, static_root, capture_date_utc, &cir, trust_anchor)?;
write_cir_file(cir_out, &cir)?;
Ok(CirExportSummary {
object_count: cir.objects.len(),
tal_count: cir.tals.len(),
static_pool,
})
}
fn ta_sha256_hex(bytes: &[u8]) -> String {
use sha2::{Digest, Sha256};
hex::encode(Sha256::digest(bytes))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::cir::decode::decode_cir;
use crate::cir::static_pool_path;
use crate::data_model::ta::TrustAnchor;
use crate::data_model::tal::Tal;
use crate::storage::{RawByHashEntry, RepositoryViewEntry, RepositoryViewState, RocksStore};
fn sample_time() -> time::OffsetDateTime {
time::OffsetDateTime::parse(
"2026-04-07T12:34:56Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap()
}
fn sample_date() -> time::Date {
time::Date::from_calendar_date(2026, time::Month::April, 7).unwrap()
}
fn sample_trust_anchor() -> TrustAnchor {
let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let tal_bytes = std::fs::read(base.join("tests/fixtures/tal/apnic-rfc7730-https.tal")).unwrap();
let ta_der = std::fs::read(base.join("tests/fixtures/ta/apnic-ta.cer")).unwrap();
let tal = Tal::decode_bytes(&tal_bytes).unwrap();
TrustAnchor::bind_der(tal, &ta_der, None).unwrap()
}
fn sha256_hex(bytes: &[u8]) -> String {
use sha2::{Digest, Sha256};
hex::encode(Sha256::digest(bytes))
}
#[test]
fn build_cir_from_run_collects_repository_view_and_tal() {
let td = tempfile::tempdir().unwrap();
let store = RocksStore::open(td.path()).unwrap();
let bytes = b"object-a".to_vec();
let hash = sha256_hex(&bytes);
let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone());
raw.origin_uris.push("rsync://example.test/repo/a.cer".into());
store.put_raw_by_hash_entry(&raw).unwrap();
store
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/a.cer".to_string(),
current_hash: Some(hash),
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
object_type: Some("cer".to_string()),
state: RepositoryViewState::Present,
})
.unwrap();
let ta = sample_trust_anchor();
let cir = build_cir_from_run(&store, &ta, "https://example.test/root.tal", sample_time())
.expect("build cir");
assert_eq!(cir.version, CIR_VERSION_V1);
assert_eq!(cir.tals.len(), 1);
assert_eq!(cir.tals[0].tal_uri, "https://example.test/root.tal");
assert!(cir
.objects
.iter()
.any(|item| item.rsync_uri == "rsync://example.test/repo/a.cer"));
assert!(cir
.objects
.iter()
.any(|item| item.rsync_uri.contains("apnic-rpki-root-iana-origin.cer")));
}
#[test]
fn export_cir_from_run_writes_der_and_static_pool() {
let td = tempfile::tempdir().unwrap();
let store_dir = td.path().join("db");
let out_dir = td.path().join("out");
let static_root = td.path().join("static");
let store = RocksStore::open(&store_dir).unwrap();
let bytes = b"object-b".to_vec();
let hash = sha256_hex(&bytes);
let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone());
raw.origin_uris.push("rsync://example.test/repo/b.roa".into());
store.put_raw_by_hash_entry(&raw).unwrap();
store
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/b.roa".to_string(),
current_hash: Some(hash.clone()),
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
})
.unwrap();
let ta = sample_trust_anchor();
let cir_path = out_dir.join("example.cir");
let summary = export_cir_from_run(
&store,
&ta,
"https://example.test/root.tal",
sample_time(),
&cir_path,
&static_root,
sample_date(),
)
.expect("export cir");
assert_eq!(summary.tal_count, 1);
assert!(summary.object_count >= 2);
let der = std::fs::read(&cir_path).unwrap();
let cir = decode_cir(&der).unwrap();
assert_eq!(cir.tals[0].tal_uri, "https://example.test/root.tal");
let object_path = static_pool_path(&static_root, sample_date(), &hash).unwrap();
assert_eq!(std::fs::read(object_path).unwrap(), bytes);
}
}

388
src/cir/materialize.rs Normal file
View File

@@ -0,0 +1,388 @@
use std::fs;
use std::path::{Path, PathBuf};
use crate::cir::model::CanonicalInputRepresentation;
#[derive(Debug, thiserror::Error)]
pub enum CirMaterializeError {
#[error("invalid rsync URI: {0}")]
InvalidRsyncUri(String),
#[error("rsync URI must reference a file object, got directory-like URI: {0}")]
DirectoryLikeRsyncUri(String),
#[error("create mirror root failed: {path}: {detail}")]
CreateMirrorRoot { path: String, detail: String },
#[error("remove mirror root failed: {path}: {detail}")]
RemoveMirrorRoot { path: String, detail: String },
#[error("create parent directory failed: {path}: {detail}")]
CreateParent { path: String, detail: String },
#[error("remove existing target failed: {path}: {detail}")]
RemoveExistingTarget { path: String, detail: String },
#[error("static object not found for sha256={sha256_hex}")]
MissingStaticObject { sha256_hex: String },
#[error("link target failed: {src} -> {dst}: {detail}")]
Link { src: String, dst: String, detail: String },
#[error("copy target failed: {src} -> {dst}: {detail}")]
Copy { src: String, dst: String, detail: String },
#[error("mirror tree mismatch after materialize: {0}")]
TreeMismatch(String),
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CirMaterializeSummary {
pub object_count: usize,
pub linked_files: usize,
pub copied_files: usize,
}
pub fn materialize_cir(
cir: &CanonicalInputRepresentation,
static_root: &Path,
mirror_root: &Path,
clean_rebuild: bool,
) -> Result<CirMaterializeSummary, CirMaterializeError> {
cir.validate()
.map_err(CirMaterializeError::TreeMismatch)?;
if clean_rebuild && mirror_root.exists() {
fs::remove_dir_all(mirror_root).map_err(|e| CirMaterializeError::RemoveMirrorRoot {
path: mirror_root.display().to_string(),
detail: e.to_string(),
})?;
}
fs::create_dir_all(mirror_root).map_err(|e| CirMaterializeError::CreateMirrorRoot {
path: mirror_root.display().to_string(),
detail: e.to_string(),
})?;
let mut linked_files = 0usize;
let mut copied_files = 0usize;
for object in &cir.objects {
let sha256_hex = hex::encode(&object.sha256);
let source = resolve_static_pool_file(static_root, &sha256_hex)?;
let relative = mirror_relative_path_for_rsync_uri(&object.rsync_uri)?;
let target = mirror_root.join(&relative);
if let Some(parent) = target.parent() {
fs::create_dir_all(parent).map_err(|e| CirMaterializeError::CreateParent {
path: parent.display().to_string(),
detail: e.to_string(),
})?;
}
if target.exists() {
fs::remove_file(&target).map_err(|e| CirMaterializeError::RemoveExistingTarget {
path: target.display().to_string(),
detail: e.to_string(),
})?;
}
match fs::hard_link(&source, &target) {
Ok(()) => linked_files += 1,
Err(link_err) => {
fs::copy(&source, &target).map_err(|copy_err| CirMaterializeError::Copy {
src: source.display().to_string(),
dst: target.display().to_string(),
detail: format!("{copy_err}; original link error: {link_err}"),
})?;
copied_files += 1;
}
}
}
let actual = collect_materialized_uris(mirror_root)?;
let expected = cir
.objects
.iter()
.map(|item| item.rsync_uri.clone())
.collect::<std::collections::BTreeSet<_>>();
if actual != expected {
return Err(CirMaterializeError::TreeMismatch(format!(
"expected {} files, got {} files",
expected.len(),
actual.len()
)));
}
Ok(CirMaterializeSummary {
object_count: cir.objects.len(),
linked_files,
copied_files,
})
}
pub fn mirror_relative_path_for_rsync_uri(rsync_uri: &str) -> Result<PathBuf, CirMaterializeError> {
let url = url::Url::parse(rsync_uri)
.map_err(|_| CirMaterializeError::InvalidRsyncUri(rsync_uri.to_string()))?;
if url.scheme() != "rsync" {
return Err(CirMaterializeError::InvalidRsyncUri(rsync_uri.to_string()));
}
let host = url
.host_str()
.ok_or_else(|| CirMaterializeError::InvalidRsyncUri(rsync_uri.to_string()))?;
let segments = url
.path_segments()
.ok_or_else(|| CirMaterializeError::InvalidRsyncUri(rsync_uri.to_string()))?
.collect::<Vec<_>>();
if segments.is_empty() || segments.last().copied().unwrap_or_default().is_empty() {
return Err(CirMaterializeError::DirectoryLikeRsyncUri(
rsync_uri.to_string(),
));
}
let mut path = PathBuf::from(host);
for segment in segments {
if !segment.is_empty() {
path.push(segment);
}
}
Ok(path)
}
pub fn resolve_static_pool_file(
static_root: &Path,
sha256_hex: &str,
) -> Result<PathBuf, CirMaterializeError> {
if sha256_hex.len() != 64 || !sha256_hex.as_bytes().iter().all(u8::is_ascii_hexdigit) {
return Err(CirMaterializeError::MissingStaticObject {
sha256_hex: sha256_hex.to_string(),
});
}
let prefix1 = &sha256_hex[0..2];
let prefix2 = &sha256_hex[2..4];
let entries = fs::read_dir(static_root)
.map_err(|_| CirMaterializeError::MissingStaticObject {
sha256_hex: sha256_hex.to_string(),
})?;
let mut dates = entries
.filter_map(Result::ok)
.filter(|entry| entry.path().is_dir())
.map(|entry| entry.path())
.collect::<Vec<_>>();
dates.sort();
for date_dir in dates {
let candidate = date_dir.join(prefix1).join(prefix2).join(sha256_hex);
if candidate.is_file() {
return Ok(candidate);
}
}
Err(CirMaterializeError::MissingStaticObject {
sha256_hex: sha256_hex.to_string(),
})
}
fn collect_materialized_uris(
mirror_root: &Path,
) -> Result<std::collections::BTreeSet<String>, CirMaterializeError> {
let mut out = std::collections::BTreeSet::new();
let mut stack = vec![mirror_root.to_path_buf()];
while let Some(path) = stack.pop() {
for entry in fs::read_dir(&path).map_err(|e| CirMaterializeError::CreateMirrorRoot {
path: path.display().to_string(),
detail: e.to_string(),
})? {
let entry = entry.map_err(|e| CirMaterializeError::CreateMirrorRoot {
path: path.display().to_string(),
detail: e.to_string(),
})?;
let path = entry.path();
if path.is_dir() {
stack.push(path);
} else {
let rel = path
.strip_prefix(mirror_root)
.expect("materialized path under mirror root")
.to_string_lossy()
.replace('\\', "/");
let uri = format!("rsync://{rel}");
out.insert(uri);
}
}
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::{CirMaterializeError, materialize_cir, mirror_relative_path_for_rsync_uri, resolve_static_pool_file};
use crate::cir::model::{
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal,
};
use std::path::{Path, PathBuf};
fn sample_time() -> time::OffsetDateTime {
time::OffsetDateTime::parse(
"2026-04-07T12:34:56Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap()
}
fn sample_cir() -> CanonicalInputRepresentation {
CanonicalInputRepresentation {
version: CIR_VERSION_V1,
hash_alg: CirHashAlgorithm::Sha256,
validation_time: sample_time(),
objects: vec![
CirObject {
rsync_uri: "rsync://example.net/repo/a.cer".to_string(),
sha256: hex::decode(
"1111111111111111111111111111111111111111111111111111111111111111",
)
.unwrap(),
},
CirObject {
rsync_uri: "rsync://example.net/repo/nested/b.roa".to_string(),
sha256: hex::decode(
"2222222222222222222222222222222222222222222222222222222222222222",
)
.unwrap(),
},
],
tals: vec![CirTal {
tal_uri: "https://tal.example.net/root.tal".to_string(),
tal_bytes: b"x".to_vec(),
}],
}
}
#[test]
fn mirror_relative_path_for_rsync_uri_maps_host_and_path() {
let path =
mirror_relative_path_for_rsync_uri("rsync://example.net/repo/nested/b.roa").unwrap();
assert_eq!(path, PathBuf::from("example.net").join("repo").join("nested").join("b.roa"));
}
#[test]
fn resolve_static_pool_file_finds_hash_across_dates() {
let td = tempfile::tempdir().unwrap();
let path = td
.path()
.join("20260407")
.join("11")
.join("11");
std::fs::create_dir_all(&path).unwrap();
let file = path.join("1111111111111111111111111111111111111111111111111111111111111111");
std::fs::write(&file, b"x").unwrap();
let resolved = resolve_static_pool_file(
td.path(),
"1111111111111111111111111111111111111111111111111111111111111111",
)
.unwrap();
assert_eq!(resolved, file);
}
#[test]
fn resolve_static_pool_file_rejects_invalid_hash_and_missing_hash() {
let td = tempfile::tempdir().unwrap();
let err = resolve_static_pool_file(td.path(), "not-a-hash")
.expect_err("invalid hash should fail");
assert!(matches!(err, CirMaterializeError::MissingStaticObject { .. }));
let err = resolve_static_pool_file(
td.path(),
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
)
.expect_err("missing hash should fail");
assert!(matches!(err, CirMaterializeError::MissingStaticObject { .. }));
}
#[test]
fn mirror_relative_path_rejects_non_rsync_and_directory_like_uris() {
let err = mirror_relative_path_for_rsync_uri("https://example.net/repo/a.roa")
.expect_err("non-rsync uri must fail");
assert!(matches!(err, CirMaterializeError::InvalidRsyncUri(_)));
let err = mirror_relative_path_for_rsync_uri("rsync://example.net/repo/")
.expect_err("directory-like uri must fail");
assert!(matches!(err, CirMaterializeError::DirectoryLikeRsyncUri(_)));
}
#[test]
fn materialize_clean_rebuild_creates_exact_tree_and_removes_stale_files() {
let td = tempfile::tempdir().unwrap();
let static_root = td.path().join("static");
let mirror_root = td.path().join("mirror");
write_static(
&static_root,
"20260407",
"1111111111111111111111111111111111111111111111111111111111111111",
b"a",
);
write_static(
&static_root,
"20260407",
"2222222222222222222222222222222222222222222222222222222222222222",
b"b",
);
std::fs::create_dir_all(mirror_root.join("stale")).unwrap();
std::fs::write(mirror_root.join("stale/old.txt"), b"old").unwrap();
let summary = materialize_cir(&sample_cir(), &static_root, &mirror_root, true).unwrap();
assert_eq!(summary.object_count, 2);
assert_eq!(std::fs::read(mirror_root.join("example.net/repo/a.cer")).unwrap(), b"a");
assert_eq!(
std::fs::read(mirror_root.join("example.net/repo/nested/b.roa")).unwrap(),
b"b"
);
assert!(!mirror_root.join("stale/old.txt").exists());
}
#[test]
fn materialize_fails_when_static_object_missing() {
let td = tempfile::tempdir().unwrap();
let err = materialize_cir(
&sample_cir(),
td.path(),
&td.path().join("mirror"),
true,
)
.expect_err("missing static object must fail");
assert!(matches!(err, CirMaterializeError::MissingStaticObject { .. }));
}
#[test]
fn materialize_without_clean_rebuild_detects_stale_extra_files() {
let td = tempfile::tempdir().unwrap();
let static_root = td.path().join("static");
let mirror_root = td.path().join("mirror");
write_static(
&static_root,
"20260407",
"1111111111111111111111111111111111111111111111111111111111111111",
b"a",
);
write_static(
&static_root,
"20260407",
"2222222222222222222222222222222222222222222222222222222222222222",
b"b",
);
std::fs::create_dir_all(mirror_root.join("extra")).unwrap();
std::fs::write(mirror_root.join("extra/stale.txt"), b"stale").unwrap();
let err = materialize_cir(&sample_cir(), &static_root, &mirror_root, false)
.expect_err("stale extra files should fail exact tree check");
assert!(matches!(err, CirMaterializeError::TreeMismatch(_)));
}
fn write_static(root: &Path, date: &str, hash: &str, bytes: &[u8]) {
let path = root.join(date).join(&hash[0..2]).join(&hash[2..4]);
std::fs::create_dir_all(&path).unwrap();
std::fs::write(path.join(hash), bytes).unwrap();
}
}

335
src/cir/mod.rs Normal file
View File

@@ -0,0 +1,335 @@
pub mod decode;
pub mod encode;
pub mod materialize;
pub mod model;
#[cfg(feature = "full")]
pub mod export;
#[cfg(feature = "full")]
pub mod static_pool;
pub use decode::{CirDecodeError, decode_cir};
pub use encode::{CirEncodeError, encode_cir};
pub use materialize::{
CirMaterializeError, CirMaterializeSummary, materialize_cir, mirror_relative_path_for_rsync_uri,
resolve_static_pool_file,
};
pub use model::{
CIR_VERSION_V1, CirHashAlgorithm, CirObject, CirTal, CanonicalInputRepresentation,
};
#[cfg(feature = "full")]
pub use export::{CirExportError, CirExportSummary, build_cir_from_run, export_cir_from_run, write_cir_file};
#[cfg(feature = "full")]
pub use static_pool::{
CirStaticPoolError, CirStaticPoolExportSummary, CirStaticPoolWriteResult,
export_hashes_from_store, static_pool_path, static_pool_relative_path,
write_bytes_to_static_pool, write_raw_entry_to_static_pool,
};
#[cfg(test)]
mod tests {
use super::{
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal,
decode_cir, encode_cir,
};
fn sample_time() -> time::OffsetDateTime {
time::OffsetDateTime::parse(
"2026-04-07T12:34:56Z",
&time::format_description::well_known::Rfc3339,
)
.expect("valid rfc3339")
}
fn sample_cir() -> CanonicalInputRepresentation {
CanonicalInputRepresentation {
version: CIR_VERSION_V1,
hash_alg: CirHashAlgorithm::Sha256,
validation_time: sample_time(),
objects: vec![
CirObject {
rsync_uri: "rsync://example.net/repo/a.cer".to_string(),
sha256: vec![0x11; 32],
},
CirObject {
rsync_uri: "rsync://example.net/repo/b.roa".to_string(),
sha256: vec![0x22; 32],
},
],
tals: vec![CirTal {
tal_uri: "https://tal.example.net/root.tal".to_string(),
tal_bytes:
b"https://tal.example.net/ta.cer\nrsync://example.net/repo/ta.cer\nMIIB"
.to_vec(),
}],
}
}
fn test_encode_tlv(tag: u8, value: &[u8]) -> Vec<u8> {
let mut out = Vec::with_capacity(8 + value.len());
out.push(tag);
if value.len() < 0x80 {
out.push(value.len() as u8);
} else {
let len = value.len();
let bytes = len.to_be_bytes();
let first_non_zero = bytes.iter().position(|&b| b != 0).unwrap_or(bytes.len() - 1);
let len_bytes = &bytes[first_non_zero..];
out.push(0x80 | len_bytes.len() as u8);
out.extend_from_slice(len_bytes);
}
out.extend_from_slice(value);
out
}
#[test]
fn cir_roundtrip_full_succeeds() {
let cir = sample_cir();
let der = encode_cir(&cir).expect("encode cir");
let decoded = decode_cir(&der).expect("decode cir");
assert_eq!(decoded, cir);
}
#[test]
fn cir_roundtrip_minimal_succeeds() {
let cir = CanonicalInputRepresentation {
version: CIR_VERSION_V1,
hash_alg: CirHashAlgorithm::Sha256,
validation_time: sample_time(),
objects: Vec::new(),
tals: vec![CirTal {
tal_uri: "https://tal.example.net/minimal.tal".to_string(),
tal_bytes: b"rsync://example.net/repo/ta.cer\nMIIB".to_vec(),
}],
};
let der = encode_cir(&cir).expect("encode minimal cir");
let decoded = decode_cir(&der).expect("decode minimal cir");
assert_eq!(decoded, cir);
}
#[test]
fn cir_model_rejects_unsorted_duplicate_objects() {
let cir = CanonicalInputRepresentation {
version: CIR_VERSION_V1,
hash_alg: CirHashAlgorithm::Sha256,
validation_time: sample_time(),
objects: vec![
CirObject {
rsync_uri: "rsync://example.net/repo/z.roa".to_string(),
sha256: vec![0x11; 32],
},
CirObject {
rsync_uri: "rsync://example.net/repo/a.roa".to_string(),
sha256: vec![0x22; 32],
},
],
tals: vec![CirTal {
tal_uri: "https://tal.example.net/root.tal".to_string(),
tal_bytes: b"x".to_vec(),
}],
};
let err = encode_cir(&cir).expect_err("unsorted objects must fail");
assert!(err.to_string().contains("CIR.objects"), "{err}");
}
#[test]
fn cir_model_rejects_duplicate_tals() {
let cir = CanonicalInputRepresentation {
version: CIR_VERSION_V1,
hash_alg: CirHashAlgorithm::Sha256,
validation_time: sample_time(),
objects: Vec::new(),
tals: vec![
CirTal {
tal_uri: "https://tal.example.net/root.tal".to_string(),
tal_bytes: b"a".to_vec(),
},
CirTal {
tal_uri: "https://tal.example.net/root.tal".to_string(),
tal_bytes: b"b".to_vec(),
},
],
};
let err = encode_cir(&cir).expect_err("duplicate tals must fail");
assert!(err.to_string().contains("CIR.tals"), "{err}");
}
#[test]
fn cir_decode_rejects_wrong_version() {
let mut der = encode_cir(&sample_cir()).expect("encode cir");
let pos = der
.windows(3)
.position(|window| window == [0x02, 0x01, CIR_VERSION_V1 as u8])
.expect("find version integer");
der[pos + 2] = 2;
let err = decode_cir(&der).expect_err("wrong version must fail");
assert!(err.to_string().contains("unexpected CIR version"), "{err}");
}
#[test]
fn cir_decode_rejects_wrong_hash_oid() {
let mut der = encode_cir(&sample_cir()).expect("encode cir");
let sha256_bytes = crate::data_model::oid::OID_SHA256_RAW;
let idx = der
.windows(sha256_bytes.len())
.position(|window| window == sha256_bytes)
.expect("find sha256 oid");
der[idx + sha256_bytes.len() - 1] ^= 0x01;
let err = decode_cir(&der).expect_err("wrong oid must fail");
assert!(
err.to_string()
.contains(crate::data_model::oid::OID_SHA256),
"{err}"
);
}
#[test]
fn cir_decode_rejects_bad_generalized_time() {
let mut der = encode_cir(&sample_cir()).expect("encode cir");
let pos = der
.windows(15)
.position(|window| window == b"20260407123456Z")
.expect("find generalized time");
der[pos + 14] = b'X';
let err = decode_cir(&der).expect_err("bad time must fail");
assert!(err.to_string().contains("GeneralizedTime"), "{err}");
}
#[test]
fn cir_model_rejects_non_rsync_object_uri_and_empty_tals() {
let bad_object = CanonicalInputRepresentation {
version: CIR_VERSION_V1,
hash_alg: CirHashAlgorithm::Sha256,
validation_time: sample_time(),
objects: vec![CirObject {
rsync_uri: "https://example.net/repo/a.roa".to_string(),
sha256: vec![0x11; 32],
}],
tals: vec![CirTal {
tal_uri: "https://tal.example.net/root.tal".to_string(),
tal_bytes: b"x".to_vec(),
}],
};
let err = encode_cir(&bad_object).expect_err("non-rsync object uri must fail");
assert!(err.to_string().contains("rsync://"), "{err}");
let no_tals = CanonicalInputRepresentation {
version: CIR_VERSION_V1,
hash_alg: CirHashAlgorithm::Sha256,
validation_time: sample_time(),
objects: Vec::new(),
tals: Vec::new(),
};
let err = encode_cir(&no_tals).expect_err("empty tals must fail");
assert!(err.to_string().contains("CIR.tals must be non-empty"), "{err}");
}
#[test]
fn cir_model_rejects_non_utc_time_bad_hash_len_and_non_http_tal_uri() {
let bad_time = CanonicalInputRepresentation {
version: CIR_VERSION_V1,
hash_alg: CirHashAlgorithm::Sha256,
validation_time: sample_time().to_offset(time::UtcOffset::from_hms(8, 0, 0).unwrap()),
objects: Vec::new(),
tals: vec![CirTal {
tal_uri: "https://tal.example.net/root.tal".to_string(),
tal_bytes: b"x".to_vec(),
}],
};
let err = encode_cir(&bad_time).expect_err("non-utc validation time must fail");
assert!(err.to_string().contains("UTC"), "{err}");
let bad_hash = CanonicalInputRepresentation {
version: CIR_VERSION_V1,
hash_alg: CirHashAlgorithm::Sha256,
validation_time: sample_time(),
objects: vec![CirObject {
rsync_uri: "rsync://example.net/repo/a.roa".to_string(),
sha256: vec![0x11; 31],
}],
tals: vec![CirTal {
tal_uri: "https://tal.example.net/root.tal".to_string(),
tal_bytes: b"x".to_vec(),
}],
};
let err = encode_cir(&bad_hash).expect_err("bad digest len must fail");
assert!(err.to_string().contains("32 bytes"), "{err}");
let bad_tal_uri = CanonicalInputRepresentation {
version: CIR_VERSION_V1,
hash_alg: CirHashAlgorithm::Sha256,
validation_time: sample_time(),
objects: Vec::new(),
tals: vec![CirTal {
tal_uri: "ftp://tal.example.net/root.tal".to_string(),
tal_bytes: b"x".to_vec(),
}],
};
let err = encode_cir(&bad_tal_uri).expect_err("bad tal uri must fail");
assert!(err.to_string().contains("http:// or https://"), "{err}");
}
#[test]
fn cir_decode_rejects_trailing_bytes_and_trailing_fields() {
let cir = sample_cir();
let mut der = encode_cir(&cir).expect("encode cir");
der.push(0);
let err = decode_cir(&der).expect_err("trailing bytes after cir must fail");
assert!(err.to_string().contains("trailing bytes after CIR"), "{err}");
let object = test_encode_tlv(
0x30,
&[
test_encode_tlv(0x16, b"rsync://example.net/repo/a.roa"),
test_encode_tlv(0x04, &[0x11; 32]),
test_encode_tlv(0x02, &[0x01]),
]
.concat(),
);
let tal = test_encode_tlv(
0x30,
&[
test_encode_tlv(0x16, b"https://tal.example.net/root.tal"),
test_encode_tlv(0x04, b"x"),
]
.concat(),
);
let bad = test_encode_tlv(
0x30,
&[
test_encode_tlv(0x02, &[CIR_VERSION_V1 as u8]),
test_encode_tlv(0x06, crate::data_model::oid::OID_SHA256_RAW),
test_encode_tlv(0x18, b"20260407123456Z"),
test_encode_tlv(0x30, &object),
test_encode_tlv(0x30, &tal),
]
.concat(),
);
let err = decode_cir(&bad).expect_err("trailing field in object must fail");
assert!(err.to_string().contains("trailing fields in CirObject"), "{err}");
}
#[test]
fn cir_decode_rejects_invalid_object_and_tal_shapes() {
let cir = sample_cir();
let mut der = encode_cir(&cir).expect("encode cir");
let rsync_text = b"rsync://example.net/repo/a.cer";
let idx = der
.windows(rsync_text.len())
.position(|window| window == rsync_text)
.expect("find object uri");
der[idx] = 0xFF;
let err = decode_cir(&der).expect_err("invalid utf8 object uri must fail");
assert!(err.to_string().contains("utf-8"), "{err}");
let mut der = encode_cir(&cir).expect("encode cir");
let tal_text = b"https://tal.example.net/root.tal";
let idx = der
.windows(tal_text.len())
.position(|window| window == tal_text)
.expect("find tal uri");
der[idx] = 0xFF;
let err = decode_cir(&der).expect_err("invalid utf8 tal uri must fail");
assert!(err.to_string().contains("utf-8"), "{err}");
}
}

120
src/cir/model.rs Normal file
View File

@@ -0,0 +1,120 @@
use crate::data_model::oid::OID_SHA256;
pub const CIR_VERSION_V1: u32 = 1;
pub const DIGEST_LEN_SHA256: usize = 32;
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum CirHashAlgorithm {
Sha256,
}
impl CirHashAlgorithm {
pub fn oid(&self) -> &'static str {
match self {
Self::Sha256 => OID_SHA256,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CanonicalInputRepresentation {
pub version: u32,
pub hash_alg: CirHashAlgorithm,
pub validation_time: time::OffsetDateTime,
pub objects: Vec<CirObject>,
pub tals: Vec<CirTal>,
}
impl CanonicalInputRepresentation {
pub fn validate(&self) -> Result<(), String> {
if self.version != CIR_VERSION_V1 {
return Err(format!(
"CIR version must be {CIR_VERSION_V1}, got {}",
self.version
));
}
if !matches!(self.hash_alg, CirHashAlgorithm::Sha256) {
return Err("CIR hashAlg must be SHA-256".into());
}
if self.validation_time.offset() != time::UtcOffset::UTC {
return Err("CIR validationTime must be UTC".into());
}
validate_sorted_unique_strings(
self.objects.iter().map(|item| item.rsync_uri.as_str()),
"CIR.objects must be sorted by rsyncUri and unique",
)?;
validate_sorted_unique_strings(
self.tals.iter().map(|item| item.tal_uri.as_str()),
"CIR.tals must be sorted by talUri and unique",
)?;
if self.tals.is_empty() {
return Err("CIR.tals must be non-empty".into());
}
for object in &self.objects {
object.validate()?;
}
for tal in &self.tals {
tal.validate()?;
}
Ok(())
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CirObject {
pub rsync_uri: String,
pub sha256: Vec<u8>,
}
impl CirObject {
pub fn validate(&self) -> Result<(), String> {
if !self.rsync_uri.starts_with("rsync://") {
return Err(format!(
"CirObject.rsync_uri must start with rsync://, got {}",
self.rsync_uri
));
}
if self.sha256.len() != DIGEST_LEN_SHA256 {
return Err(format!(
"CirObject.sha256 must be {DIGEST_LEN_SHA256} bytes, got {}",
self.sha256.len()
));
}
Ok(())
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CirTal {
pub tal_uri: String,
pub tal_bytes: Vec<u8>,
}
impl CirTal {
pub fn validate(&self) -> Result<(), String> {
if !(self.tal_uri.starts_with("https://") || self.tal_uri.starts_with("http://")) {
return Err(format!(
"CirTal.tal_uri must start with http:// or https://, got {}",
self.tal_uri
));
}
if self.tal_bytes.is_empty() {
return Err("CirTal.tal_bytes must be non-empty".into());
}
Ok(())
}
}
fn validate_sorted_unique_strings<'a>(
items: impl IntoIterator<Item = &'a str>,
message: &str,
) -> Result<(), String> {
let mut prev: Option<&'a str> = None;
for key in items {
if let Some(prev_key) = prev && key <= prev_key {
return Err(message.into());
}
prev = Some(key);
}
Ok(())
}

376
src/cir/static_pool.rs Normal file
View File

@@ -0,0 +1,376 @@
use std::collections::BTreeSet;
use std::fs::{self, OpenOptions};
use std::io::Write;
use std::path::{Path, PathBuf};
use crate::storage::{RawByHashEntry, RocksStore};
#[derive(Debug, thiserror::Error)]
pub enum CirStaticPoolError {
#[error("invalid sha256 hex: {0}")]
InvalidSha256Hex(String),
#[error("raw bytes are empty for sha256={sha256_hex}")]
EmptyBytes { sha256_hex: String },
#[error("raw bytes do not match sha256 hex: {sha256_hex}")]
HashMismatch { sha256_hex: String },
#[error("create directory failed: {path}: {detail}")]
CreateDir { path: String, detail: String },
#[error("create temp file failed: {path}: {detail}")]
CreateTemp { path: String, detail: String },
#[error("write temp file failed: {path}: {detail}")]
WriteTemp { path: String, detail: String },
#[error("sync temp file failed: {path}: {detail}")]
SyncTemp { path: String, detail: String },
#[error("publish temp file failed: {temp_path} -> {final_path}: {detail}")]
Publish {
temp_path: String,
final_path: String,
detail: String,
},
#[error("remove temp file failed: {path}: {detail}")]
RemoveTemp { path: String, detail: String },
#[error("raw_by_hash entry missing for sha256={sha256_hex}")]
MissingRawByHash { sha256_hex: String },
#[error("storage error: {0}")]
Storage(String),
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CirStaticPoolWriteResult {
pub final_path: PathBuf,
pub written: bool,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CirStaticPoolExportSummary {
pub unique_hashes: usize,
pub written_files: usize,
pub reused_files: usize,
}
pub fn static_pool_relative_path(
capture_date_utc: time::Date,
sha256_hex: &str,
) -> Result<PathBuf, CirStaticPoolError> {
validate_sha256_hex(sha256_hex)?;
let date = format_utc_date(capture_date_utc);
Ok(PathBuf::from(date)
.join(&sha256_hex[0..2])
.join(&sha256_hex[2..4])
.join(sha256_hex))
}
pub fn static_pool_path(
static_root: &Path,
capture_date_utc: time::Date,
sha256_hex: &str,
) -> Result<PathBuf, CirStaticPoolError> {
Ok(static_root.join(static_pool_relative_path(
capture_date_utc,
sha256_hex,
)?))
}
pub fn write_bytes_to_static_pool(
static_root: &Path,
capture_date_utc: time::Date,
sha256_hex_value: &str,
bytes: &[u8],
) -> Result<CirStaticPoolWriteResult, CirStaticPoolError> {
validate_sha256_hex(sha256_hex_value)?;
if bytes.is_empty() {
return Err(CirStaticPoolError::EmptyBytes {
sha256_hex: sha256_hex_value.to_string(),
});
}
let computed = compute_sha256_hex(bytes);
if computed != sha256_hex_value.to_ascii_lowercase() {
return Err(CirStaticPoolError::HashMismatch {
sha256_hex: sha256_hex_value.to_string(),
});
}
let final_path = static_pool_path(static_root, capture_date_utc, sha256_hex_value)?;
if final_path.exists() {
return Ok(CirStaticPoolWriteResult {
final_path,
written: false,
});
}
let parent = final_path.parent().expect("static pool file has parent");
fs::create_dir_all(parent).map_err(|e| CirStaticPoolError::CreateDir {
path: parent.display().to_string(),
detail: e.to_string(),
})?;
let temp_path = parent.join(format!("{sha256_hex_value}.tmp.{}", uuid::Uuid::new_v4()));
let mut file = OpenOptions::new()
.create_new(true)
.write(true)
.open(&temp_path)
.map_err(|e| CirStaticPoolError::CreateTemp {
path: temp_path.display().to_string(),
detail: e.to_string(),
})?;
file.write_all(bytes)
.map_err(|e| CirStaticPoolError::WriteTemp {
path: temp_path.display().to_string(),
detail: e.to_string(),
})?;
file.sync_all().map_err(|e| CirStaticPoolError::SyncTemp {
path: temp_path.display().to_string(),
detail: e.to_string(),
})?;
drop(file);
match fs::hard_link(&temp_path, &final_path) {
Ok(()) => {
fs::remove_file(&temp_path).map_err(|e| CirStaticPoolError::RemoveTemp {
path: temp_path.display().to_string(),
detail: e.to_string(),
})?;
Ok(CirStaticPoolWriteResult {
final_path,
written: true,
})
}
Err(e) if final_path.exists() => {
fs::remove_file(&temp_path).map_err(|remove_err| CirStaticPoolError::RemoveTemp {
path: temp_path.display().to_string(),
detail: remove_err.to_string(),
})?;
let _ = e;
Ok(CirStaticPoolWriteResult {
final_path,
written: false,
})
}
Err(e) => Err(CirStaticPoolError::Publish {
temp_path: temp_path.display().to_string(),
final_path: final_path.display().to_string(),
detail: e.to_string(),
}),
}
}
pub fn write_raw_entry_to_static_pool(
static_root: &Path,
capture_date_utc: time::Date,
entry: &RawByHashEntry,
) -> Result<CirStaticPoolWriteResult, CirStaticPoolError> {
write_bytes_to_static_pool(
static_root,
capture_date_utc,
&entry.sha256_hex,
&entry.bytes,
)
}
pub fn export_hashes_from_store(
store: &RocksStore,
static_root: &Path,
capture_date_utc: time::Date,
sha256_hexes: &[String],
) -> Result<CirStaticPoolExportSummary, CirStaticPoolError> {
let unique: BTreeSet<String> = sha256_hexes
.iter()
.map(|item| item.to_ascii_lowercase())
.collect();
let mut written_files = 0usize;
let mut reused_files = 0usize;
for sha256_hex in &unique {
let entry = store
.get_raw_by_hash_entry(sha256_hex)
.map_err(|e| CirStaticPoolError::Storage(e.to_string()))?
.ok_or_else(|| CirStaticPoolError::MissingRawByHash {
sha256_hex: sha256_hex.clone(),
})?;
let result = write_raw_entry_to_static_pool(static_root, capture_date_utc, &entry)?;
if result.written {
written_files += 1;
} else {
reused_files += 1;
}
}
Ok(CirStaticPoolExportSummary {
unique_hashes: unique.len(),
written_files,
reused_files,
})
}
fn format_utc_date(date: time::Date) -> String {
format!(
"{:04}{:02}{:02}",
date.year(),
u8::from(date.month()),
date.day()
)
}
fn validate_sha256_hex(sha256_hex: &str) -> Result<(), CirStaticPoolError> {
if sha256_hex.len() != 64 || !sha256_hex.as_bytes().iter().all(u8::is_ascii_hexdigit) {
return Err(CirStaticPoolError::InvalidSha256Hex(
sha256_hex.to_string(),
));
}
Ok(())
}
fn compute_sha256_hex(bytes: &[u8]) -> String {
use sha2::{Digest, Sha256};
hex::encode(Sha256::digest(bytes))
}
#[cfg(test)]
mod tests {
use super::{
CirStaticPoolError, compute_sha256_hex, export_hashes_from_store, static_pool_relative_path,
write_bytes_to_static_pool,
};
use crate::storage::{RawByHashEntry, RepositoryViewEntry, RepositoryViewState, RocksStore};
use std::fs;
fn sample_date() -> time::Date {
time::Date::from_calendar_date(2026, time::Month::April, 7).unwrap()
}
#[test]
fn static_pool_relative_path_uses_date_and_hash_prefixes() {
let path = static_pool_relative_path(
sample_date(),
"abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789",
)
.expect("relative path");
assert_eq!(
path,
std::path::PathBuf::from("20260407")
.join("ab")
.join("cd")
.join("abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789")
);
}
#[test]
fn write_bytes_to_static_pool_is_idempotent_and_leaves_no_temp_files() {
let td = tempfile::tempdir().expect("tempdir");
let bytes = b"static-pool-object";
let sha = compute_sha256_hex(bytes);
let first = write_bytes_to_static_pool(td.path(), sample_date(), &sha, bytes)
.expect("first write");
let second = write_bytes_to_static_pool(td.path(), sample_date(), &sha, bytes)
.expect("second write");
assert!(first.written);
assert!(!second.written);
assert_eq!(fs::read(&first.final_path).expect("read final"), bytes);
let all_files: Vec<_> = walk_files(td.path());
assert_eq!(all_files.len(), 1);
assert!(!all_files[0]
.file_name()
.and_then(|name| name.to_str())
.unwrap_or_default()
.contains(".tmp."));
}
#[test]
fn write_bytes_to_static_pool_rejects_bad_hash_and_empty_bytes() {
let td = tempfile::tempdir().expect("tempdir");
let err = write_bytes_to_static_pool(td.path(), sample_date(), "not-a-hash", b"x")
.expect_err("bad hash must fail");
assert!(matches!(err, CirStaticPoolError::InvalidSha256Hex(_)));
let err = write_bytes_to_static_pool(
td.path(),
sample_date(),
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
b"",
)
.expect_err("empty bytes must fail");
assert!(matches!(err, CirStaticPoolError::EmptyBytes { .. }));
}
#[test]
fn export_hashes_from_store_writes_unique_entries_and_fails_when_missing() {
let td = tempfile::tempdir().expect("tempdir");
let store_dir = td.path().join("db");
let static_root = td.path().join("static");
let store = RocksStore::open(&store_dir).expect("open rocksdb");
let bytes = b"store-object".to_vec();
let sha = compute_sha256_hex(&bytes);
let mut entry = RawByHashEntry::from_bytes(sha.clone(), bytes.clone());
entry.origin_uris.push("rsync://example.test/repo/object.cer".to_string());
store.put_raw_by_hash_entry(&entry).expect("put raw entry");
store
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/object.cer".to_string(),
current_hash: Some(sha.clone()),
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
object_type: Some("cer".to_string()),
state: RepositoryViewState::Present,
})
.expect("put repository view");
let summary = export_hashes_from_store(
&store,
&static_root,
sample_date(),
&[sha.clone(), sha.clone()],
)
.expect("export hashes");
assert_eq!(summary.unique_hashes, 1);
assert_eq!(summary.written_files, 1);
assert_eq!(summary.reused_files, 0);
let summary = export_hashes_from_store(&store, &static_root, sample_date(), &[sha.clone()])
.expect("re-export hashes");
assert_eq!(summary.unique_hashes, 1);
assert_eq!(summary.written_files, 0);
assert_eq!(summary.reused_files, 1);
let err = export_hashes_from_store(
&store,
&static_root,
sample_date(),
&[String::from(
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
)],
)
.expect_err("missing raw_by_hash must fail");
assert!(matches!(err, CirStaticPoolError::MissingRawByHash { .. }));
}
fn walk_files(root: &std::path::Path) -> Vec<std::path::PathBuf> {
let mut out = Vec::new();
let mut stack = vec![root.to_path_buf()];
while let Some(path) = stack.pop() {
for entry in fs::read_dir(path).expect("read_dir") {
let entry = entry.expect("dir entry");
let path = entry.path();
if path.is_dir() {
stack.push(path);
} else {
out.push(path);
}
}
}
out.sort();
out
}
}