初始化 Panda RPKI v0.1.0 开源候选版本
Some checks failed
ci / rust (push) Has been cancelled
ci / docker (push) Has been cancelled

This commit is contained in:
Panda RPKI OSS Local
2026-09-09 18:01:15 +08:00
commit 277cbca878
203 changed files with 92495 additions and 0 deletions

View File

@@ -0,0 +1,835 @@
use std::path::PathBuf;
use std::sync::Arc;
use rocksdb::{DB, Options, WriteBatch};
use crate::repository::storage::{
RawByHashEntry, RocksDbMemoryDbSnapshot, RocksStore, StorageError, StorageResult,
memory_db_snapshot_for_column_families,
};
const RAW_BY_HASH_KEY_PREFIX: &str = "rawbyhash:";
const RAW_BLOB_KEY_PREFIX: &str = "rawblob:";
const REPO_BYTES_KEY_PREFIX: &str = "sha256:";
fn raw_by_hash_key(sha256_hex: &str) -> String {
format!("{RAW_BY_HASH_KEY_PREFIX}{sha256_hex}")
}
fn raw_blob_key(sha256_hex: &str) -> String {
format!("{RAW_BLOB_KEY_PREFIX}{sha256_hex}")
}
fn repo_bytes_key(sha256_hex: &str) -> String {
format!("{REPO_BYTES_KEY_PREFIX}{sha256_hex}")
}
fn validate_blob_sha256_hex(sha256_hex: &str) -> StorageResult<()> {
if sha256_hex.len() != 64 || !sha256_hex.as_bytes().iter().all(u8::is_ascii_hexdigit) {
return Err(StorageError::InvalidData {
entity: "raw_blob",
detail: format!("invalid sha256 hex: {sha256_hex}"),
});
}
Ok(())
}
fn validate_blob_bytes(bytes: &[u8]) -> StorageResult<()> {
if bytes.is_empty() {
return Err(StorageError::InvalidData {
entity: "raw_blob",
detail: "bytes must not be empty".to_string(),
});
}
Ok(())
}
pub trait RawObjectStore {
fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>>;
fn get_raw_entries_batch(
&self,
sha256_hexes: &[String],
) -> StorageResult<Vec<Option<RawByHashEntry>>>;
fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult<Option<Vec<u8>>> {
self.get_raw_entry(sha256_hex)
.map(|entry| entry.map(|entry| entry.bytes))
}
fn get_blob_bytes_batch(&self, sha256_hexes: &[String]) -> StorageResult<Vec<Option<Vec<u8>>>> {
self.get_raw_entries_batch(sha256_hexes).map(|entries| {
entries
.into_iter()
.map(|entry| entry.map(|entry| entry.bytes))
.collect()
})
}
}
#[derive(Clone, Debug)]
pub struct ExternalRawStoreDb {
path: PathBuf,
db: Arc<DB>,
}
#[derive(Clone, Debug)]
pub struct ExternalRepoBytesDb {
path: PathBuf,
db: Arc<DB>,
read_only: bool,
secondary: bool,
}
impl ExternalRawStoreDb {
pub fn open(path: impl Into<PathBuf>) -> StorageResult<Self> {
let path = path.into();
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).map_err(|e| StorageError::RocksDb(e.to_string()))?;
}
let mut opts = Options::default();
opts.create_if_missing(true);
opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
opts.set_max_open_files(512);
let db = DB::open(&opts, &path).map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(Self {
path,
db: Arc::new(db),
})
}
pub fn put_raw_entry(&self, entry: &RawByHashEntry) -> StorageResult<()> {
entry.validate_internal()?;
let key = raw_by_hash_key(&entry.sha256_hex);
let blob_key = raw_blob_key(&entry.sha256_hex);
let value = serde_cbor::to_vec(entry).map_err(|e| StorageError::Codec {
entity: "raw_by_hash",
detail: e.to_string(),
})?;
let blob_value = entry.bytes.clone();
self.db
.write({
let mut batch = WriteBatch::default();
batch.put(key.as_bytes(), value);
batch.put(blob_key.as_bytes(), blob_value);
batch
})
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(())
}
pub fn put_raw_entries_batch(&self, entries: &[RawByHashEntry]) -> StorageResult<()> {
if entries.is_empty() {
return Ok(());
}
let mut batch = WriteBatch::default();
for entry in entries {
entry.validate_internal()?;
let key = raw_by_hash_key(&entry.sha256_hex);
let blob_key = raw_blob_key(&entry.sha256_hex);
let value = serde_cbor::to_vec(entry).map_err(|e| StorageError::Codec {
entity: "raw_by_hash",
detail: e.to_string(),
})?;
batch.put(key.as_bytes(), value);
batch.put(blob_key.as_bytes(), entry.bytes.as_slice());
}
self.db
.write(batch)
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(())
}
pub fn put_blob_bytes_batch(&self, blobs: &[(String, Vec<u8>)]) -> StorageResult<()> {
if blobs.is_empty() {
return Ok(());
}
let mut batch = WriteBatch::default();
for (sha256_hex, bytes) in blobs {
validate_blob_sha256_hex(sha256_hex)?;
validate_blob_bytes(bytes)?;
let blob_key = raw_blob_key(sha256_hex);
batch.put(blob_key.as_bytes(), bytes.as_slice());
}
self.db
.write(batch)
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(())
}
pub fn delete_raw_entry(&self, sha256_hex: &str) -> StorageResult<()> {
let key = raw_by_hash_key(sha256_hex);
let blob_key = raw_blob_key(sha256_hex);
self.db
.write({
let mut batch = WriteBatch::default();
batch.delete(key.as_bytes());
batch.delete(blob_key.as_bytes());
batch
})
.map_err(|e| StorageError::RocksDb(e.to_string()))
}
pub fn path(&self) -> &PathBuf {
&self.path
}
pub(crate) fn memory_snapshot(&self, label: impl Into<String>) -> RocksDbMemoryDbSnapshot {
memory_db_snapshot_for_column_families(label, self.db.as_ref(), None)
}
}
impl ExternalRepoBytesDb {
pub fn open(path: impl Into<PathBuf>) -> StorageResult<Self> {
let path = path.into();
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).map_err(|e| StorageError::RocksDb(e.to_string()))?;
}
let mut opts = Options::default();
opts.create_if_missing(true);
opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
opts.set_max_open_files(512);
let db = DB::open(&opts, &path).map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(Self {
path,
db: Arc::new(db),
read_only: false,
secondary: false,
})
}
pub fn open_read_only(path: impl Into<PathBuf>) -> StorageResult<Self> {
let path = path.into();
let mut opts = Options::default();
opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
opts.set_max_open_files(512);
let db = DB::open_for_read_only(&opts, &path, false)
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(Self {
path,
db: Arc::new(db),
read_only: true,
secondary: false,
})
}
/// Open the repo-bytes DB as a RocksDB secondary instance.
///
/// Unlike `open_read_only` (a frozen point-in-time view), a secondary
/// instance can follow the live primary via `try_catch_up_with_primary`,
/// which is required when the soak keeps writing new object bytes while
/// the query service is running.
pub fn open_as_secondary(
path: impl Into<PathBuf>,
secondary_path: impl Into<PathBuf>,
) -> StorageResult<Self> {
let path = path.into();
let secondary_path = secondary_path.into();
if let Some(parent) = secondary_path.parent() {
std::fs::create_dir_all(parent).map_err(|e| StorageError::RocksDb(e.to_string()))?;
}
let mut opts = Options::default();
opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
opts.set_max_open_files(512);
let db = DB::open_as_secondary(&opts, &path, &secondary_path)
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(Self {
path,
db: Arc::new(db),
read_only: true,
secondary: true,
})
}
/// Pull the secondary view up to the primary's current state. No-op for
/// non-secondary handles.
pub fn try_catch_up_with_primary(&self) -> StorageResult<()> {
if self.secondary {
self.db
.try_catch_up_with_primary()
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
}
Ok(())
}
pub fn put_blob_bytes_batch(&self, blobs: &[(String, Vec<u8>)]) -> StorageResult<()> {
if blobs.is_empty() {
return Ok(());
}
if self.read_only {
return Err(StorageError::RocksDb(format!(
"repo-bytes DB is read-only: {}",
self.path.display()
)));
}
let mut batch = WriteBatch::default();
for (sha256_hex, bytes) in blobs {
validate_blob_sha256_hex(sha256_hex)?;
validate_blob_bytes(bytes)?;
let key = repo_bytes_key(sha256_hex);
batch.put(key.as_bytes(), bytes.as_slice());
}
self.db
.write(batch)
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(())
}
pub(crate) fn is_read_only(&self) -> bool {
self.read_only
}
pub(crate) fn require_existing_blob_bytes_batch(
&self,
blobs: &[(String, Vec<u8>)],
) -> StorageResult<()> {
if blobs.is_empty() {
return Ok(());
}
let hashes = blobs
.iter()
.map(|(sha256_hex, _)| sha256_hex.clone())
.collect::<Vec<_>>();
let existing = self.get_blob_bytes_batch(&hashes)?;
for ((sha256_hex, expected_bytes), actual_bytes) in blobs.iter().zip(existing) {
match actual_bytes {
Some(actual_bytes) if actual_bytes == *expected_bytes => {}
Some(_) => {
return Err(StorageError::InvalidData {
entity: "read_only_repo_bytes",
detail: format!("existing bytes differ for SHA-256 {sha256_hex}"),
});
}
None => {
return Err(StorageError::InvalidData {
entity: "read_only_repo_bytes",
detail: format!("blob is missing for SHA-256 {sha256_hex}"),
});
}
}
}
Ok(())
}
pub fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult<Option<Vec<u8>>> {
validate_blob_sha256_hex(sha256_hex)?;
let key = repo_bytes_key(sha256_hex);
self.db
.get(key.as_bytes())
.map_err(|e| StorageError::RocksDb(e.to_string()))
}
pub fn get_blob_bytes_batch(
&self,
sha256_hexes: &[String],
) -> StorageResult<Vec<Option<Vec<u8>>>> {
if sha256_hexes.is_empty() {
return Ok(Vec::new());
}
let keys: Vec<String> = sha256_hexes
.iter()
.map(|hash| {
validate_blob_sha256_hex(hash)?;
Ok::<String, StorageError>(repo_bytes_key(hash))
})
.collect::<Result<_, _>>()?;
self.db
.multi_get(keys.iter().map(|key| key.as_bytes()))
.into_iter()
.map(|res| res.map_err(|e| StorageError::RocksDb(e.to_string())))
.collect()
}
pub fn path(&self) -> &PathBuf {
&self.path
}
pub(crate) fn memory_snapshot(&self, label: impl Into<String>) -> RocksDbMemoryDbSnapshot {
memory_db_snapshot_for_column_families(label, self.db.as_ref(), None)
}
}
impl RawObjectStore for RocksStore {
fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>> {
self.get_raw_by_hash_entry(sha256_hex)
}
fn get_raw_entries_batch(
&self,
sha256_hexes: &[String],
) -> StorageResult<Vec<Option<RawByHashEntry>>> {
self.get_raw_by_hash_entries_batch(sha256_hexes)
}
fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult<Option<Vec<u8>>> {
RocksStore::get_blob_bytes(self, sha256_hex)
}
fn get_blob_bytes_batch(&self, sha256_hexes: &[String]) -> StorageResult<Vec<Option<Vec<u8>>>> {
RocksStore::get_blob_bytes_batch(self, sha256_hexes)
}
}
impl RawObjectStore for ExternalRawStoreDb {
fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>> {
let key = raw_by_hash_key(sha256_hex);
let Some(bytes) = self
.db
.get(key.as_bytes())
.map_err(|e| StorageError::RocksDb(e.to_string()))?
else {
return Ok(None);
};
let entry =
serde_cbor::from_slice::<RawByHashEntry>(&bytes).map_err(|e| StorageError::Codec {
entity: "raw_by_hash",
detail: e.to_string(),
})?;
entry.validate_internal()?;
Ok(Some(entry))
}
fn get_raw_entries_batch(
&self,
sha256_hexes: &[String],
) -> StorageResult<Vec<Option<RawByHashEntry>>> {
if sha256_hexes.is_empty() {
return Ok(Vec::new());
}
let keys: Vec<String> = sha256_hexes
.iter()
.map(|hash| raw_by_hash_key(hash))
.collect();
self.db
.multi_get(keys.iter().map(|key| key.as_bytes()))
.into_iter()
.map(|res| {
let maybe = res.map_err(|e| StorageError::RocksDb(e.to_string()))?;
match maybe {
Some(bytes) => {
let entry =
serde_cbor::from_slice::<RawByHashEntry>(&bytes).map_err(|e| {
StorageError::Codec {
entity: "raw_by_hash",
detail: e.to_string(),
}
})?;
entry.validate_internal()?;
Ok(Some(entry))
}
None => Ok(None),
}
})
.collect()
}
fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult<Option<Vec<u8>>> {
let key = raw_blob_key(sha256_hex);
self.db
.get(key.as_bytes())
.map_err(|e| StorageError::RocksDb(e.to_string()))
}
fn get_blob_bytes_batch(&self, sha256_hexes: &[String]) -> StorageResult<Vec<Option<Vec<u8>>>> {
if sha256_hexes.is_empty() {
return Ok(Vec::new());
}
let keys: Vec<String> = sha256_hexes.iter().map(|hash| raw_blob_key(hash)).collect();
self.db
.multi_get(keys.iter().map(|key| key.as_bytes()))
.into_iter()
.map(|res| res.map_err(|e| StorageError::RocksDb(e.to_string())))
.collect()
}
}
#[cfg(test)]
mod tests {
use super::{ExternalRawStoreDb, ExternalRepoBytesDb, RawObjectStore};
use crate::repository::storage::{RawByHashEntry, RocksStore, StorageError, StorageResult};
use std::collections::HashMap;
fn sha256_hex(bytes: &[u8]) -> String {
use sha2::{Digest, Sha256};
hex::encode(Sha256::digest(bytes))
}
#[derive(Default)]
struct MockRawStore {
entries: HashMap<String, RawByHashEntry>,
}
impl RawObjectStore for MockRawStore {
fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>> {
Ok(self.entries.get(sha256_hex).cloned())
}
fn get_raw_entries_batch(
&self,
sha256_hexes: &[String],
) -> StorageResult<Vec<Option<RawByHashEntry>>> {
Ok(sha256_hexes
.iter()
.map(|hash| self.entries.get(hash).cloned())
.collect())
}
}
#[test]
fn rocks_store_raw_object_store_reads_single_and_batch_entries() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let a = b"object-a".to_vec();
let b = b"object-b".to_vec();
let a_hash = sha256_hex(&a);
let b_hash = sha256_hex(&b);
store
.put_raw_by_hash_entry(&RawByHashEntry::from_bytes(a_hash.clone(), a.clone()))
.expect("put a");
store
.put_raw_by_hash_entry(&RawByHashEntry::from_bytes(b_hash.clone(), b.clone()))
.expect("put b");
let single = store
.get_raw_entry(&a_hash)
.expect("get single")
.expect("present");
assert_eq!(single.bytes, a);
let batch = store
.get_raw_entries_batch(&[a_hash.clone(), "00".repeat(32), b_hash.clone()])
.expect("get batch");
assert_eq!(batch.len(), 3);
assert_eq!(
batch[0].as_ref().map(|entry| entry.bytes.as_slice()),
Some(a.as_slice())
);
assert!(batch[1].is_none());
assert_eq!(
batch[2].as_ref().map(|entry| entry.bytes.as_slice()),
Some(b.as_slice())
);
}
#[test]
fn external_raw_store_db_roundtrips_entries() {
let td = tempfile::tempdir().expect("tempdir");
let raw_store =
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let mut entry = RawByHashEntry::from_bytes(sha256_hex(b"blob"), b"blob".to_vec());
entry
.origin_uris
.push("rsync://example.test/repo/a.cer".to_string());
entry.object_type = Some("cer".to_string());
raw_store.put_raw_entry(&entry).expect("put raw entry");
let got = raw_store
.get_raw_entry(&entry.sha256_hex)
.expect("read raw entry")
.expect("entry exists");
assert_eq!(got, entry);
}
#[test]
fn external_raw_store_db_batch_writes_and_reads() {
let td = tempfile::tempdir().expect("tempdir");
let raw_store =
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let a = RawByHashEntry::from_bytes(sha256_hex(b"a"), b"a".to_vec());
let b = RawByHashEntry::from_bytes(sha256_hex(b"b"), b"b".to_vec());
raw_store
.put_raw_entries_batch(&[a.clone(), b.clone()])
.expect("batch put");
let batch = raw_store
.get_raw_entries_batch(&[a.sha256_hex.clone(), b.sha256_hex.clone()])
.expect("batch get");
assert_eq!(batch.len(), 2);
assert_eq!(batch[0], Some(a));
assert_eq!(batch[1], Some(b));
}
#[test]
fn raw_object_store_default_blob_helpers_return_bytes_only() {
let td = tempfile::tempdir().expect("tempdir");
let raw_store = ExternalRawStoreDb::open(td.path().join("nested/raw-store.db"))
.expect("open raw store");
let mut entry = RawByHashEntry::from_bytes(sha256_hex(b"blob"), b"blob".to_vec());
entry
.origin_uris
.push("rsync://example.test/repo/blob.roa".to_string());
raw_store.put_raw_entry(&entry).expect("put raw entry");
let single = raw_store
.get_blob_bytes(&entry.sha256_hex)
.expect("get blob bytes")
.expect("entry exists");
assert_eq!(single, b"blob".to_vec());
let batch = raw_store
.get_blob_bytes_batch(&[entry.sha256_hex.clone(), "00".repeat(32)])
.expect("get blob bytes batch");
assert_eq!(batch, vec![Some(b"blob".to_vec()), None]);
}
#[test]
fn raw_object_store_default_blob_helpers_work_for_custom_store() {
let mut store = MockRawStore::default();
let a = RawByHashEntry::from_bytes(sha256_hex(b"a"), b"a".to_vec());
let b = RawByHashEntry::from_bytes(sha256_hex(b"b"), b"b".to_vec());
store.entries.insert(a.sha256_hex.clone(), a.clone());
store.entries.insert(b.sha256_hex.clone(), b.clone());
let single = store
.get_blob_bytes(&a.sha256_hex)
.expect("single blob bytes")
.expect("present");
assert_eq!(single, b"a".to_vec());
let batch = store
.get_blob_bytes_batch(&[a.sha256_hex.clone(), "00".repeat(32), b.sha256_hex.clone()])
.expect("batch blob bytes");
assert_eq!(batch, vec![Some(b"a".to_vec()), None, Some(b"b".to_vec())]);
}
#[test]
fn rocks_store_blob_helpers_use_external_raw_store_fast_path() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open_with_external_raw_store(
&td.path().join("db"),
&td.path().join("raw-store.db"),
)
.expect("open store with external raw store");
let entry = RawByHashEntry::from_bytes(sha256_hex(b"blob-fast"), b"blob-fast".to_vec());
store.put_raw_by_hash_entry(&entry).expect("put");
let single = store
.get_blob_bytes(&entry.sha256_hex)
.expect("single blob bytes")
.expect("present");
assert_eq!(single, b"blob-fast".to_vec());
let batch = store
.get_blob_bytes_batch(&[entry.sha256_hex.clone(), "00".repeat(32)])
.expect("batch blob bytes");
assert_eq!(batch, vec![Some(b"blob-fast".to_vec()), None]);
}
#[test]
fn external_raw_store_db_delete_removes_entry() {
let td = tempfile::tempdir().expect("tempdir");
let raw_store =
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let entry = RawByHashEntry::from_bytes(sha256_hex(b"gone"), b"gone".to_vec());
raw_store.put_raw_entry(&entry).expect("put");
assert!(
raw_store
.get_raw_entry(&entry.sha256_hex)
.unwrap()
.is_some()
);
raw_store
.delete_raw_entry(&entry.sha256_hex)
.expect("delete entry");
assert!(
raw_store
.get_raw_entry(&entry.sha256_hex)
.unwrap()
.is_none()
);
}
#[test]
fn put_blob_bytes_batch_round_trips_without_raw_entry() {
let td = tempfile::tempdir().expect("tempdir");
let raw_store =
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let a = (sha256_hex(b"blob-a"), b"blob-a".to_vec());
let b = (sha256_hex(b"blob-b"), b"blob-b".to_vec());
raw_store
.put_blob_bytes_batch(&[a.clone(), b.clone()])
.expect("put blobs");
assert_eq!(
raw_store.get_blob_bytes(&a.0).expect("get blob a"),
Some(a.1.clone())
);
assert_eq!(
raw_store.get_blob_bytes(&b.0).expect("get blob b"),
Some(b.1.clone())
);
assert!(raw_store.get_raw_entry(&a.0).expect("get raw a").is_none());
assert!(raw_store.get_raw_entry(&b.0).expect("get raw b").is_none());
}
#[test]
fn put_blob_bytes_batch_rejects_invalid_inputs() {
let td = tempfile::tempdir().expect("tempdir");
let raw_store =
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let err = raw_store
.put_blob_bytes_batch(&[("zz".repeat(32), b"blob".to_vec())])
.expect_err("invalid hash should fail");
assert!(matches!(err, StorageError::InvalidData { .. }));
let err = raw_store
.put_blob_bytes_batch(&[(sha256_hex(b"blob"), Vec::new())])
.expect_err("empty bytes should fail");
assert!(matches!(err, StorageError::InvalidData { .. }));
}
#[test]
fn external_raw_store_db_rejects_invalid_entry_on_put() {
let td = tempfile::tempdir().expect("tempdir");
let raw_store =
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let bad = RawByHashEntry {
sha256_hex: "11".repeat(32),
bytes: b"blob".to_vec(),
origin_uris: Vec::new(),
object_type: None,
encoding: None,
};
let err = raw_store
.put_raw_entry(&bad)
.expect_err("invalid hash should fail");
assert!(matches!(err, StorageError::InvalidData { .. }));
}
#[test]
fn external_raw_store_db_reports_codec_error_for_corrupt_value() {
let td = tempfile::tempdir().expect("tempdir");
let raw_store =
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
raw_store
.db
.put(b"rawbyhash:deadbeef", b"not-cbor")
.expect("inject corrupt bytes");
let err = raw_store
.get_raw_entry("deadbeef")
.expect_err("corrupt value should fail");
assert!(matches!(
err,
StorageError::Codec {
entity: "raw_by_hash",
..
}
));
}
#[test]
fn external_raw_store_db_batch_returns_empty_for_empty_request() {
let td = tempfile::tempdir().expect("tempdir");
let raw_store =
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let entries = raw_store
.get_raw_entries_batch(&[])
.expect("empty batch succeeds");
assert!(entries.is_empty());
raw_store
.put_raw_entries_batch(&[])
.expect("empty put succeeds");
}
#[test]
fn external_repo_bytes_db_roundtrips_blob_bytes_without_raw_entry() {
let td = tempfile::tempdir().expect("tempdir");
let repo_bytes =
ExternalRepoBytesDb::open(td.path().join("repo-bytes.db")).expect("open repo bytes");
let bytes = b"repo-bytes-object".to_vec();
let hash = sha256_hex(&bytes);
repo_bytes
.put_blob_bytes_batch(&[(hash.clone(), bytes.clone())])
.expect("put repo bytes");
assert_eq!(
repo_bytes.get_blob_bytes(&hash).expect("get repo bytes"),
Some(bytes.clone())
);
assert_eq!(
repo_bytes
.get_blob_bytes_batch(&[hash, "00".repeat(32)])
.expect("get repo bytes batch"),
vec![Some(bytes), None]
);
}
#[test]
fn external_repo_bytes_db_rejects_invalid_inputs() {
let td = tempfile::tempdir().expect("tempdir");
let repo_bytes =
ExternalRepoBytesDb::open(td.path().join("repo-bytes.db")).expect("open repo bytes");
assert!(
repo_bytes
.put_blob_bytes_batch(&[("not-a-valid-hash".to_string(), b"blob".to_vec())])
.is_err()
);
assert!(
repo_bytes
.put_blob_bytes_batch(&[(sha256_hex(b"blob"), Vec::new())])
.is_err()
);
assert!(repo_bytes.get_blob_bytes("not-a-valid-hash").is_err());
}
#[test]
fn external_repo_bytes_db_secondary_catches_up_with_live_primary() {
let td = tempfile::tempdir().expect("tempdir");
let primary_path = td.path().join("repo-bytes.db");
let secondary_path = td.path().join("repo-bytes.secondary");
let primary = ExternalRepoBytesDb::open(&primary_path).expect("open primary");
let bytes_a = b"repo-bytes-a".to_vec();
let hash_a = sha256_hex(&bytes_a);
primary
.put_blob_bytes_batch(&[(hash_a.clone(), bytes_a.clone())])
.expect("put a");
let secondary = ExternalRepoBytesDb::open_as_secondary(&primary_path, &secondary_path)
.expect("open secondary");
assert!(secondary.secondary);
// A secondary open does not necessarily see pre-existing data until it
// catches up with the primary's manifest.
secondary
.try_catch_up_with_primary()
.expect("initial catch up");
assert_eq!(
secondary
.get_blob_bytes(&hash_a)
.expect("get a via secondary"),
Some(bytes_a)
);
// Bytes written by the primary *after* the secondary open become
// visible after another catch-up (this is the live-soak scenario).
let bytes_b = b"repo-bytes-b".to_vec();
let hash_b = sha256_hex(&bytes_b);
primary
.put_blob_bytes_batch(&[(hash_b.clone(), bytes_b.clone())])
.expect("put b after secondary open");
secondary
.try_catch_up_with_primary()
.expect("second catch up");
assert_eq!(
secondary
.get_blob_bytes(&hash_b)
.expect("get b via secondary"),
Some(bytes_b)
);
}
}

View File

@@ -0,0 +1,289 @@
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, RwLock};
use crate::repository::storage::{RepositoryViewEntry, RepositoryViewState};
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CurrentRepoEntry {
pub current_hash: [u8; 32],
pub current_hash_hex: String,
pub repository_source: String,
pub object_type: Option<String>,
pub state: RepositoryViewState,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct CurrentRepoObject {
pub rsync_uri: String,
pub current_hash_hex: String,
pub repository_source: String,
pub object_type: Option<String>,
}
#[derive(Default, Debug)]
pub struct CurrentRepoIndex {
by_uri: HashMap<String, CurrentRepoEntry>,
}
/// Shared handle to the run-wide current repository index. Readers (phase 2
/// publication point staging, cache lookups, output snapshots) take
/// `.read()` and no longer exclude each other; writers (repo sync transports
/// applying repository view entries, run-state reset) take `.write()` and
/// stay exclusive.
pub type CurrentRepoIndexHandle = Arc<RwLock<CurrentRepoIndex>>;
impl CurrentRepoIndex {
pub fn new() -> Self {
Self::default()
}
pub fn shared() -> CurrentRepoIndexHandle {
Arc::new(RwLock::new(Self::new()))
}
pub fn get_by_uri(&self, rsync_uri: &str) -> Option<&CurrentRepoEntry> {
self.by_uri.get(rsync_uri)
}
pub fn list_scope_uris(&self, repository_source: &str) -> Vec<String> {
let mut out = self
.by_uri
.iter()
.filter(|&(_rsync_uri, entry)| entry.repository_source == repository_source)
.map(|(rsync_uri, _entry)| rsync_uri.clone())
.collect::<Vec<_>>();
out.sort();
out
}
pub fn active_uri_count(&self) -> usize {
self.by_uri.len()
}
pub fn scope_count(&self) -> usize {
self.by_uri
.values()
.map(|entry| entry.repository_source.as_str())
.collect::<HashSet<_>>()
.len()
}
pub fn snapshot_objects(&self) -> Vec<CurrentRepoObject> {
let mut out = self
.by_uri
.iter()
.map(|(rsync_uri, entry)| CurrentRepoObject {
rsync_uri: rsync_uri.clone(),
current_hash_hex: entry.current_hash_hex.clone(),
repository_source: entry.repository_source.clone(),
object_type: entry.object_type.clone(),
})
.collect::<Vec<_>>();
out.sort();
out
}
pub fn clear(&mut self) {
self.by_uri.clear();
}
pub fn apply_repository_view_entries(
&mut self,
entries: &[RepositoryViewEntry],
) -> Result<(), String> {
for entry in entries {
self.apply_repository_view_entry(entry)?;
}
Ok(())
}
fn apply_repository_view_entry(&mut self, entry: &RepositoryViewEntry) -> Result<(), String> {
entry.validate_internal().map_err(|e| e.to_string())?;
match entry.state {
RepositoryViewState::Present | RepositoryViewState::Replaced => {
let repository_source = entry.repository_source.clone().ok_or_else(|| {
format!(
"repository_view entry missing repository_source for current object {}",
entry.rsync_uri
)
})?;
let current_hash_hex = entry.current_hash.clone().ok_or_else(|| {
format!(
"repository_view entry missing current_hash for current object {}",
entry.rsync_uri
)
})?;
let current_hash = decode_sha256_hex_32(&current_hash_hex)?;
self.by_uri.insert(
entry.rsync_uri.clone(),
CurrentRepoEntry {
current_hash,
current_hash_hex: current_hash_hex.to_ascii_lowercase(),
repository_source,
object_type: entry.object_type.clone(),
state: entry.state,
},
);
}
RepositoryViewState::Withdrawn => {
self.by_uri.remove(&entry.rsync_uri);
}
}
Ok(())
}
}
fn decode_sha256_hex_32(value: &str) -> Result<[u8; 32], String> {
if value.len() != 64 || !value.as_bytes().iter().all(u8::is_ascii_hexdigit) {
return Err(format!("invalid sha256 hex: {value}"));
}
let mut out = [0u8; 32];
hex::decode_to_slice(value, &mut out).map_err(|e| format!("hex decode failed: {e}"))?;
Ok(out)
}
#[cfg(test)]
mod tests {
use super::CurrentRepoIndex;
use crate::repository::storage::{RepositoryViewEntry, RepositoryViewState};
fn present(source: &str, uri: &str, hash: &str) -> RepositoryViewEntry {
RepositoryViewEntry {
rsync_uri: uri.to_string(),
current_hash: Some(hash.to_string()),
repository_source: Some(source.to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
}
}
#[test]
fn current_repo_index_tracks_present_and_withdrawn_entries() {
let mut index = CurrentRepoIndex::new();
let uri = "rsync://example.test/repo/a.roa";
let source = "rsync://example.test/repo/";
let hash = &"11".repeat(32);
index
.apply_repository_view_entries(&[present(source, uri, hash)])
.expect("apply present");
let got = index.get_by_uri(uri).expect("current entry");
assert_eq!(got.current_hash_hex, hash.to_string());
assert_eq!(index.list_scope_uris(source), vec![uri.to_string()]);
index
.apply_repository_view_entries(&[RepositoryViewEntry {
rsync_uri: uri.to_string(),
current_hash: Some(hash.to_string()),
repository_source: Some(source.to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Withdrawn,
}])
.expect("apply withdrawn");
assert!(index.get_by_uri(uri).is_none());
assert!(index.list_scope_uris(source).is_empty());
}
#[test]
fn current_repo_index_moves_uri_between_scopes() {
let mut index = CurrentRepoIndex::new();
let uri = "rsync://example.test/repo/a.roa";
let old_scope = "rsync://example.test/repo/";
let new_scope = "https://rrdp.example.test/notification.xml";
index
.apply_repository_view_entries(&[present(old_scope, uri, &"22".repeat(32))])
.expect("apply old scope");
index
.apply_repository_view_entries(&[present(new_scope, uri, &"33".repeat(32))])
.expect("apply new scope");
assert!(index.list_scope_uris(old_scope).is_empty());
assert_eq!(index.list_scope_uris(new_scope), vec![uri.to_string()]);
assert_eq!(
index.get_by_uri(uri).expect("entry").current_hash_hex,
"33".repeat(32)
);
}
#[test]
fn current_repo_index_snapshot_objects_and_counts_are_sorted() {
let handle = CurrentRepoIndex::shared();
let mut index = handle.write().expect("write-lock index");
index
.apply_repository_view_entries(&[
present(
"rsync://example.test/repo-b/",
"rsync://example.test/repo-b/b.roa",
&"22".repeat(32),
),
present(
"rsync://example.test/repo-a/",
"rsync://example.test/repo-a/a.roa",
&"11".repeat(32),
),
])
.expect("apply present entries");
assert_eq!(index.active_uri_count(), 2);
assert_eq!(index.scope_count(), 2);
let snapshot = index.snapshot_objects();
assert_eq!(snapshot.len(), 2);
assert_eq!(snapshot[0].rsync_uri, "rsync://example.test/repo-a/a.roa");
assert_eq!(snapshot[1].rsync_uri, "rsync://example.test/repo-b/b.roa");
}
#[test]
fn current_repo_index_reports_missing_fields_and_invalid_hash() {
let mut index = CurrentRepoIndex::new();
let err = index
.apply_repository_view_entries(&[RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
current_hash: Some("11".repeat(32)),
repository_source: None,
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
}])
.expect_err("missing source should fail");
assert!(err.contains("missing repository_source"), "{err}");
let err = index
.apply_repository_view_entries(&[RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
current_hash: Some("not-a-valid-sha256".to_string()),
repository_source: Some("rsync://example.test/repo/".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
}])
.expect_err("invalid hash should fail");
assert!(err.contains("invalid"), "{err}");
index
.apply_repository_view_entries(&[RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/b.roa".to_string(),
current_hash: Some("22".repeat(32)),
repository_source: Some("rsync://example.test/repo/".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
}])
.expect("valid entry");
let got = index.get_by_uri("rsync://example.test/repo/b.roa").unwrap();
assert_eq!(got.current_hash_hex, "22".repeat(32));
}
#[test]
fn current_repo_index_withdraw_unknown_uri_is_noop() {
let mut index = CurrentRepoIndex::new();
index
.apply_repository_view_entries(&[RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/missing.roa".to_string(),
current_hash: None,
repository_source: Some("rsync://example.test/repo/".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Withdrawn,
}])
.expect("withdraw unknown should not fail");
assert_eq!(index.active_uri_count(), 0);
assert_eq!(index.scope_count(), 0);
}
}

View File

@@ -0,0 +1,234 @@
use std::sync::Arc;
use crate::repository::fetch::rsync::{RsyncFetchError, RsyncFetchResult, RsyncFetcher};
use crate::repository::fetch::rsync_system::{
RsyncScopePolicy, scoped_rsync_failure_dedup_key, scoped_rsync_fetch_uri,
};
use crate::repository::storage::{RepositoryViewState, RocksStore};
#[derive(Clone)]
pub struct CurrentRepositoryViewRsyncFetcher {
store: Arc<RocksStore>,
scope_policy: RsyncScopePolicy,
}
impl CurrentRepositoryViewRsyncFetcher {
pub fn new(store: Arc<RocksStore>, scope_policy: RsyncScopePolicy) -> Self {
Self {
store,
scope_policy,
}
}
}
impl RsyncFetcher for CurrentRepositoryViewRsyncFetcher {
fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult<Vec<(String, Vec<u8>)>> {
// The source run's live fetcher may have fetched a module root rather
// than the publication-point URI that triggered it. Read that same
// frozen-view prefix here so the materialized object set stays
// equivalent to the publication-point URI that triggered the fetch.
let base = scoped_rsync_fetch_uri(self.scope_policy, rsync_base_uri);
let entries = self
.store
.list_repository_view_entries_with_prefix(&base)
.map_err(|error| {
RsyncFetchError::Fetch(format!(
"list frozen repository view failed for {base}: {error}"
))
})?;
let mut objects = Vec::with_capacity(entries.len());
for entry in entries {
if !matches!(
entry.state,
RepositoryViewState::Present | RepositoryViewState::Replaced
) {
continue;
}
let bytes = self
.store
.load_current_object_bytes_by_uri(&entry.rsync_uri)
.map_err(|error| {
RsyncFetchError::Fetch(format!(
"load frozen repository object failed for {}: {error}",
entry.rsync_uri
))
})?
.ok_or_else(|| {
RsyncFetchError::Fetch(format!(
"frozen repository object missing for {}",
entry.rsync_uri
))
})?;
objects.push((entry.rsync_uri, bytes));
}
objects.sort_by(|left, right| left.0.cmp(&right.0));
if objects.is_empty() {
return Err(RsyncFetchError::Fetch(format!(
"frozen repository view contains no current objects under {base}"
)));
}
Ok(objects)
}
fn fetch_object(&self, rsync_uri: &str) -> RsyncFetchResult<Vec<u8>> {
self.store
.load_current_object_bytes_by_uri(rsync_uri)
.map_err(|error| {
RsyncFetchError::Fetch(format!(
"load frozen repository object failed for {rsync_uri}: {error}"
))
})?
.ok_or_else(|| {
RsyncFetchError::Fetch(format!("frozen repository object not found: {rsync_uri}"))
})
}
fn dedup_key(&self, rsync_base_uri: &str) -> String {
scoped_rsync_fetch_uri(self.scope_policy, rsync_base_uri)
}
fn failure_dedup_key(&self, rsync_base_uri: &str) -> Option<String> {
scoped_rsync_failure_dedup_key(self.scope_policy, rsync_base_uri)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::repository::storage::{RepositoryViewEntry, RocksStore};
#[test]
fn current_repository_fetcher_returns_sorted_present_objects() {
let dir = tempfile::tempdir().expect("tempdir");
let store = Arc::new(RocksStore::open(dir.path()).expect("open store"));
let entries = [
("rsync://example.test/repo/b.roa", b"b".as_slice()),
("rsync://example.test/repo/a.mft", b"a".as_slice()),
];
for (uri, bytes) in entries {
let hash = crate::repository::sync::store_projection::compute_sha256_hex(bytes);
store
.put_blob_bytes_batch(&[(hash.clone(), bytes.to_vec())])
.expect("put blob");
store
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: uri.to_string(),
repository_source: Some("fixture".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
current_hash: Some(hash),
})
.expect("put view");
}
let fetcher = CurrentRepositoryViewRsyncFetcher::new(store, RsyncScopePolicy::default());
let objects = fetcher
.fetch_objects("rsync://example.test/repo/")
.expect("fetch objects");
assert_eq!(
objects
.iter()
.map(|(uri, _)| uri.as_str())
.collect::<Vec<_>>(),
vec![
"rsync://example.test/repo/a.mft",
"rsync://example.test/repo/b.roa"
]
);
assert_eq!(
fetcher
.fetch_object("rsync://example.test/repo/a.mft")
.expect("fetch one"),
b"a"
);
assert!(
fetcher
.fetch_object("rsync://example.test/repo/missing.roa")
.is_err()
);
assert_eq!(
fetcher.dedup_key("rsync://example.test/repo"),
"rsync://example.test/repo/"
);
assert!(
fetcher
.fetch_objects("rsync://empty.example/repo/")
.unwrap_err()
.to_string()
.contains("no current objects")
);
}
#[test]
fn current_repository_fetcher_ignores_withdrawn_and_reports_missing_blob() {
let dir = tempfile::tempdir().expect("tempdir");
let store = Arc::new(RocksStore::open(dir.path()).expect("open store"));
store
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/withdrawn.roa".to_string(),
repository_source: Some("fixture".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Withdrawn,
current_hash: None,
})
.expect("put withdrawn view");
let missing_hash = "ab".repeat(32);
store
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/missing.roa".to_string(),
repository_source: Some("fixture".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
current_hash: Some(missing_hash),
})
.expect("put missing view");
let fetcher = CurrentRepositoryViewRsyncFetcher::new(store, RsyncScopePolicy::default());
let error = fetcher
.fetch_objects("rsync://example.test/repo/")
.unwrap_err()
.to_string();
assert!(error.contains("blob bytes missing"));
}
#[test]
fn current_repository_fetcher_uses_module_scope_and_failure_dedup() {
let dir = tempfile::tempdir().expect("tempdir");
let store = Arc::new(RocksStore::open(dir.path()).expect("open store"));
for (uri, bytes) in [
("rsync://example.test/repo/ca/a.mft", b"a".as_slice()),
("rsync://example.test/repo/other/b.roa", b"b".as_slice()),
] {
let hash = crate::repository::sync::store_projection::compute_sha256_hex(bytes);
store
.put_blob_bytes_batch(&[(hash.clone(), bytes.to_vec())])
.expect("put blob");
store
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: uri.to_string(),
repository_source: Some("fixture".to_string()),
object_type: Some("fixture".to_string()),
state: RepositoryViewState::Present,
current_hash: Some(hash),
})
.expect("put view");
}
let module_fetcher = CurrentRepositoryViewRsyncFetcher::new(
Arc::clone(&store),
RsyncScopePolicy::ModuleRoot,
);
let objects = module_fetcher
.fetch_objects("rsync://example.test/repo/ca/")
.expect("fetch module scope");
assert_eq!(objects.len(), 2);
assert_eq!(
module_fetcher.dedup_key("rsync://example.test/repo/ca/"),
"rsync://example.test/repo/"
);
let host_fetcher = CurrentRepositoryViewRsyncFetcher::new(store, RsyncScopePolicy::Host);
assert_eq!(
host_fetcher.failure_dedup_key("rsync://example.test/repo/ca/"),
Some("rsync://example.test/".to_string())
);
}
}

View File

@@ -0,0 +1,922 @@
use std::cell::RefCell;
use std::io::Write;
use std::time::Duration;
use reqwest::blocking::Client;
use reqwest::header::{HeaderMap, LOCATION};
use url::Url;
use crate::repository::sync::rrdp::{
Fetcher, RrdpError, RrdpFetchError, RrdpOrigin, RrdpResourceKind,
};
const MAX_RRDP_REDIRECTS: usize = 10;
thread_local! {
static HTTP_TIMEOUT_OVERRIDE: RefCell<Option<Duration>> = const { RefCell::new(None) };
}
pub fn with_scoped_http_timeout_override<R>(timeout: Duration, f: impl FnOnce() -> R) -> R {
HTTP_TIMEOUT_OVERRIDE.with(|cell| {
let previous = cell.replace(Some(timeout));
let result = f();
let _ = cell.replace(previous);
result
})
}
/// Default User-Agent sent with every outgoing HTTP request (RRDP
/// notification/snapshot/delta and TAL/TA downloads).
pub const DEFAULT_HTTP_USER_AGENT: &str = "panda-rpki/0.2";
/// Environment variable that overrides the HTTP User-Agent. Unset, empty, or
/// values containing characters that are illegal in a header value fall back
/// to [`DEFAULT_HTTP_USER_AGENT`].
pub const HTTP_USER_AGENT_ENV: &str = "RPKI_HTTP_USER_AGENT";
fn resolve_http_user_agent(env_value: Option<String>) -> String {
let Some(value) = env_value else {
return DEFAULT_HTTP_USER_AGENT.to_string();
};
let trimmed = value.trim();
let valid = !trimmed.is_empty() && trimmed.bytes().all(|b| (0x20..=0x7e).contains(&b));
if valid {
trimmed.to_string()
} else {
DEFAULT_HTTP_USER_AGENT.to_string()
}
}
#[derive(Clone, Debug)]
pub struct HttpFetcherConfig {
/// Connection-establishment timeout for HTTP requests.
pub connect_timeout: Duration,
/// Short timeout used for connection establishment and small metadata objects.
pub timeout: Duration,
/// Larger timeout used for RRDP snapshot / delta bodies.
pub large_body_timeout: Duration,
pub user_agent: String,
/// Extra PEM trust anchors for HTTPS transport tests or controlled endpoints.
pub extra_root_certificates_pem: Vec<Vec<u8>>,
}
impl Default for HttpFetcherConfig {
fn default() -> Self {
Self {
connect_timeout: Duration::from_secs(15),
timeout: Duration::from_secs(30),
large_body_timeout: Duration::from_secs(180),
user_agent: resolve_http_user_agent(std::env::var(HTTP_USER_AGENT_ENV).ok()),
extra_root_certificates_pem: Vec::new(),
}
}
}
/// Minimal blocking HTTP(S) fetcher for validation runs.
///
/// This is used for:
/// - downloading TAL / TA certificates (RFC 8630 §2)
/// - fetching RRDP notification/snapshot files (RFC 8182 §3.4)
#[derive(Clone, Debug)]
pub struct BlockingHttpFetcher {
short_client: Client,
large_body_client: Client,
retry_short_client: Client,
rrdp_short_client: Client,
rrdp_large_body_client: Client,
rrdp_retry_short_client: Client,
short_timeout: Duration,
large_body_timeout: Duration,
}
impl BlockingHttpFetcher {
pub fn new(config: HttpFetcherConfig) -> Result<Self, String> {
let short_timeout = config.timeout;
let large_body_timeout = std::cmp::max(config.large_body_timeout, config.timeout);
let connect_timeout = std::cmp::min(config.connect_timeout, config.timeout);
let short_client = Self::client_builder(
&config,
connect_timeout,
config.timeout,
config.user_agent.clone(),
)?
.build()
.map_err(|e| e.to_string())?;
let large_body_client = Self::client_builder(
&config,
connect_timeout,
large_body_timeout,
config.user_agent.clone(),
)?
.build()
.map_err(|e| e.to_string())?;
let retry_short_client = Self::client_builder(
&config,
Duration::from_secs(1),
Duration::from_secs(1),
config.user_agent.clone(),
)?
.build()
.map_err(|e| e.to_string())?;
let rrdp_short_client = Self::rrdp_client_builder(
&config,
connect_timeout,
config.timeout,
config.user_agent.clone(),
)?
.build()
.map_err(|e| e.to_string())?;
let rrdp_large_body_client = Self::rrdp_client_builder(
&config,
connect_timeout,
large_body_timeout,
config.user_agent.clone(),
)?
.build()
.map_err(|e| e.to_string())?;
let rrdp_retry_short_client = Self::rrdp_client_builder(
&config,
Duration::from_secs(1),
Duration::from_secs(1),
config.user_agent.clone(),
)?
.build()
.map_err(|e| e.to_string())?;
Ok(Self {
short_client,
large_body_client,
retry_short_client,
rrdp_short_client,
rrdp_large_body_client,
rrdp_retry_short_client,
short_timeout,
large_body_timeout,
})
}
fn client_builder(
config: &HttpFetcherConfig,
connect_timeout: Duration,
timeout: Duration,
user_agent: String,
) -> Result<reqwest::blocking::ClientBuilder, String> {
let mut builder = Client::builder()
.connect_timeout(connect_timeout)
.timeout(timeout)
.user_agent(user_agent);
for (idx, pem) in config.extra_root_certificates_pem.iter().enumerate() {
let certificate = reqwest::Certificate::from_pem(pem)
.map_err(|e| format!("parse HTTP root certificate #{idx} failed: {e}"))?;
builder = builder.add_root_certificate(certificate);
}
Ok(builder)
}
fn rrdp_client_builder(
config: &HttpFetcherConfig,
connect_timeout: Duration,
timeout: Duration,
user_agent: String,
) -> Result<reqwest::blocking::ClientBuilder, String> {
Ok(
Self::client_builder(config, connect_timeout, timeout, user_agent)?
.redirect(reqwest::redirect::Policy::none()),
)
}
pub fn fetch_bytes(&self, uri: &str) -> Result<Vec<u8>, String> {
let started = std::time::Instant::now();
let (client, timeout_profile, timeout_value) = self.client_for_uri(uri);
let resp = client.get(uri).send().map_err(|e| {
let msg = format!("http request failed: {e:?}");
crate::logging::progress::emit!(
"http_fetch_failed",
serde_json::json!({
"uri": uri,
"stage": "request",
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": started.elapsed().as_millis() as u64,
"error": msg,
}),
);
msg
})?;
let status = resp.status();
let headers = resp.headers().clone();
if !status.is_success() {
let msg = format!(
"http status {status}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}",
header_value(&headers, "content-type"),
header_value(&headers, "content-encoding"),
header_value(&headers, "content-length"),
header_value(&headers, "transfer-encoding"),
);
crate::logging::progress::emit!(
"http_fetch_failed",
serde_json::json!({
"uri": uri,
"stage": "status",
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": started.elapsed().as_millis() as u64,
"status": status.as_u16(),
"content_type": header_value_opt(&headers, "content-type"),
"content_encoding": header_value_opt(&headers, "content-encoding"),
"content_length": header_value_opt(&headers, "content-length"),
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
"error": msg,
}),
);
return Err(msg);
}
match resp.bytes() {
Ok(bytes) => {
let duration_ms = started.elapsed().as_millis() as u64;
if (duration_ms as f64) / 1000.0 >= crate::logging::progress::slow_threshold_secs()
{
crate::logging::progress::emit!(
"http_fetch_slow",
serde_json::json!({
"uri": uri,
"status": status.as_u16(),
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": duration_ms,
"bytes": bytes.len(),
"content_type": header_value_opt(&headers, "content-type"),
"content_encoding": header_value_opt(&headers, "content-encoding"),
"content_length": header_value_opt(&headers, "content-length"),
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
}),
);
}
Ok(bytes.to_vec())
}
Err(e) => {
let msg = format!(
"http read body failed: {e}; status={}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}",
status,
header_value(&headers, "content-type"),
header_value(&headers, "content-encoding"),
header_value(&headers, "content-length"),
header_value(&headers, "transfer-encoding"),
);
crate::logging::progress::emit!(
"http_fetch_failed",
serde_json::json!({
"uri": uri,
"stage": "read_body",
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": started.elapsed().as_millis() as u64,
"status": status.as_u16(),
"content_type": header_value_opt(&headers, "content-type"),
"content_encoding": header_value_opt(&headers, "content-encoding"),
"content_length": header_value_opt(&headers, "content-length"),
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
"error": msg,
}),
);
Err(msg)
}
}
}
fn client_for_uri(&self, uri: &str) -> (&Client, &'static str, Duration) {
if let Some(timeout) = HTTP_TIMEOUT_OVERRIDE.with(|cell| *cell.borrow()) {
return (&self.retry_short_client, "retry_short", timeout);
}
if uses_large_body_timeout(uri) {
(
&self.large_body_client,
"large_body",
self.large_body_timeout,
)
} else {
(&self.short_client, "short", self.short_timeout)
}
}
fn rrdp_client_for_uri(&self, uri: &str) -> (&Client, &'static str, Duration) {
if let Some(timeout) = HTTP_TIMEOUT_OVERRIDE.with(|cell| *cell.borrow()) {
return (&self.rrdp_retry_short_client, "retry_short", timeout);
}
if uses_large_body_timeout(uri) {
(
&self.rrdp_large_body_client,
"large_body",
self.large_body_timeout,
)
} else {
(&self.rrdp_short_client, "short", self.short_timeout)
}
}
fn rrdp_response(
&self,
resource: RrdpResourceKind,
uri: &str,
notification_origin: &RrdpOrigin,
) -> Result<(reqwest::blocking::Response, &'static str, Duration), RrdpFetchError> {
let mut current = Url::parse(uri).map_err(|e| RrdpError::InvalidRrdpUri {
resource,
detail: e.to_string(),
})?;
let current_origin = RrdpOrigin::from_url(&current, resource)?;
if &current_origin != notification_origin {
return Err(RrdpError::CrossOriginReference {
resource,
notification_origin: notification_origin.to_string(),
resource_origin: current_origin.to_string(),
}
.into());
}
for redirects in 0..=MAX_RRDP_REDIRECTS {
let (client, profile, timeout) = self.rrdp_client_for_uri(current.as_str());
let response = client
.get(current.clone())
.send()
.map_err(|e| RrdpFetchError::Fetch(format!("http request failed: {e:?}")))?;
if !response.status().is_redirection() {
return Ok((response, profile, timeout));
}
if redirects == MAX_RRDP_REDIRECTS {
return Err(RrdpError::RedirectLimitExceeded {
resource,
max: MAX_RRDP_REDIRECTS,
}
.into());
}
let location = response
.headers()
.get(LOCATION)
.ok_or(RrdpError::RedirectLocationMissing { resource })?
.to_str()
.map_err(|_| RrdpError::InvalidRedirectLocation {
resource,
location: "<non-UTF-8 header>".to_string(),
})?;
let next = current
.join(location)
.map_err(|_| RrdpError::InvalidRedirectLocation {
resource,
location: location.to_string(),
})?;
let next_origin = RrdpOrigin::from_url(&next, resource)?;
crate::logging::progress::emit!(
"rrdp_redirect_checked",
serde_json::json!({
"resource": resource.to_string(),
"from": current.as_str(),
"to": next.as_str(),
"notification_origin": notification_origin.to_string(),
"same_origin": &next_origin == notification_origin,
"rfc": "RFC 9674 §3.2",
}),
);
if &next_origin != notification_origin {
crate::logging::progress::emit!(
"rrdp_cross_origin_rejected",
serde_json::json!({
"resource": resource.to_string(),
"notification_origin": notification_origin.to_string(),
"resource_origin": next_origin.to_string(),
"reason": "redirect",
"rfc": "RFC 9674 §3.2",
}),
);
return Err(RrdpError::CrossOriginRedirect {
resource,
notification_origin: notification_origin.to_string(),
redirect_origin: next_origin.to_string(),
}
.into());
}
current = next;
}
unreachable!("redirect loop either returns or reaches its limit")
}
fn rrdp_fetch_bytes(
&self,
resource: RrdpResourceKind,
uri: &str,
notification_origin: &RrdpOrigin,
) -> Result<Vec<u8>, RrdpFetchError> {
let (response, _profile, _timeout) =
self.rrdp_response(resource, uri, notification_origin)?;
let status = response.status();
if !status.is_success() {
return Err(RrdpFetchError::Fetch(format!("http status {status}")));
}
response
.bytes()
.map(|b| b.to_vec())
.map_err(|e| RrdpFetchError::Fetch(format!("http read body failed: {e}")))
}
fn rrdp_fetch_to_writer(
&self,
resource: RrdpResourceKind,
uri: &str,
notification_origin: &RrdpOrigin,
out: &mut dyn Write,
) -> Result<u64, RrdpFetchError> {
let (mut response, _profile, _timeout) =
self.rrdp_response(resource, uri, notification_origin)?;
let status = response.status();
if !status.is_success() {
return Err(RrdpFetchError::Fetch(format!("http status {status}")));
}
response
.copy_to(out)
.map_err(|e| RrdpFetchError::Fetch(format!("http stream body failed: {e}")))
}
}
impl Fetcher for BlockingHttpFetcher {
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
self.fetch_bytes(uri)
}
fn fetch_to_writer(&self, uri: &str, out: &mut dyn Write) -> Result<u64, String> {
let started = std::time::Instant::now();
let (client, timeout_profile, timeout_value) = self.client_for_uri(uri);
let resp = client.get(uri).send().map_err(|e| {
let msg = format!("http request failed: {e:?}");
crate::logging::progress::emit!(
"http_fetch_failed",
serde_json::json!({
"uri": uri,
"stage": "request",
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": started.elapsed().as_millis() as u64,
"error": msg,
}),
);
msg
})?;
let status = resp.status();
let headers = resp.headers().clone();
if !status.is_success() {
let msg = format!(
"http status {status}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}",
header_value(&headers, "content-type"),
header_value(&headers, "content-encoding"),
header_value(&headers, "content-length"),
header_value(&headers, "transfer-encoding"),
);
crate::logging::progress::emit!(
"http_fetch_failed",
serde_json::json!({
"uri": uri,
"stage": "status",
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": started.elapsed().as_millis() as u64,
"status": status.as_u16(),
"content_type": header_value_opt(&headers, "content-type"),
"content_encoding": header_value_opt(&headers, "content-encoding"),
"content_length": header_value_opt(&headers, "content-length"),
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
"error": msg,
}),
);
return Err(msg);
}
let mut resp = resp;
match resp.copy_to(out) {
Ok(bytes) => {
let duration_ms = started.elapsed().as_millis() as u64;
if (duration_ms as f64) / 1000.0 >= crate::logging::progress::slow_threshold_secs()
{
crate::logging::progress::emit!(
"http_fetch_slow",
serde_json::json!({
"uri": uri,
"status": status.as_u16(),
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": duration_ms,
"bytes": bytes,
"content_type": header_value_opt(&headers, "content-type"),
"content_encoding": header_value_opt(&headers, "content-encoding"),
"content_length": header_value_opt(&headers, "content-length"),
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
}),
);
}
Ok(bytes)
}
Err(e) => {
let msg = format!(
"http stream body failed: {e}; status={}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}",
status,
header_value(&headers, "content-type"),
header_value(&headers, "content-encoding"),
header_value(&headers, "content-length"),
header_value(&headers, "transfer-encoding"),
);
crate::logging::progress::emit!(
"http_fetch_failed",
serde_json::json!({
"uri": uri,
"stage": "stream_body",
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": started.elapsed().as_millis() as u64,
"status": status.as_u16(),
"content_type": header_value_opt(&headers, "content-type"),
"content_encoding": header_value_opt(&headers, "content-encoding"),
"content_length": header_value_opt(&headers, "content-length"),
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
"error": msg,
}),
);
Err(msg)
}
}
}
fn fetch_rrdp(
&self,
resource: RrdpResourceKind,
uri: &str,
notification_origin: &RrdpOrigin,
) -> Result<Vec<u8>, RrdpFetchError> {
self.rrdp_fetch_bytes(resource, uri, notification_origin)
}
fn fetch_rrdp_to_writer(
&self,
resource: RrdpResourceKind,
uri: &str,
notification_origin: &RrdpOrigin,
out: &mut dyn Write,
) -> Result<u64, RrdpFetchError> {
self.rrdp_fetch_to_writer(resource, uri, notification_origin, out)
}
}
fn header_value(headers: &HeaderMap, name: &str) -> String {
header_value_opt(headers, name).unwrap_or_else(|| "<none>".to_string())
}
fn header_value_opt(headers: &HeaderMap, name: &str) -> Option<String> {
headers
.get(name)
.and_then(|v| v.to_str().ok())
.map(|v| v.to_string())
}
fn uses_large_body_timeout(uri: &str) -> bool {
uri.starts_with("https://") && uri.ends_with(".xml") && !uri.ends_with("notification.xml")
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::{Read, Write};
use std::net::TcpListener;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::thread;
use std::time::Duration as StdDuration;
fn spawn_one_shot_http_server(status_line: &'static str, body: &'static [u8]) -> String {
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
let addr = listener.local_addr().expect("addr");
thread::spawn(move || {
let (mut stream, _) = listener.accept().expect("accept");
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf);
let hdr = format!(
"{status_line}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
stream.write_all(hdr.as_bytes()).expect("write hdr");
stream.write_all(body).expect("write body");
});
format!("http://{}/", addr)
}
#[test]
fn fetch_bytes_returns_body_on_success() {
let url = spawn_one_shot_http_server("HTTP/1.1 200 OK", b"hello");
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
timeout: Duration::from_secs(2),
..HttpFetcherConfig::default()
})
.expect("http");
let got = http.fetch_bytes(&url).expect("fetch");
assert_eq!(got, b"hello");
}
#[test]
fn fetch_bytes_rejects_non_success_status() {
let url = spawn_one_shot_http_server("HTTP/1.1 404 Not Found", b"");
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
timeout: Duration::from_secs(2),
..HttpFetcherConfig::default()
})
.expect("http");
let err = http.fetch_bytes(&url).unwrap_err();
assert!(err.contains("http status"), "{err}");
}
#[test]
fn fetch_bytes_times_out_on_idle_body_read() {
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
let addr = listener.local_addr().expect("addr");
thread::spawn(move || {
let (mut stream, _) = listener.accept().expect("accept");
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf);
stream
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nh")
.expect("write partial body");
std::thread::sleep(StdDuration::from_secs(2));
let _ = stream.write_all(b"ello");
});
let url = format!("http://{}/", addr);
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
timeout: Duration::from_secs(1),
..HttpFetcherConfig::default()
})
.expect("http");
let err = http.fetch_bytes(&url).unwrap_err();
assert!(err.contains("http read body failed"), "{err}");
}
#[test]
fn fetch_to_writer_streams_body_on_success() {
let url = spawn_one_shot_http_server("HTTP/1.1 200 OK", b"writer-body");
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
timeout: Duration::from_secs(2),
..HttpFetcherConfig::default()
})
.expect("http");
let mut out = Vec::new();
let bytes = http.fetch_to_writer(&url, &mut out).expect("stream");
assert_eq!(bytes, 11);
assert_eq!(out, b"writer-body");
}
#[test]
fn fetch_to_writer_rejects_non_success_status() {
let url = spawn_one_shot_http_server("HTTP/1.1 500 Internal Server Error", b"boom");
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
timeout: Duration::from_secs(2),
..HttpFetcherConfig::default()
})
.expect("http");
let mut out = Vec::new();
let err = http.fetch_to_writer(&url, &mut out).unwrap_err();
assert!(err.contains("http status"), "{err}");
assert!(out.is_empty());
}
#[test]
fn fetch_to_writer_times_out_on_idle_stream_read() {
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
let addr = listener.local_addr().expect("addr");
thread::spawn(move || {
let (mut stream, _) = listener.accept().expect("accept");
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf);
stream
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nh")
.expect("write partial body");
std::thread::sleep(StdDuration::from_secs(2));
let _ = stream.write_all(b"ello");
});
let url = format!("http://{}/", addr);
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
timeout: Duration::from_secs(1),
..HttpFetcherConfig::default()
})
.expect("http");
let mut out = Vec::new();
let err = http.fetch_to_writer(&url, &mut out).unwrap_err();
assert!(err.contains("http stream body failed"), "{err}");
}
#[test]
fn uses_large_body_timeout_selects_rrdp_snapshot_and_delta_not_notification() {
assert!(!uses_large_body_timeout(
"https://rrdp.example.test/notification.xml"
));
assert!(uses_large_body_timeout(
"https://rrdp.example.test/session/123/snapshot.xml"
));
assert!(uses_large_body_timeout(
"https://rrdp.example.test/session/123/delta-42.xml"
));
assert!(!uses_large_body_timeout(
"https://tal.example.test/example.tal"
));
}
#[test]
fn client_for_uri_selects_expected_timeout_profile() {
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
timeout: Duration::from_secs(3),
large_body_timeout: Duration::from_secs(9),
..HttpFetcherConfig::default()
})
.expect("http");
let (_, profile_short, timeout_short) =
http.client_for_uri("https://example.test/root.tal");
assert_eq!(profile_short, "short");
assert_eq!(timeout_short, Duration::from_secs(3));
let (_, profile_large, timeout_large) =
http.client_for_uri("https://rrdp.example.test/session/1/snapshot.xml");
assert_eq!(profile_large, "large_body");
assert_eq!(timeout_large, Duration::from_secs(9));
}
#[test]
fn resolve_http_user_agent_falls_back_to_default() {
assert_eq!(resolve_http_user_agent(None), DEFAULT_HTTP_USER_AGENT);
assert_eq!(
resolve_http_user_agent(Some(String::new())),
DEFAULT_HTTP_USER_AGENT
);
assert_eq!(
resolve_http_user_agent(Some(" ".to_string())),
DEFAULT_HTTP_USER_AGENT
);
// Control characters are illegal in header values; fall back instead
// of letting reqwest fail to build the client.
assert_eq!(
resolve_http_user_agent(Some("bad\u{1}ua".to_string())),
DEFAULT_HTTP_USER_AGENT
);
assert_eq!(
resolve_http_user_agent(Some("non-ascii-ua-中".to_string())),
DEFAULT_HTTP_USER_AGENT
);
}
#[test]
fn resolve_http_user_agent_accepts_custom_value() {
assert_eq!(
resolve_http_user_agent(Some("panda-rpki/9.9 (test)".to_string())),
"panda-rpki/9.9 (test)"
);
// Surrounding whitespace is trimmed.
assert_eq!(
resolve_http_user_agent(Some(" custom/1.0 ".to_string())),
"custom/1.0"
);
}
fn spawn_user_agent_capture_server() -> (String, std::sync::mpsc::Receiver<String>) {
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
let addr = listener.local_addr().expect("addr");
let (tx, rx) = std::sync::mpsc::channel();
thread::spawn(move || {
let (mut stream, _) = listener.accept().expect("accept");
let mut buf = [0u8; 4096];
let read = stream.read(&mut buf).expect("read request");
let request = String::from_utf8_lossy(&buf[..read]).to_string();
let body = b"ok";
let hdr = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
stream.write_all(hdr.as_bytes()).expect("write hdr");
stream.write_all(body).expect("write body");
let _ = tx.send(request);
});
(format!("http://{}/", addr), rx)
}
#[test]
fn fetch_bytes_sends_configured_user_agent_header() {
let (url, rx) = spawn_user_agent_capture_server();
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
timeout: Duration::from_secs(2),
user_agent: "panda-rpki/0.2".to_string(),
..HttpFetcherConfig::default()
})
.expect("http");
http.fetch_bytes(&url).expect("fetch");
let request = rx.recv().expect("request captured");
assert!(
request
.to_ascii_lowercase()
.contains("user-agent: panda-rpki/0.2"),
"{request}"
);
}
#[test]
fn fetch_bytes_sends_custom_user_agent_header() {
let (url, rx) = spawn_user_agent_capture_server();
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
timeout: Duration::from_secs(2),
user_agent: "custom-marker/7.7".to_string(),
..HttpFetcherConfig::default()
})
.expect("http");
http.fetch_bytes(&url).expect("fetch");
let request = rx.recv().expect("request captured");
assert!(
request
.to_ascii_lowercase()
.contains("user-agent: custom-marker/7.7"),
"{request}"
);
}
#[test]
fn rrdp_same_origin_redirect_is_followed_manually() {
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
let addr = listener.local_addr().expect("addr");
thread::spawn(move || {
for response in [
b"HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
.as_slice(),
b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok".as_slice(),
] {
let (mut stream, _) = listener.accept().expect("accept");
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf);
stream.write_all(response).expect("response");
}
});
let uri = format!("http://{addr}/start");
let origin = RrdpOrigin::parse(&uri, RrdpResourceKind::Notification).expect("origin");
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
timeout: Duration::from_secs(2),
..HttpFetcherConfig::default()
})
.expect("http");
let body = http
.fetch_rrdp(RrdpResourceKind::Notification, &uri, &origin)
.expect("same-origin redirect accepted");
assert_eq!(body, b"ok");
}
#[test]
fn rrdp_cross_origin_redirect_is_rejected_before_target_request() {
let target = TcpListener::bind(("127.0.0.1", 0)).expect("bind target");
target.set_nonblocking(true).expect("nonblocking target");
let target_addr = target.local_addr().expect("target addr");
let target_hits = Arc::new(AtomicUsize::new(0));
let target_hits_worker = Arc::clone(&target_hits);
let target_thread = thread::spawn(move || {
let deadline = std::time::Instant::now() + StdDuration::from_millis(400);
while std::time::Instant::now() < deadline {
match target.accept() {
Ok((mut stream, _)) => {
target_hits_worker.fetch_add(1, Ordering::SeqCst);
let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n");
}
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
std::thread::sleep(StdDuration::from_millis(5));
}
Err(e) => panic!("target accept: {e}"),
}
}
});
let source = TcpListener::bind(("127.0.0.1", 0)).expect("bind source");
let source_addr = source.local_addr().expect("source addr");
thread::spawn(move || {
let (mut stream, _) = source.accept().expect("accept source");
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf);
let response = format!(
"HTTP/1.1 302 Found\r\nLocation: http://{target_addr}/foreign\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
);
stream.write_all(response.as_bytes()).expect("redirect");
});
let uri = format!("http://{source_addr}/start");
let origin = RrdpOrigin::parse(&uri, RrdpResourceKind::Notification).expect("origin");
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
timeout: Duration::from_secs(2),
..HttpFetcherConfig::default()
})
.expect("http");
let err = http
.fetch_rrdp(RrdpResourceKind::Notification, &uri, &origin)
.unwrap_err();
assert!(matches!(
err,
RrdpFetchError::Rrdp(RrdpError::CrossOriginRedirect { .. })
));
target_thread.join().expect("target thread");
assert_eq!(target_hits.load(Ordering::SeqCst), 0);
}
}

View File

@@ -0,0 +1,4 @@
pub mod current_repository;
pub mod http;
pub mod rsync;
pub mod rsync_system;

View File

@@ -0,0 +1,214 @@
use std::path::{Path, PathBuf};
#[derive(Debug, thiserror::Error)]
pub enum RsyncFetchError {
#[error("rsync fetch error: {0}")]
Fetch(String),
}
pub type RsyncFetchResult<T> = Result<T, RsyncFetchError>;
pub fn normalize_rsync_base_uri(s: &str) -> String {
if s.ends_with('/') {
s.to_string()
} else {
format!("{s}/")
}
}
/// Fetch repository objects from a publication point.
///
/// v1: this is intentionally abstract so unit tests can use a mock, and later we can
/// back it by calling the system `rsync` binary (RFC 6481 §5; RFC 8182 §3.4.5).
pub trait RsyncFetcher: Send + Sync {
/// Return a list of objects as `(rsync_uri, bytes)` pairs.
fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult<Vec<(String, Vec<u8>)>>;
/// Fetch one object by exact rsync URI.
///
/// The default implementation fetches the parent directory and filters the exact
/// object. Live fetchers should override this to avoid widening one-object TAL
/// bootstrap fetches into whole publication point or module synchronizations.
fn fetch_object(&self, rsync_uri: &str) -> RsyncFetchResult<Vec<u8>> {
let base = parent_rsync_uri(rsync_uri).map_err(RsyncFetchError::Fetch)?;
self.fetch_objects(&base)?
.into_iter()
.find(|(uri, _)| uri == rsync_uri)
.map(|(_, bytes)| bytes)
.ok_or_else(|| RsyncFetchError::Fetch(format!("rsync object not found: {rsync_uri}")))
}
/// Stream fetched objects to a visitor without requiring callers to materialize the
/// full result vector in memory.
fn visit_objects(
&self,
rsync_base_uri: &str,
visitor: &mut dyn FnMut(String, Vec<u8>) -> Result<(), String>,
) -> RsyncFetchResult<(usize, u64)> {
let objects = self.fetch_objects(rsync_base_uri)?;
let mut count = 0usize;
let mut bytes_total = 0u64;
for (uri, bytes) in objects {
bytes_total += bytes.len() as u64;
count += 1;
visitor(uri, bytes).map_err(RsyncFetchError::Fetch)?;
}
Ok((count, bytes_total))
}
/// Return the deduplication key used by orchestration layers.
///
/// By default this is the normalized publication point base URI. Fetchers that
/// intentionally widen their fetch scope (for example to a full rsync module)
/// should override this so callers can safely deduplicate at the same scope.
fn dedup_key(&self, rsync_base_uri: &str) -> String {
normalize_rsync_base_uri(rsync_base_uri)
}
/// Return an optional failure-only deduplication key.
///
/// This key is only used to short-circuit repeated failed rsync fallbacks. It must
/// not be used to reuse successful fetch results unless it is identical to
/// `dedup_key`, because a successful fetch for one publication point does not imply
/// that another publication point under the same host has been fetched.
fn failure_dedup_key(&self, _rsync_base_uri: &str) -> Option<String> {
None
}
}
fn parent_rsync_uri(rsync_uri: &str) -> Result<String, String> {
let parsed = url::Url::parse(rsync_uri).map_err(|e| e.to_string())?;
if parsed.scheme() != "rsync" {
return Err(format!("not an rsync URI: {rsync_uri}"));
}
let host = parsed
.host_str()
.ok_or_else(|| format!("missing host in rsync URI: {rsync_uri}"))?;
let segments = parsed
.path_segments()
.ok_or_else(|| format!("missing path in rsync URI: {rsync_uri}"))?
.collect::<Vec<_>>();
if segments.is_empty() || segments.last().copied().unwrap_or_default().is_empty() {
return Err(format!(
"rsync URI must reference a file object: {rsync_uri}"
));
}
let parent_segments = &segments[..segments.len() - 1];
// Preserve an explicitly supplied port. Dropping it changes the URI
// identity and prevents the default one-object implementation from
// matching objects returned by a fetcher (notably for test/private
// publication points that specify :873 explicitly).
let authority = match parsed.port() {
Some(port) => format!("{host}:{port}"),
None => host.to_string(),
};
let mut parent = format!("rsync://{authority}/");
if !parent_segments.is_empty() {
parent.push_str(&parent_segments.join("/"));
parent.push('/');
}
Ok(parent)
}
/// A simple "rsync" implementation backed by a local directory.
///
/// This is primarily meant for offline tests and fixtures. The key generation mimics rsync URIs:
/// `rsync_base_uri` + relative path (with `/` separators).
#[derive(Clone, Debug)]
pub struct LocalDirRsyncFetcher {
pub root_dir: PathBuf,
}
impl LocalDirRsyncFetcher {
pub fn new(root_dir: impl Into<PathBuf>) -> Self {
Self {
root_dir: root_dir.into(),
}
}
}
impl RsyncFetcher for LocalDirRsyncFetcher {
fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult<Vec<(String, Vec<u8>)>> {
let base = normalize_rsync_base_uri(rsync_base_uri);
let mut out = Vec::new();
walk_dir_collect(&self.root_dir, &self.root_dir, &base, &mut out)
.map_err(RsyncFetchError::Fetch)?;
Ok(out)
}
}
fn walk_dir_collect(
root: &Path,
current: &Path,
rsync_base_uri: &str,
out: &mut Vec<(String, Vec<u8>)>,
) -> Result<(), String> {
let rd = std::fs::read_dir(current).map_err(|e| e.to_string())?;
for entry in rd {
let entry = entry.map_err(|e| e.to_string())?;
let path = entry.path();
let meta = entry.metadata().map_err(|e| e.to_string())?;
if meta.is_dir() {
walk_dir_collect(root, &path, rsync_base_uri, out)?;
continue;
}
if !meta.is_file() {
continue;
}
let rel = path
.strip_prefix(root)
.map_err(|e| e.to_string())?
.to_string_lossy()
.replace('\\', "/");
let uri = format!("{rsync_base_uri}{rel}");
let bytes = std::fs::read(&path).map_err(|e| e.to_string())?;
out.push((uri, bytes));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn local_dir_rsync_fetcher_collects_files_and_normalizes_base_uri() {
let tmp = tempfile::tempdir().expect("tempdir");
std::fs::create_dir_all(tmp.path().join("nested")).expect("mkdir");
std::fs::write(tmp.path().join("a.mft"), b"a").expect("write");
std::fs::write(tmp.path().join("nested").join("b.roa"), b"b").expect("write");
let f = LocalDirRsyncFetcher::new(tmp.path());
let mut objects = f
.fetch_objects("rsync://example.net/repo")
.expect("fetch_objects");
objects.sort_by(|(a, _), (b, _)| a.cmp(b));
assert_eq!(objects.len(), 2);
assert_eq!(objects[0].0, "rsync://example.net/repo/a.mft");
assert_eq!(objects[0].1, b"a");
assert_eq!(objects[1].0, "rsync://example.net/repo/nested/b.roa");
assert_eq!(objects[1].1, b"b");
}
#[test]
fn local_dir_rsync_fetcher_reports_read_dir_errors() {
let tmp = tempfile::tempdir().expect("tempdir");
let missing = tmp.path().join("missing");
let f = LocalDirRsyncFetcher::new(missing);
let err = f.fetch_objects("rsync://example.net/repo").unwrap_err();
match err {
RsyncFetchError::Fetch(msg) => assert!(!msg.is_empty()),
}
}
#[test]
fn default_dedup_key_is_normalized_base_uri() {
let tmp = tempfile::tempdir().expect("tempdir");
let fetcher = LocalDirRsyncFetcher::new(tmp.path());
assert_eq!(
fetcher.dedup_key("rsync://example.net/repo"),
"rsync://example.net/repo/"
);
}
}

View File

@@ -0,0 +1,927 @@
use std::cell::RefCell;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::process::Stdio;
use std::thread;
use std::time::Duration;
use std::time::Instant;
use sha2::Digest;
use uuid::Uuid;
use crate::repository::fetch::rsync::{
RsyncFetchError, RsyncFetchResult, RsyncFetcher, normalize_rsync_base_uri,
};
#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "kebab-case")]
#[derive(Default)]
pub enum RsyncScopePolicy {
Host,
PublicationPoint,
#[default]
ModuleRoot,
}
impl RsyncScopePolicy {
pub fn parse_cli_value(value: &str) -> Result<Self, String> {
match value {
"host" => Ok(Self::Host),
"publication-point" => Ok(Self::PublicationPoint),
"module-root" => Ok(Self::ModuleRoot),
_ => Err(format!(
"invalid --rsync-scope: {value}; expected host, publication-point, or module-root"
)),
}
}
}
#[derive(Clone, Debug)]
pub struct SystemRsyncConfig {
pub rsync_bin: PathBuf,
pub connect_timeout: Duration,
pub timeout: Duration,
pub extra_args: Vec<String>,
/// Optional root directory for persistent rsync mirrors.
///
/// When set, callers may choose to sync into stable subdirectories under this
/// root (instead of a temporary directory) to benefit from rsync's incremental
/// behavior across runs.
///
/// Note: actual mirror behavior is implemented separately from config wiring.
pub mirror_root: Option<PathBuf>,
pub scope_policy: RsyncScopePolicy,
}
impl Default for SystemRsyncConfig {
fn default() -> Self {
Self {
rsync_bin: PathBuf::from("rsync"),
connect_timeout: Duration::from_secs(15),
timeout: Duration::from_secs(30),
extra_args: Vec::new(),
mirror_root: None,
scope_policy: RsyncScopePolicy::default(),
}
}
}
/// A `RsyncFetcher` implementation backed by the system `rsync` binary.
///
/// This is intended for live synchronization runs. For unit tests and local fixtures,
/// prefer `LocalDirRsyncFetcher`.
#[derive(Clone, Debug)]
pub struct SystemRsyncFetcher {
config: SystemRsyncConfig,
}
thread_local! {
static RSYNC_TIMEOUT_OVERRIDE: RefCell<Option<Duration>> = const { RefCell::new(None) };
static RSYNC_FAIL_FAST_PROFILE: RefCell<Option<RsyncFailFastProfile>> = const { RefCell::new(None) };
}
pub fn with_scoped_rsync_timeout_override<R>(timeout: Duration, f: impl FnOnce() -> R) -> R {
RSYNC_TIMEOUT_OVERRIDE.with(|cell| {
let previous = cell.replace(Some(timeout));
let result = f();
let _ = cell.replace(previous);
result
})
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct RsyncFailFastProfile {
pub initial_wall_clock_timeout: Duration,
pub max_wall_clock_timeout: Duration,
pub max_attempts: usize,
}
pub fn with_scoped_rsync_fail_fast_profile<R>(
profile: RsyncFailFastProfile,
f: impl FnOnce() -> R,
) -> R {
RSYNC_FAIL_FAST_PROFILE.with(|cell| {
let previous = cell.replace(Some(profile));
let result = f();
let _ = cell.replace(previous);
result
})
}
impl SystemRsyncFetcher {
pub fn new(config: SystemRsyncConfig) -> Self {
Self { config }
}
fn mirror_dst_dir(&self, normalized_rsync_base_uri: &str) -> Result<Option<PathBuf>, String> {
let Some(root) = self.config.mirror_root.as_ref() else {
return Ok(None);
};
std::fs::create_dir_all(root)
.map_err(|e| format!("create rsync mirror root failed: {}: {e}", root.display()))?;
let hash = hex::encode(sha2::Sha256::digest(normalized_rsync_base_uri.as_bytes()));
let dir = root.join(hash);
std::fs::create_dir_all(&dir).map_err(|e| {
format!(
"create rsync mirror directory failed: {}: {e}",
dir.display()
)
})?;
Ok(Some(dir))
}
fn run_rsync(&self, src: &str, dst: &Path) -> Result<(), String> {
let fail_fast = RSYNC_FAIL_FAST_PROFILE.with(|cell| *cell.borrow());
if let Some(profile) = fail_fast {
return self.run_rsync_fail_fast(src, dst, profile);
}
self.run_rsync_once(src, dst, None, false)
}
fn run_rsync_once(
&self,
src: &str,
dst: &Path,
wall_clock_timeout: Option<Duration>,
keep_partial: bool,
) -> Result<(), String> {
// `--timeout` is I/O timeout in seconds (applies to network reads/writes).
let timeout =
RSYNC_TIMEOUT_OVERRIDE.with(|cell| cell.borrow().unwrap_or(self.config.timeout));
let connect_timeout_secs = self.config.connect_timeout.as_secs().max(1).to_string();
let timeout_secs = timeout.as_secs().max(1).to_string();
let is_remote_rsync = src.starts_with("rsync://");
let mut cmd = Command::new(&self.config.rsync_bin);
cmd.arg("-rt")
.arg("--delete")
.arg("--timeout")
.arg(timeout_secs)
.args(&self.config.extra_args);
if is_remote_rsync {
cmd.arg("--contimeout").arg(connect_timeout_secs);
}
if keep_partial {
cmd.arg("--partial");
}
cmd.arg(src)
.arg(dst)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
let mut child = cmd
.spawn()
.map_err(|e| format!("rsync spawn failed: {e}"))?;
if let Some(limit) = wall_clock_timeout {
let started = Instant::now();
loop {
match child
.try_wait()
.map_err(|e| format!("rsync wait failed: {e}"))?
{
Some(_status) => {
let out = child
.wait_with_output()
.map_err(|e| format!("rsync wait_with_output failed: {e}"))?;
if out.status.success() {
return Ok(());
}
let stderr = String::from_utf8_lossy(&out.stderr);
let stdout = String::from_utf8_lossy(&out.stdout);
return Err(format!(
"rsync failed: status={} stdout={} stderr={}",
out.status,
stdout.trim(),
stderr.trim()
));
}
None => {
if started.elapsed() >= limit {
let _ = child.kill();
let out = child
.wait_with_output()
.map_err(|e| format!("rsync wait_with_output failed: {e}"))?;
let stderr = String::from_utf8_lossy(&out.stderr);
let stdout = String::from_utf8_lossy(&out.stdout);
return Err(format!(
"rsync wall-clock timeout after {}s: stdout={} stderr={}",
limit.as_secs(),
stdout.trim(),
stderr.trim()
));
}
thread::sleep(Duration::from_millis(100));
}
}
}
}
let out = child
.wait_with_output()
.map_err(|e| format!("rsync wait_with_output failed: {e}"))?;
if out.status.success() {
return Ok(());
}
let stderr = String::from_utf8_lossy(&out.stderr);
let stdout = String::from_utf8_lossy(&out.stdout);
Err(format!(
"rsync failed: status={} stdout={} stderr={}",
out.status,
stdout.trim(),
stderr.trim()
))
}
fn run_rsync_fail_fast(
&self,
src: &str,
dst: &Path,
profile: RsyncFailFastProfile,
) -> Result<(), String> {
let mut attempt = 0usize;
let mut timeout = profile.initial_wall_clock_timeout;
let mut previous_progress = (0usize, 0u64);
let mut zero_progress_attempts = 0usize;
let max_timeout = std::cmp::max(
profile.max_wall_clock_timeout,
profile.initial_wall_clock_timeout,
);
loop {
attempt += 1;
match self.run_rsync_once(src, dst, Some(timeout), true) {
Ok(()) => return Ok(()),
Err(err) => {
if is_hard_fail_rsync_error(&err) {
return Err(format!(
"rsync fail-fast hard-fail on attempt {}: {}",
attempt, err
));
}
if !err.contains("wall-clock timeout") {
return Err(err);
}
let progress = dir_progress(dst)
.map_err(|e| format!("rsync fail-fast progress stat failed: {e}"))?;
if progress == (0, 0) {
zero_progress_attempts += 1;
if zero_progress_attempts >= 2 || attempt >= profile.max_attempts {
return Err(format!(
"rsync fail-fast gave up after {} attempts with no progress: {}",
attempt, err
));
}
} else if progress == previous_progress {
return Err(format!(
"rsync fail-fast gave up after {} attempts with no additional progress: {}",
attempt, err
));
} else {
previous_progress = progress;
}
if attempt >= profile.max_attempts {
return Err(format!(
"rsync fail-fast exhausted {} attempts: {}",
profile.max_attempts, err
));
}
timeout = std::cmp::min(timeout.saturating_mul(2), max_timeout);
}
}
}
}
fn scope_fetch_uri(&self, rsync_base_uri: &str) -> String {
scoped_rsync_fetch_uri(self.config.scope_policy, rsync_base_uri)
}
}
impl RsyncFetcher for SystemRsyncFetcher {
fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult<Vec<(String, Vec<u8>)>> {
let mut out = Vec::new();
self.visit_objects(rsync_base_uri, &mut |uri, bytes| {
out.push((uri, bytes));
Ok(())
})?;
Ok(out)
}
fn fetch_object(&self, rsync_uri: &str) -> RsyncFetchResult<Vec<u8>> {
let parsed =
url::Url::parse(rsync_uri).map_err(|e| RsyncFetchError::Fetch(e.to_string()))?;
if parsed.scheme() != "rsync" {
return Err(RsyncFetchError::Fetch(format!(
"not an rsync URI: {rsync_uri}"
)));
}
let file_name = parsed
.path_segments()
.and_then(|mut segments| segments.rfind(|segment| !segment.is_empty()))
.ok_or_else(|| {
RsyncFetchError::Fetch(format!(
"rsync URI must reference a file object: {rsync_uri}"
))
})?;
let tmp = TempDir::new().map_err(RsyncFetchError::Fetch)?;
self.run_rsync(rsync_uri, tmp.path())
.map_err(RsyncFetchError::Fetch)?;
let object_path = tmp.path().join(file_name);
std::fs::read(&object_path).map_err(|e| {
RsyncFetchError::Fetch(format!(
"read fetched rsync object failed: {}: {e}",
object_path.display()
))
})
}
fn visit_objects(
&self,
rsync_base_uri: &str,
visitor: &mut dyn FnMut(String, Vec<u8>) -> Result<(), String>,
) -> RsyncFetchResult<(usize, u64)> {
let base = self.scope_fetch_uri(rsync_base_uri);
let mut count = 0usize;
let mut bytes_total = 0u64;
let mut wrapped = |uri: String, bytes: Vec<u8>| -> Result<(), String> {
bytes_total += bytes.len() as u64;
count += 1;
visitor(uri, bytes)
};
if let Some(dst) = self
.mirror_dst_dir(&base)
.map_err(|e| RsyncFetchError::Fetch(e.to_string()))?
{
self.run_rsync(&base, &dst)
.map_err(RsyncFetchError::Fetch)?;
walk_dir_visit(&dst, &dst, &base, &mut wrapped).map_err(RsyncFetchError::Fetch)?;
return Ok((count, bytes_total));
}
let tmp = TempDir::new().map_err(|e| RsyncFetchError::Fetch(e.to_string()))?;
self.run_rsync(&base, tmp.path())
.map_err(RsyncFetchError::Fetch)?;
walk_dir_visit(tmp.path(), tmp.path(), &base, &mut wrapped)
.map_err(RsyncFetchError::Fetch)?;
Ok((count, bytes_total))
}
fn dedup_key(&self, rsync_base_uri: &str) -> String {
self.scope_fetch_uri(rsync_base_uri)
}
fn failure_dedup_key(&self, rsync_base_uri: &str) -> Option<String> {
scoped_rsync_failure_dedup_key(self.config.scope_policy, rsync_base_uri)
}
}
/// Return the exact successful-fetch scope used by the live rsync fetcher.
///
/// The request scope is shared by live and materialized repository paths so their
/// request keys and object projection scope remain equivalent.
pub fn scoped_rsync_fetch_uri(scope_policy: RsyncScopePolicy, rsync_base_uri: &str) -> String {
match scope_policy {
RsyncScopePolicy::Host | RsyncScopePolicy::PublicationPoint => {
normalize_rsync_base_uri(rsync_base_uri)
}
RsyncScopePolicy::ModuleRoot => rsync_module_root_uri(rsync_base_uri)
.unwrap_or_else(|| normalize_rsync_base_uri(rsync_base_uri)),
}
}
/// Return the live fetcher's failure-deduplication key for the configured
/// scope. This is part of the transport request identity, even though host
/// scope deliberately does not widen the successful fetch URI.
pub fn scoped_rsync_failure_dedup_key(
scope_policy: RsyncScopePolicy,
rsync_base_uri: &str,
) -> Option<String> {
match scope_policy {
RsyncScopePolicy::Host => rsync_host_scope_uri(rsync_base_uri),
RsyncScopePolicy::PublicationPoint | RsyncScopePolicy::ModuleRoot => None,
}
}
fn rsync_host_scope_uri(rsync_base_uri: &str) -> Option<String> {
let parsed = url::Url::parse(rsync_base_uri).ok()?;
if parsed.scheme() != "rsync" {
return None;
}
Some(format!("rsync://{}/", parsed.host_str()?))
}
struct TempDir {
path: PathBuf,
}
impl TempDir {
fn new() -> Result<Self, String> {
let mut p = std::env::temp_dir();
p.push(format!("rpki-system-rsync-{}", Uuid::new_v4()));
std::fs::create_dir_all(&p).map_err(|e| e.to_string())?;
Ok(Self { path: p })
}
fn path(&self) -> &Path {
&self.path
}
}
impl Drop for TempDir {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.path);
}
}
fn rsync_module_root_uri(s: &str) -> Option<String> {
let normalized = normalize_rsync_base_uri(s);
let rest = normalized.strip_prefix("rsync://")?;
let (authority, path) = rest.split_once('/')?;
let mut segments: Vec<&str> = path
.split('/')
.filter(|segment| !segment.is_empty())
.collect();
if segments.is_empty() {
return None;
}
let module = segments.remove(0);
Some(format!("rsync://{authority}/{module}/"))
}
fn dir_progress(root: &Path) -> Result<(usize, u64), String> {
if !root.exists() {
return Ok((0, 0));
}
let mut files = 0usize;
let mut bytes = 0u64;
let mut stack = vec![root.to_path_buf()];
while let Some(path) = stack.pop() {
let rd = std::fs::read_dir(&path).map_err(|e| e.to_string())?;
for entry in rd {
let entry = entry.map_err(|e| e.to_string())?;
let path = entry.path();
let meta = entry.metadata().map_err(|e| e.to_string())?;
if meta.is_dir() {
stack.push(path);
} else if meta.is_file() {
files += 1;
bytes += meta.len();
}
}
}
Ok((files, bytes))
}
fn is_hard_fail_rsync_error(msg: &str) -> bool {
let lower = msg.to_ascii_lowercase();
lower.contains("no route to host")
|| lower.contains("network is unreachable")
|| lower.contains("connection refused")
|| lower.contains("name or service not known")
}
fn walk_dir_visit(
root: &Path,
current: &Path,
rsync_base_uri: &str,
visitor: &mut dyn FnMut(String, Vec<u8>) -> Result<(), String>,
) -> Result<(), String> {
let rd = std::fs::read_dir(current).map_err(|e| e.to_string())?;
for entry in rd {
let entry = entry.map_err(|e| e.to_string())?;
let path = entry.path();
let meta = entry.metadata().map_err(|e| e.to_string())?;
if meta.is_dir() {
walk_dir_visit(root, &path, rsync_base_uri, visitor)?;
continue;
}
if !meta.is_file() {
continue;
}
let rel = path
.strip_prefix(root)
.map_err(|e| e.to_string())?
.to_string_lossy()
.replace('\\', "/");
let uri = format!("{rsync_base_uri}{rel}");
let bytes = std::fs::read(&path).map_err(|e| e.to_string())?;
visitor(uri, bytes)?;
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn normalize_rsync_base_uri_appends_slash_when_missing() {
assert_eq!(
normalize_rsync_base_uri("rsync://example.net/repo"),
"rsync://example.net/repo/".to_string()
);
assert_eq!(
normalize_rsync_base_uri("rsync://example.net/repo/"),
"rsync://example.net/repo/".to_string()
);
}
#[test]
fn walk_dir_collect_collects_files_and_normalizes_backslashes_in_uri() {
let temp = tempfile::tempdir().expect("tempdir");
let root = temp.path();
std::fs::create_dir_all(root.join("sub")).expect("mkdir");
std::fs::write(root.join("sub").join("a.cer"), b"x").expect("write");
std::fs::write(root.join("b\\c.mft"), b"y").expect("write backslash file");
let mut out: Vec<(String, Vec<u8>)> = Vec::new();
walk_dir_visit(
root,
root,
"rsync://example.net/repo/",
&mut |uri, bytes| {
out.push((uri, bytes));
Ok(())
},
)
.expect("walk");
out.sort_by(|a, b| a.0.cmp(&b.0));
assert_eq!(out.len(), 2);
assert_eq!(out[0].0, "rsync://example.net/repo/b/c.mft");
assert_eq!(out[0].1, b"y");
assert_eq!(out[1].0, "rsync://example.net/repo/sub/a.cer");
assert_eq!(out[1].1, b"x");
}
#[test]
fn rsync_module_root_uri_returns_host_and_module_only() {
assert_eq!(
rsync_module_root_uri("rsync://example.net/repo/ta/ca/publication-point/"),
Some("rsync://example.net/repo/".to_string())
);
assert_eq!(
rsync_module_root_uri("rsync://example.net/repo/ta/"),
Some("rsync://example.net/repo/".to_string())
);
assert_eq!(
rsync_module_root_uri("rsync://example.net/repo/"),
Some("rsync://example.net/repo/".to_string())
);
assert_eq!(rsync_module_root_uri("https://example.net/repo"), None);
}
#[test]
fn rsync_host_scope_uri_returns_host_only() {
assert_eq!(
rsync_host_scope_uri("rsync://example.net/repo/ta/ca/publication-point/"),
Some("rsync://example.net/".to_string())
);
assert_eq!(rsync_host_scope_uri("https://example.net/repo"), None);
}
#[test]
fn system_rsync_dedup_key_uses_module_root_by_default() {
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig::default());
assert_eq!(
fetcher.dedup_key("rsync://example.net/repo/ta/ca/publication-point/"),
"rsync://example.net/repo/"
);
assert_eq!(
fetcher.failure_dedup_key("rsync://example.net/repo/ta/ca/publication-point/"),
None
);
}
#[test]
fn system_rsync_host_scope_does_not_widen_success_fetch_scope() {
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
scope_policy: RsyncScopePolicy::Host,
..SystemRsyncConfig::default()
});
assert_eq!(
fetcher.dedup_key("rsync://example.net/repo/ta/ca/publication-point/"),
"rsync://example.net/repo/ta/ca/publication-point/"
);
assert_eq!(
fetcher.scope_fetch_uri("rsync://example.net/repo/ta/ca/publication-point/"),
"rsync://example.net/repo/ta/ca/publication-point/"
);
assert_eq!(
fetcher.failure_dedup_key("rsync://example.net/repo/ta/ca/publication-point/"),
Some("rsync://example.net/".to_string())
);
}
#[test]
fn system_rsync_dedup_key_uses_publication_point_scope_when_configured() {
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
scope_policy: RsyncScopePolicy::PublicationPoint,
..SystemRsyncConfig::default()
});
assert_eq!(
fetcher.dedup_key("rsync://example.net/repo/ta/ca/publication-point/"),
"rsync://example.net/repo/ta/ca/publication-point/"
);
}
#[test]
fn system_rsync_dedup_key_uses_module_root_when_configured() {
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
scope_policy: RsyncScopePolicy::ModuleRoot,
..SystemRsyncConfig::default()
});
assert_eq!(
fetcher.dedup_key("rsync://example.net/repo/ta/ca/publication-point/"),
"rsync://example.net/repo/"
);
}
#[test]
fn system_rsync_fetcher_reports_spawn_and_exit_errors() {
let dst = tempfile::tempdir().expect("tempdir");
// 1) Spawn error.
let f = SystemRsyncFetcher::new(SystemRsyncConfig {
rsync_bin: PathBuf::from("/this/does/not/exist/rsync"),
connect_timeout: Duration::from_secs(1),
timeout: Duration::from_secs(1),
extra_args: Vec::new(),
mirror_root: None,
scope_policy: RsyncScopePolicy::default(),
});
let e = f
.run_rsync("rsync://example.net/repo/", dst.path())
.expect_err("spawn must fail");
assert!(e.contains("rsync spawn failed:"), "{e}");
// 2) Non-zero exit status.
let f = SystemRsyncFetcher::new(SystemRsyncConfig {
rsync_bin: PathBuf::from("false"),
connect_timeout: Duration::from_secs(1),
timeout: Duration::from_secs(1),
extra_args: Vec::new(),
mirror_root: None,
scope_policy: RsyncScopePolicy::default(),
});
let e = f
.run_rsync("rsync://example.net/repo/", dst.path())
.expect_err("false must fail");
assert!(e.contains("rsync failed:"), "{e}");
assert!(e.contains("status="), "{e}");
}
#[test]
fn mirror_dst_dir_reports_root_creation_error() {
let temp = tempfile::tempdir().expect("tempdir");
let root_file = temp.path().join("mirror-root-file");
std::fs::write(&root_file, b"not a directory").expect("write root file");
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
rsync_bin: PathBuf::from("rsync"),
connect_timeout: Duration::from_secs(1),
timeout: Duration::from_secs(1),
extra_args: Vec::new(),
mirror_root: Some(root_file.clone()),
scope_policy: RsyncScopePolicy::default(),
});
let err = fetcher
.mirror_dst_dir("rsync://example.net/repo/")
.expect_err("file mirror root must fail");
assert!(err.contains("create rsync mirror root failed"), "{err}");
assert!(err.contains(&root_file.display().to_string()), "{err}");
}
#[cfg(unix)]
#[test]
fn mirror_dst_dir_reports_directory_creation_error_inside_root() {
use std::os::unix::fs::PermissionsExt;
let temp = tempfile::tempdir().expect("tempdir");
let root = temp.path().join("mirror");
std::fs::create_dir_all(&root).expect("mkdir root");
let mut perms = std::fs::metadata(&root).expect("metadata").permissions();
perms.set_mode(0o555);
std::fs::set_permissions(&root, perms).expect("chmod root readonly");
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
rsync_bin: PathBuf::from("rsync"),
connect_timeout: Duration::from_secs(1),
timeout: Duration::from_secs(1),
extra_args: Vec::new(),
mirror_root: Some(root.clone()),
scope_policy: RsyncScopePolicy::default(),
});
let err = fetcher
.mirror_dst_dir("rsync://example.net/repo/")
.expect_err("readonly mirror root must fail");
assert!(
err.contains("create rsync mirror directory failed"),
"{err}"
);
let mut perms = std::fs::metadata(&root).expect("metadata").permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&root, perms).expect("restore perms");
}
#[cfg(unix)]
#[test]
fn walk_dir_collect_ignores_non_file_entries() {
use std::os::unix::net::UnixListener;
let temp = tempfile::tempdir().expect("tempdir");
let root = temp.path();
std::fs::write(root.join("a.cer"), b"x").expect("write file");
let socket_path = root.join("skip.sock");
let _listener = UnixListener::bind(&socket_path).expect("bind socket");
let mut out: Vec<(String, Vec<u8>)> = Vec::new();
walk_dir_visit(
root,
root,
"rsync://example.net/repo/",
&mut |uri, bytes| {
out.push((uri, bytes));
Ok(())
},
)
.expect("walk");
assert_eq!(out.len(), 1);
assert_eq!(out[0].0, "rsync://example.net/repo/a.cer");
}
#[cfg(unix)]
#[test]
fn rsync_fail_fast_retries_when_progress_is_made() {
use std::os::unix::fs::PermissionsExt;
let temp = tempfile::tempdir().expect("tempdir");
let script = temp.path().join("fake-rsync.sh");
let state = temp.path().join("state.txt");
std::fs::write(
&script,
format!(
"#!/usr/bin/env bash\nset -euo pipefail\nSTATE=\"{}\"\nDST=\"${{@: -1}}\"\nCOUNT=0\nif [[ -f \"$STATE\" ]]; then COUNT=$(cat \"$STATE\"); fi\nCOUNT=$((COUNT+1))\necho \"$COUNT\" > \"$STATE\"\nmkdir -p \"$DST\"\nif [[ \"$COUNT\" -eq 1 ]]; then\n echo first > \"$DST/part1\"\n sleep 2\nelse\n echo second > \"$DST/part2\"\nfi\n",
state.display()
),
)
.expect("write script");
let mut perms = std::fs::metadata(&script).unwrap().permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&script, perms).unwrap();
let dst = temp.path().join("dst");
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
rsync_bin: script,
connect_timeout: Duration::from_secs(15),
timeout: Duration::from_secs(60),
extra_args: Vec::new(),
mirror_root: None,
scope_policy: RsyncScopePolicy::default(),
});
fetcher
.run_rsync_fail_fast(
"rsync://example.net/repo/",
&dst,
RsyncFailFastProfile {
initial_wall_clock_timeout: Duration::from_secs(1),
max_wall_clock_timeout: Duration::from_secs(4),
max_attempts: 3,
},
)
.expect("eventual success");
assert!(dst.join("part1").exists());
assert!(dst.join("part2").exists());
}
#[cfg(unix)]
#[test]
fn rsync_fail_fast_gives_up_after_two_zero_progress_timeouts() {
use std::os::unix::fs::PermissionsExt;
let temp = tempfile::tempdir().expect("tempdir");
let script = temp.path().join("fake-rsync.sh");
std::fs::write(&script, "#!/usr/bin/env bash\nset -euo pipefail\nsleep 5\n")
.expect("write script");
let mut perms = std::fs::metadata(&script).unwrap().permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&script, perms).unwrap();
let dst = temp.path().join("dst");
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
rsync_bin: script,
connect_timeout: Duration::from_secs(15),
timeout: Duration::from_secs(60),
extra_args: Vec::new(),
mirror_root: None,
scope_policy: RsyncScopePolicy::default(),
});
let err = fetcher
.run_rsync_fail_fast(
"rsync://example.net/repo/",
&dst,
RsyncFailFastProfile {
initial_wall_clock_timeout: Duration::from_secs(1),
max_wall_clock_timeout: Duration::from_secs(2),
max_attempts: 4,
},
)
.expect_err("must fail");
assert!(err.contains("no progress"), "{err}");
}
#[cfg(unix)]
#[test]
fn rsync_fail_fast_hard_fail_stops_after_first_attempt() {
use std::os::unix::fs::PermissionsExt;
let temp = tempfile::tempdir().expect("tempdir");
let script = temp.path().join("fake-rsync.sh");
let state = temp.path().join("state.txt");
std::fs::write(
&script,
format!(
"#!/usr/bin/env bash\nset -euo pipefail\nSTATE=\"{}\"\nCOUNT=0\nif [[ -f \"$STATE\" ]]; then COUNT=$(cat \"$STATE\"); fi\nCOUNT=$((COUNT+1))\necho \"$COUNT\" > \"$STATE\"\necho 'rsync: [Receiver] failed to connect to host (1.2.3.4): Connection refused (111)' >&2\nexit 10\n",
state.display()
),
)
.expect("write script");
let mut perms = std::fs::metadata(&script).unwrap().permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&script, perms).unwrap();
let dst = temp.path().join("dst");
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
rsync_bin: script,
connect_timeout: Duration::from_secs(15),
timeout: Duration::from_secs(60),
extra_args: Vec::new(),
mirror_root: None,
scope_policy: RsyncScopePolicy::default(),
});
let err = fetcher
.run_rsync_fail_fast(
"rsync://example.net/repo/",
&dst,
RsyncFailFastProfile {
initial_wall_clock_timeout: Duration::from_secs(10),
max_wall_clock_timeout: Duration::from_secs(80),
max_attempts: 4,
},
)
.expect_err("must hard fail");
assert!(err.contains("hard-fail"), "{err}");
let count = std::fs::read_to_string(&state).unwrap();
assert_eq!(count.trim(), "1");
}
#[cfg(unix)]
#[test]
fn run_rsync_once_passes_contimeout_and_timeout_args() {
use std::os::unix::fs::PermissionsExt;
let temp = tempfile::tempdir().expect("tempdir");
let script = temp.path().join("capture-rsync.sh");
let args_file = temp.path().join("args.txt");
std::fs::write(
&script,
format!(
"#!/usr/bin/env bash\nset -euo pipefail\nprintf '%s\\n' \"$@\" > \"{}\"\nDST=\"${{@: -1}}\"\nmkdir -p \"$DST\"\n",
args_file.display()
),
)
.expect("write script");
let mut perms = std::fs::metadata(&script).unwrap().permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&script, perms).unwrap();
let dst = temp.path().join("dst");
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
rsync_bin: script,
connect_timeout: Duration::from_secs(15),
timeout: Duration::from_secs(30),
extra_args: Vec::new(),
mirror_root: None,
scope_policy: RsyncScopePolicy::default(),
});
fetcher
.run_rsync_once("rsync://example.net/repo/", &dst, None, false)
.expect("rsync");
let args = std::fs::read_to_string(&args_file).expect("read args");
assert!(args.contains("--contimeout\n15\n"), "{args}");
assert!(args.contains("--timeout\n30\n"), "{args}");
}
}

6
src/repository/mod.rs Normal file
View File

@@ -0,0 +1,6 @@
//! Repository transport, protocol state, and object storage.
pub mod blob_store;
pub mod current_repo_index;
pub mod fetch;
pub mod storage;
pub mod sync;

43
src/repository/storage.rs Normal file
View File

@@ -0,0 +1,43 @@
mod config;
mod keys;
mod memory;
mod pack;
use std::collections::HashSet;
use std::path::Path;
use rocksdb::checkpoint::Checkpoint;
use rocksdb::{ColumnFamily, DB, Direction, IteratorMode, Options, WriteBatch};
use serde::{Deserialize, Serialize};
use crate::repository::blob_store::{ExternalRawStoreDb, ExternalRepoBytesDb, RawObjectStore};
use config::*;
pub use config::{
ALL_COLUMN_FAMILY_NAMES, CF_MANIFEST_ANTI_ROLLBACK, CF_RAW_BY_HASH, CF_REPOSITORY_VIEW,
CF_RRDP_SOURCE, CF_RRDP_SOURCE_MEMBER, CF_RRDP_URI_OWNER, column_family_descriptors,
};
use keys::*;
pub(crate) use memory::memory_db_snapshot_for_column_families;
pub use memory::{
RocksDbColumnFamilyMemoryProperties, RocksDbMemoryDbSnapshot, RocksDbMemoryProperties,
RocksDbMemorySnapshot, RocksDbMemoryTotals,
};
use pack::compute_sha256_32;
pub use pack::{PackBytes, PackFile, PackTime};
pub struct RocksStore {
db: DB,
external_raw_store: Option<ExternalRawStoreDb>,
external_repo_bytes: Option<ExternalRepoBytesDb>,
}
include!("storage/models_core.rs");
include!("storage/models_summary.rs");
include!("storage/store_lifecycle.rs");
include!("storage/store_repository.rs");
include!("storage/store_manifest.rs");
include!("storage/store_transport_rrdp.rs");
include!("storage/verification.rs");
#[cfg(test)]
#[path = "storage/tests.rs"]
mod tests;

View File

@@ -0,0 +1,43 @@
use rocksdb::{ColumnFamilyDescriptor, DBCompressionType, Options};
pub const CF_REPOSITORY_VIEW: &str = "repository_view";
pub const CF_RAW_BY_HASH: &str = "raw_by_hash";
pub const CF_RAW_BLOB: &str = "raw_blob";
/// Persistent RFC 9286 manifest freshness metadata. This is protocol state,
/// not a validation-result cache.
pub const CF_MANIFEST_ANTI_ROLLBACK: &str = "manifest_anti_rollback";
pub const CF_RRDP_SOURCE: &str = "rrdp_source";
pub const CF_RRDP_SOURCE_MEMBER: &str = "rrdp_source_member";
pub const CF_RRDP_URI_OWNER: &str = "rrdp_uri_owner";
pub const ALL_COLUMN_FAMILY_NAMES: &[&str] = &[
CF_REPOSITORY_VIEW,
CF_RAW_BY_HASH,
CF_RAW_BLOB,
CF_MANIFEST_ANTI_ROLLBACK,
CF_RRDP_SOURCE,
CF_RRDP_SOURCE_MEMBER,
CF_RRDP_URI_OWNER,
];
pub(super) const REPOSITORY_VIEW_KEY_PREFIX: &str = "repo_view:";
pub(super) const RAW_BY_HASH_KEY_PREFIX: &str = "rawbyhash:";
pub(super) const RAW_BLOB_KEY_PREFIX: &str = "rawblob:";
pub(super) const MANIFEST_ANTI_ROLLBACK_KEY_PREFIX: &str = "manifest_anti_rollback:";
pub(super) const RRDP_SOURCE_KEY_PREFIX: &str = "rrdp_source:";
pub(super) const RRDP_SOURCE_MEMBER_KEY_PREFIX: &str = "rrdp_source_member:";
pub(super) const RRDP_URI_OWNER_KEY_PREFIX: &str = "rrdp_uri_owner:";
pub(super) fn configure_work_db_options(opts: &mut Options) {
opts.set_compression_type(DBCompressionType::Lz4);
}
pub fn column_family_descriptors() -> Vec<ColumnFamilyDescriptor> {
ALL_COLUMN_FAMILY_NAMES
.iter()
.map(|name| {
let mut opts = Options::default();
configure_work_db_options(&mut opts);
ColumnFamilyDescriptor::new(*name, opts)
})
.collect()
}

View File

@@ -0,0 +1,130 @@
use serde::{Serialize, de::DeserializeOwned};
use super::config::*;
use super::pack::PackTime;
use super::{StorageError, StorageResult};
pub(super) fn repository_view_key(rsync_uri: &str) -> String {
format!("{REPOSITORY_VIEW_KEY_PREFIX}{rsync_uri}")
}
pub(super) fn repository_view_prefix(rsync_uri_prefix: &str) -> String {
format!("{REPOSITORY_VIEW_KEY_PREFIX}{rsync_uri_prefix}")
}
pub(super) fn raw_by_hash_key(sha256_hex: &str) -> String {
format!("{RAW_BY_HASH_KEY_PREFIX}{sha256_hex}")
}
pub(super) fn raw_blob_key(sha256_hex: &str) -> String {
format!("{RAW_BLOB_KEY_PREFIX}{sha256_hex}")
}
pub(super) fn manifest_anti_rollback_key(manifest_rsync_uri: &str) -> String {
format!("{MANIFEST_ANTI_ROLLBACK_KEY_PREFIX}{manifest_rsync_uri}")
}
pub(super) fn rrdp_source_key(notify_uri: &str) -> String {
format!("{RRDP_SOURCE_KEY_PREFIX}{notify_uri}")
}
pub(super) fn rrdp_source_member_key(notify_uri: &str, rsync_uri: &str) -> String {
format!("{RRDP_SOURCE_MEMBER_KEY_PREFIX}{notify_uri}:{rsync_uri}")
}
pub(super) fn rrdp_source_member_prefix(notify_uri: &str) -> String {
format!("{RRDP_SOURCE_MEMBER_KEY_PREFIX}{notify_uri}:")
}
pub(super) fn rrdp_uri_owner_key(rsync_uri: &str) -> String {
format!("{RRDP_URI_OWNER_KEY_PREFIX}{rsync_uri}")
}
pub(super) fn encode_cbor<T: Serialize>(value: &T, entity: &'static str) -> StorageResult<Vec<u8>> {
serde_cbor::to_vec(value).map_err(|e| StorageError::Codec {
entity,
detail: e.to_string(),
})
}
pub(super) fn decode_cbor<T: DeserializeOwned>(
bytes: &[u8],
entity: &'static str,
) -> StorageResult<T> {
serde_cbor::from_slice(bytes).map_err(|e| StorageError::Codec {
entity,
detail: e.to_string(),
})
}
pub(super) fn validate_non_empty(field: &'static str, value: &str) -> StorageResult<()> {
if value.is_empty() {
return Err(StorageError::InvalidData {
entity: field,
detail: "must not be empty".to_string(),
});
}
Ok(())
}
pub(super) fn validate_sha256_hex(field: &'static str, value: &str) -> StorageResult<()> {
if value.len() != 64 || !value.as_bytes().iter().all(u8::is_ascii_hexdigit) {
return Err(StorageError::InvalidData {
entity: field,
detail: "must be a 64-character lowercase or uppercase SHA-256 hex string".to_string(),
});
}
Ok(())
}
pub(super) fn decode_sha256_hex_32(field: &'static str, value: &str) -> StorageResult<[u8; 32]> {
validate_sha256_hex(field, value)?;
let mut out = [0u8; 32];
hex::decode_to_slice(value, &mut out).map_err(|e| StorageError::InvalidData {
entity: field,
detail: format!("hex decode failed: {e}"),
})?;
Ok(out)
}
pub(super) fn validate_manifest_number_be(field: &'static str, value: &[u8]) -> StorageResult<()> {
if value.is_empty() {
return Err(StorageError::InvalidData {
entity: field,
detail: "must not be empty".to_string(),
});
}
if value.len() > 20 {
return Err(StorageError::InvalidData {
entity: field,
detail: "must be at most 20 octets".to_string(),
});
}
if value.len() > 1 && value[0] == 0 {
return Err(StorageError::InvalidData {
entity: field,
detail: "must be minimal big-endian without leading zeros".to_string(),
});
}
Ok(())
}
pub(super) fn validate_sha256_digest_bytes(field: &'static str, value: &[u8]) -> StorageResult<()> {
if value.len() != 32 {
return Err(StorageError::InvalidData {
entity: field,
detail: format!("must be 32 bytes, got {}", value.len()),
});
}
Ok(())
}
pub(super) fn parse_time(
field: &'static str,
value: &PackTime,
) -> StorageResult<time::OffsetDateTime> {
value.parse().map_err(|detail| StorageError::InvalidData {
entity: field,
detail,
})
}

View File

@@ -0,0 +1,191 @@
//! RocksDB and process-memory observation.
//!
//! This module owns the serialization-friendly memory snapshot model and the
//! mapping from RocksDB property names to that model. Storage mutation stays
//! in the parent module.
use rocksdb::DB;
use serde::Serialize;
const ROCKSDB_MEMORY_PROPERTY_NAMES: &[(&str, &str)] = &[
("cur_size_all_mem_tables", "rocksdb.cur-size-all-mem-tables"),
("size_all_mem_tables", "rocksdb.size-all-mem-tables"),
(
"estimate_table_readers_mem",
"rocksdb.estimate-table-readers-mem",
),
("block_cache_capacity", "rocksdb.block-cache-capacity"),
("block_cache_usage", "rocksdb.block-cache-usage"),
(
"block_cache_pinned_usage",
"rocksdb.block-cache-pinned-usage",
),
("num_snapshots", "rocksdb.num-snapshots"),
("background_errors", "rocksdb.background-errors"),
];
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
pub struct RocksDbMemoryProperties {
pub cur_size_all_mem_tables: Option<u64>,
pub size_all_mem_tables: Option<u64>,
pub estimate_table_readers_mem: Option<u64>,
pub block_cache_capacity: Option<u64>,
pub block_cache_usage: Option<u64>,
pub block_cache_pinned_usage: Option<u64>,
pub num_snapshots: Option<u64>,
pub background_errors: Option<u64>,
pub errors: Vec<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct RocksDbColumnFamilyMemoryProperties {
pub name: String,
pub properties: RocksDbMemoryProperties,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct RocksDbMemoryDbSnapshot {
pub label: String,
pub properties: RocksDbMemoryProperties,
pub column_families: Vec<RocksDbColumnFamilyMemoryProperties>,
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
pub struct RocksDbMemoryTotals {
pub cur_size_all_mem_tables: u64,
pub size_all_mem_tables: u64,
pub estimate_table_readers_mem: u64,
pub block_cache_capacity: u64,
pub block_cache_usage: u64,
pub block_cache_pinned_usage: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct RocksDbMemorySnapshot {
pub databases: Vec<RocksDbMemoryDbSnapshot>,
pub totals: RocksDbMemoryTotals,
}
impl RocksDbMemoryTotals {
pub(super) fn add_properties(&mut self, properties: &RocksDbMemoryProperties) {
self.cur_size_all_mem_tables += properties.cur_size_all_mem_tables.unwrap_or(0);
self.size_all_mem_tables += properties.size_all_mem_tables.unwrap_or(0);
self.estimate_table_readers_mem += properties.estimate_table_readers_mem.unwrap_or(0);
self.block_cache_capacity += properties.block_cache_capacity.unwrap_or(0);
self.block_cache_usage += properties.block_cache_usage.unwrap_or(0);
self.block_cache_pinned_usage += properties.block_cache_pinned_usage.unwrap_or(0);
}
}
fn set_memory_property(properties: &mut RocksDbMemoryProperties, name: &str, value: u64) {
match name {
"cur_size_all_mem_tables" => properties.cur_size_all_mem_tables = Some(value),
"size_all_mem_tables" => properties.size_all_mem_tables = Some(value),
"estimate_table_readers_mem" => properties.estimate_table_readers_mem = Some(value),
"block_cache_capacity" => properties.block_cache_capacity = Some(value),
"block_cache_usage" => properties.block_cache_usage = Some(value),
"block_cache_pinned_usage" => properties.block_cache_pinned_usage = Some(value),
"num_snapshots" => properties.num_snapshots = Some(value),
"background_errors" => properties.background_errors = Some(value),
_ => {}
}
}
fn parse_rocksdb_property_int(raw: Option<String>) -> Option<u64> {
raw.and_then(|value| value.trim().parse::<u64>().ok())
}
fn memory_properties_for_db(db: &DB) -> RocksDbMemoryProperties {
let mut properties = RocksDbMemoryProperties::default();
for (field_name, property_name) in ROCKSDB_MEMORY_PROPERTY_NAMES {
match db.property_value(*property_name) {
Ok(value) => {
if let Some(parsed) = parse_rocksdb_property_int(value) {
set_memory_property(&mut properties, field_name, parsed);
}
}
Err(err) => properties.errors.push(format!("{property_name}: {}", err)),
}
}
properties
}
fn memory_properties_for_cf(db: &DB, cf_name: &'static str) -> RocksDbColumnFamilyMemoryProperties {
let mut properties = RocksDbMemoryProperties::default();
let Some(cf) = db.cf_handle(cf_name) else {
properties
.errors
.push(format!("missing column family: {cf_name}"));
return RocksDbColumnFamilyMemoryProperties {
name: cf_name.to_string(),
properties,
};
};
for (field_name, property_name) in ROCKSDB_MEMORY_PROPERTY_NAMES {
match db.property_value_cf(cf, *property_name) {
Ok(value) => {
if let Some(parsed) = parse_rocksdb_property_int(value) {
set_memory_property(&mut properties, field_name, parsed);
}
}
Err(err) => properties.errors.push(format!("{property_name}: {}", err)),
}
}
RocksDbColumnFamilyMemoryProperties {
name: cf_name.to_string(),
properties,
}
}
pub(crate) fn memory_db_snapshot_for_column_families(
label: impl Into<String>,
db: &DB,
column_families: Option<&[&'static str]>,
) -> RocksDbMemoryDbSnapshot {
RocksDbMemoryDbSnapshot {
label: label.into(),
properties: memory_properties_for_db(db),
column_families: column_families
.map(|names| {
names
.iter()
.map(|name| memory_properties_for_cf(db, name))
.collect()
})
.unwrap_or_default(),
}
}
#[cfg(test)]
mod tests {
use super::{
RocksDbMemoryProperties, RocksDbMemoryTotals, parse_rocksdb_property_int,
set_memory_property,
};
#[test]
fn parses_rocksdb_integer_properties_without_panicking() {
assert_eq!(
parse_rocksdb_property_int(Some(" 42 ".to_string())),
Some(42)
);
assert_eq!(
parse_rocksdb_property_int(Some("not-a-number".to_string())),
None
);
assert_eq!(parse_rocksdb_property_int(None), None);
}
#[test]
fn aggregates_only_known_memory_properties() {
let mut properties = RocksDbMemoryProperties::default();
set_memory_property(&mut properties, "block_cache_usage", 11);
set_memory_property(&mut properties, "unknown", 99);
let mut totals = RocksDbMemoryTotals::default();
totals.add_properties(&properties);
assert_eq!(totals.block_cache_usage, 11);
assert_eq!(totals.block_cache_capacity, 0);
}
}

View File

@@ -0,0 +1,153 @@
// Repository view, raw-object records, and protocol rollback metadata.
#[derive(Debug, thiserror::Error)]
pub enum StorageError {
#[error("rocksdb error: {0}")]
RocksDb(String),
#[error("missing column family: {0}")]
MissingColumnFamily(&'static str),
#[error("cbor codec error for {entity}: {detail}")]
Codec { entity: &'static str, detail: String },
#[error("invalid {entity}: {detail}")]
InvalidData { entity: &'static str, detail: String },
}
pub type StorageResult<T> = Result<T, StorageError>;
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
pub struct RepositoryBlobVerificationSummary {
pub current_objects: u64,
pub bytes_verified: u64,
pub batches: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum RrdpDeltaOp {
Upsert { rsync_uri: String, bytes: Vec<u8> },
Delete { rsync_uri: String },
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RepositoryViewState {
Present,
Withdrawn,
Replaced,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RepositoryViewEntry {
pub rsync_uri: String,
pub current_hash: Option<String>,
pub repository_source: Option<String>,
pub object_type: Option<String>,
pub state: RepositoryViewState,
}
impl RepositoryViewEntry {
pub fn validate_internal(&self) -> StorageResult<()> {
validate_non_empty("repository_view.rsync_uri", &self.rsync_uri)?;
if let Some(source) = &self.repository_source {
validate_non_empty("repository_view.repository_source", source)?;
}
match self.state {
RepositoryViewState::Present | RepositoryViewState::Replaced => {
let hash = self.current_hash.as_deref().ok_or(StorageError::InvalidData {
entity: "repository_view",
detail: "current_hash is required when state is present or replaced".to_string(),
})?;
validate_sha256_hex("repository_view.current_hash", hash)?;
}
RepositoryViewState::Withdrawn => {
if let Some(hash) = &self.current_hash {
validate_sha256_hex("repository_view.current_hash", hash)?;
}
}
}
Ok(())
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RawByHashEntry {
pub sha256_hex: String,
pub bytes: Vec<u8>,
pub origin_uris: Vec<String>,
pub object_type: Option<String>,
pub encoding: Option<String>,
}
impl RawByHashEntry {
pub fn from_bytes(sha256_hex: impl Into<String>, bytes: Vec<u8>) -> Self {
Self { sha256_hex: sha256_hex.into(), bytes, origin_uris: Vec::new(), object_type: None, encoding: None }
}
pub fn validate_internal(&self) -> StorageResult<()> {
validate_sha256_hex("raw_by_hash.sha256_hex", &self.sha256_hex)?;
if self.bytes.is_empty() {
return Err(StorageError::InvalidData { entity: "raw_by_hash", detail: "bytes must not be empty".to_string() });
}
let computed = hex::encode(compute_sha256_32(&self.bytes));
if computed != self.sha256_hex.to_ascii_lowercase() {
return Err(StorageError::InvalidData { entity: "raw_by_hash", detail: "sha256_hex does not match bytes".to_string() });
}
let mut seen = HashSet::with_capacity(self.origin_uris.len());
for uri in &self.origin_uris {
validate_non_empty("raw_by_hash.origin_uris[]", uri)?;
if !seen.insert(uri.as_str()) {
return Err(StorageError::InvalidData { entity: "raw_by_hash", detail: format!("duplicate origin URI: {uri}") });
}
}
Ok(())
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CurrentObjectWithHash {
pub current_hash_hex: String,
pub current_hash: [u8; 32],
pub bytes: Vec<u8>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ValidatedManifestMeta {
pub validated_manifest_number: Vec<u8>,
pub validated_manifest_this_update: PackTime,
pub validated_manifest_next_update: PackTime,
}
impl ValidatedManifestMeta {
pub fn validate_internal(&self) -> StorageResult<()> {
validate_manifest_number_be("validated_manifest_meta.validated_manifest_number", &self.validated_manifest_number)?;
let this_update = parse_time("validated_manifest_meta.validated_manifest_this_update", &self.validated_manifest_this_update)?;
let next_update = parse_time("validated_manifest_meta.validated_manifest_next_update", &self.validated_manifest_next_update)?;
if next_update < this_update {
return Err(StorageError::InvalidData { entity: "validated_manifest_meta", detail: "validated_manifest_next_update must be >= validated_manifest_this_update".to_string() });
}
Ok(())
}
}
/// Persistent manifest freshness metadata used by RFC 9286 anti-rollback
/// checks. It stores no validated object output.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ManifestAntiRollbackMeta {
pub manifest_rsync_uri: String,
pub manifest_number_be: Vec<u8>,
pub manifest_this_update: PackTime,
pub manifest_sha256: Vec<u8>,
pub updated_at_validation_time: PackTime,
}
impl ManifestAntiRollbackMeta {
pub fn validate_internal(&self) -> StorageResult<()> {
validate_non_empty("manifest_anti_rollback.manifest_rsync_uri", &self.manifest_rsync_uri)?;
validate_manifest_number_be("manifest_anti_rollback.manifest_number_be", &self.manifest_number_be)?;
parse_time("manifest_anti_rollback.manifest_this_update", &self.manifest_this_update)?;
validate_sha256_digest_bytes("manifest_anti_rollback.manifest_sha256", &self.manifest_sha256)?;
parse_time("manifest_anti_rollback.updated_at_validation_time", &self.updated_at_validation_time)?;
Ok(())
}
}
pub use crate::ccr::projection::CcrManifestProjection;

View File

@@ -0,0 +1,117 @@
// RRDP protocol state records.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RrdpSourceSyncState {
Empty,
SnapshotOnly,
DeltaReady,
Error,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RrdpSourceRecord {
pub notify_uri: String,
pub last_session_id: Option<String>,
pub last_serial: Option<u64>,
pub first_seen_at: PackTime,
pub last_seen_at: PackTime,
pub last_sync_at: Option<PackTime>,
pub sync_state: RrdpSourceSyncState,
pub last_snapshot_uri: Option<String>,
pub last_snapshot_hash: Option<String>,
pub last_error: Option<String>,
}
impl RrdpSourceRecord {
pub fn validate_internal(&self) -> StorageResult<()> {
validate_non_empty("rrdp_source.notify_uri", &self.notify_uri)?;
if let Some(session_id) = &self.last_session_id {
validate_non_empty("rrdp_source.last_session_id", session_id)?;
}
parse_time("rrdp_source.first_seen_at", &self.first_seen_at)?;
parse_time("rrdp_source.last_seen_at", &self.last_seen_at)?;
if let Some(last_sync_at) = &self.last_sync_at {
parse_time("rrdp_source.last_sync_at", last_sync_at)?;
}
if let Some(last_snapshot_uri) = &self.last_snapshot_uri {
validate_non_empty("rrdp_source.last_snapshot_uri", last_snapshot_uri)?;
}
if let Some(last_snapshot_hash) = &self.last_snapshot_hash {
validate_sha256_hex("rrdp_source.last_snapshot_hash", last_snapshot_hash)?;
}
if let Some(last_error) = &self.last_error {
validate_non_empty("rrdp_source.last_error", last_error)?;
}
Ok(())
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RrdpSourceMemberRecord {
pub notify_uri: String,
pub rsync_uri: String,
pub current_hash: Option<String>,
pub object_type: Option<String>,
pub present: bool,
pub last_confirmed_session_id: String,
pub last_confirmed_serial: u64,
pub last_changed_at: PackTime,
}
impl RrdpSourceMemberRecord {
pub fn validate_internal(&self) -> StorageResult<()> {
validate_non_empty("rrdp_source_member.notify_uri", &self.notify_uri)?;
validate_non_empty("rrdp_source_member.rsync_uri", &self.rsync_uri)?;
validate_non_empty(
"rrdp_source_member.last_confirmed_session_id",
&self.last_confirmed_session_id,
)?;
if self.present {
let hash = self.current_hash.as_deref().ok_or(StorageError::InvalidData {
entity: "rrdp_source_member",
detail: "current_hash is required when present=true".to_string(),
})?;
validate_sha256_hex("rrdp_source_member.current_hash", hash)?;
} else if let Some(hash) = &self.current_hash {
validate_sha256_hex("rrdp_source_member.current_hash", hash)?;
}
parse_time("rrdp_source_member.last_changed_at", &self.last_changed_at)?;
Ok(())
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RrdpUriOwnerState {
Active,
Conflict,
Withdrawn,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RrdpUriOwnerRecord {
pub rsync_uri: String,
pub notify_uri: String,
pub current_hash: Option<String>,
pub last_confirmed_session_id: String,
pub last_confirmed_serial: u64,
pub last_changed_at: PackTime,
pub owner_state: RrdpUriOwnerState,
}
impl RrdpUriOwnerRecord {
pub fn validate_internal(&self) -> StorageResult<()> {
validate_non_empty("rrdp_uri_owner.rsync_uri", &self.rsync_uri)?;
validate_non_empty("rrdp_uri_owner.notify_uri", &self.notify_uri)?;
validate_non_empty(
"rrdp_uri_owner.last_confirmed_session_id",
&self.last_confirmed_session_id,
)?;
if let Some(hash) = &self.current_hash {
validate_sha256_hex("rrdp_uri_owner.current_hash", hash)?;
}
parse_time("rrdp_uri_owner.last_changed_at", &self.last_changed_at)?;
Ok(())
}
}

View File

@@ -0,0 +1,200 @@
use serde::{Deserialize, Serialize};
use sha2::Digest;
use crate::repository::blob_store::{ExternalRawStoreDb, ExternalRepoBytesDb, RawObjectStore};
#[derive(Clone, Debug)]
pub enum PackBytes {
Eager(std::sync::Arc<[u8]>),
LazyExternal {
sha256_hex: String,
store: std::sync::Arc<ExternalRawStoreDb>,
cache: std::sync::Arc<std::sync::OnceLock<std::sync::Arc<[u8]>>>,
},
LazyRepoBytes {
sha256_hex: String,
store: std::sync::Arc<ExternalRepoBytesDb>,
cache: std::sync::Arc<std::sync::OnceLock<std::sync::Arc<[u8]>>>,
},
}
impl PackBytes {
pub fn eager(bytes: Vec<u8>) -> Self {
Self::Eager(std::sync::Arc::from(bytes))
}
pub fn lazy_external(sha256_hex: String, store: std::sync::Arc<ExternalRawStoreDb>) -> Self {
Self::LazyExternal {
sha256_hex,
store,
cache: std::sync::Arc::new(std::sync::OnceLock::new()),
}
}
pub fn lazy_repo_bytes(sha256_hex: String, store: std::sync::Arc<ExternalRepoBytesDb>) -> Self {
Self::LazyRepoBytes {
sha256_hex,
store,
cache: std::sync::Arc::new(std::sync::OnceLock::new()),
}
}
pub fn as_slice(&self) -> Result<&[u8], String> {
match self {
Self::Eager(bytes) => Ok(bytes.as_ref()),
Self::LazyExternal {
sha256_hex,
store,
cache,
} => {
if cache.get().is_none() {
let bytes = store
.get_blob_bytes(sha256_hex)
.map_err(|e| e.to_string())?
.ok_or_else(|| format!("missing raw blob for sha256={sha256_hex}"))?;
let _ = cache.set(std::sync::Arc::from(bytes));
}
let bytes = cache
.get()
.ok_or_else(|| format!("missing raw blob cache for sha256={sha256_hex}"))?;
Ok(bytes.as_ref())
}
Self::LazyRepoBytes {
sha256_hex,
store,
cache,
} => {
if cache.get().is_none() {
let bytes = store
.get_blob_bytes(sha256_hex)
.map_err(|e| e.to_string())?
.ok_or_else(|| format!("missing repo bytes for sha256={sha256_hex}"))?;
let _ = cache.set(std::sync::Arc::from(bytes));
}
let bytes = cache
.get()
.ok_or_else(|| format!("missing repo bytes cache for sha256={sha256_hex}"))?;
Ok(bytes.as_ref())
}
}
}
pub fn to_vec(&self) -> Result<Vec<u8>, String> {
Ok(self.as_slice()?.to_vec())
}
}
impl PartialEq for PackBytes {
fn eq(&self, other: &Self) -> bool {
match (self.as_slice(), other.as_slice()) {
(Ok(a), Ok(b)) => a == b,
_ => false,
}
}
}
impl Eq for PackBytes {}
#[derive(Clone, Debug)]
pub struct PackFile {
pub rsync_uri: String,
pub bytes: PackBytes,
pub sha256: [u8; 32],
}
impl PackFile {
pub fn new(rsync_uri: impl Into<String>, bytes: PackBytes, sha256: [u8; 32]) -> Self {
Self {
rsync_uri: rsync_uri.into(),
bytes,
sha256,
}
}
pub fn from_bytes_with_sha256(
rsync_uri: impl Into<String>,
bytes: Vec<u8>,
sha256: [u8; 32],
) -> Self {
Self::new(rsync_uri, PackBytes::eager(bytes), sha256)
}
pub fn from_lazy_external_raw_store(
rsync_uri: impl Into<String>,
sha256_hex: String,
sha256: [u8; 32],
store: std::sync::Arc<ExternalRawStoreDb>,
) -> Self {
Self::new(
rsync_uri,
PackBytes::lazy_external(sha256_hex, store),
sha256,
)
}
pub fn from_lazy_repo_bytes(
rsync_uri: impl Into<String>,
sha256_hex: String,
sha256: [u8; 32],
store: std::sync::Arc<ExternalRepoBytesDb>,
) -> Self {
Self::new(
rsync_uri,
PackBytes::lazy_repo_bytes(sha256_hex, store),
sha256,
)
}
pub fn from_bytes_compute_sha256(rsync_uri: impl Into<String>, bytes: Vec<u8>) -> Self {
let sha256 = compute_sha256_32(&bytes);
Self::new(rsync_uri, PackBytes::eager(bytes), sha256)
}
pub fn bytes(&self) -> Result<&[u8], String> {
self.bytes.as_slice()
}
pub fn bytes_cloned(&self) -> Result<Vec<u8>, String> {
self.bytes.to_vec()
}
pub fn compute_sha256(&self) -> Result<[u8; 32], String> {
Ok(compute_sha256_32(self.bytes()?))
}
}
impl PartialEq for PackFile {
fn eq(&self, other: &Self) -> bool {
self.rsync_uri == other.rsync_uri
&& self.sha256 == other.sha256
&& self.bytes == other.bytes
}
}
impl Eq for PackFile {}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PackTime {
pub rfc3339_utc: String,
}
impl PackTime {
pub fn from_utc_offset_datetime(t: time::OffsetDateTime) -> Self {
use time::format_description::well_known::Rfc3339;
let utc = t.to_offset(time::UtcOffset::UTC);
let s = utc.format(&Rfc3339).expect("format RFC 3339 UTC time");
Self { rfc3339_utc: s }
}
pub fn parse(&self) -> Result<time::OffsetDateTime, String> {
use time::format_description::well_known::Rfc3339;
time::OffsetDateTime::parse(&self.rfc3339_utc, &Rfc3339).map_err(|e| e.to_string())
}
}
pub(super) fn compute_sha256_32(bytes: &[u8]) -> [u8; 32] {
let digest = sha2::Sha256::digest(bytes);
let mut out = [0u8; 32];
out.copy_from_slice(&digest);
out
}

View File

@@ -0,0 +1,158 @@
// RocksDB lifecycle and external repository stores.
impl RocksStore {
pub fn create_read_only_checkpoint(source: &Path, destination: &Path) -> StorageResult<()> {
reject_unsupported_column_families(source)?;
if destination.exists() {
return Err(StorageError::InvalidData {
entity: "work_db_checkpoint",
detail: format!(
"checkpoint destination already exists: {}",
destination.display()
),
});
}
if let Some(parent) = destination.parent() {
std::fs::create_dir_all(parent)
.map_err(|error| StorageError::RocksDb(error.to_string()))?;
}
let mut options = Options::default();
configure_work_db_options(&mut options);
let descriptors = column_family_descriptors();
let db = DB::open_cf_descriptors_read_only(
&options,
source,
descriptors,
false,
)
.map_err(|error| StorageError::RocksDb(error.to_string()))?;
let checkpoint =
Checkpoint::new(&db).map_err(|error| StorageError::RocksDb(error.to_string()))?;
checkpoint
.create_checkpoint(destination)
.map_err(|error| StorageError::RocksDb(error.to_string()))
}
pub fn open(path: &Path) -> StorageResult<Self> {
reject_unsupported_column_families(path)?;
let mut base_opts = Options::default();
base_opts.create_if_missing(true);
base_opts.create_missing_column_families(true);
configure_work_db_options(&mut base_opts);
let descriptors = column_family_descriptors();
let db = DB::open_cf_descriptors(
&base_opts,
path,
descriptors,
)
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(Self {
db,
external_raw_store: None,
external_repo_bytes: None,
})
}
pub fn open_with_external_raw_store(path: &Path, raw_store_path: &Path) -> StorageResult<Self> {
Self::open_with_external_stores(path, Some(raw_store_path), None)
}
pub fn open_with_external_repo_bytes(
path: &Path,
repo_bytes_path: &Path,
) -> StorageResult<Self> {
Self::open_with_external_stores(path, None, Some(repo_bytes_path))
}
pub fn open_with_external_repo_bytes_read_only(
path: &Path,
repo_bytes_path: &Path,
) -> StorageResult<Self> {
let mut store = Self::open(path)?;
store.external_repo_bytes = Some(ExternalRepoBytesDb::open_read_only(repo_bytes_path)?);
Ok(store)
}
pub fn open_with_external_stores(
path: &Path,
raw_store_path: Option<&Path>,
repo_bytes_path: Option<&Path>,
) -> StorageResult<Self> {
let mut store = Self::open(path)?;
if let Some(raw_store_path) = raw_store_path {
store.external_raw_store = Some(ExternalRawStoreDb::open(raw_store_path)?);
}
if let Some(repo_bytes_path) = repo_bytes_path {
store.external_repo_bytes = Some(ExternalRepoBytesDb::open(repo_bytes_path)?);
}
Ok(store)
}
pub(crate) fn external_raw_store_ref(&self) -> Option<&ExternalRawStoreDb> {
self.external_raw_store.as_ref()
}
pub(crate) fn external_repo_bytes_ref(&self) -> Option<&ExternalRepoBytesDb> {
self.external_repo_bytes.as_ref()
}
pub fn memory_snapshot(&self) -> RocksDbMemorySnapshot {
let mut databases = Vec::new();
databases.push(memory_db_snapshot_for_column_families(
"work-db",
&self.db,
Some(ALL_COLUMN_FAMILY_NAMES),
));
if let Some(raw_store) = self.external_raw_store.as_ref() {
databases.push(raw_store.memory_snapshot("raw-store.db"));
}
if let Some(repo_bytes) = self.external_repo_bytes.as_ref() {
databases.push(repo_bytes.memory_snapshot("repo-bytes.db"));
}
let mut totals = RocksDbMemoryTotals::default();
for db in &databases {
totals.add_properties(&db.properties);
}
RocksDbMemorySnapshot { databases, totals }
}
fn cf(&self, name: &'static str) -> StorageResult<&ColumnFamily> {
self.db
.cf_handle(name)
.ok_or(StorageError::MissingColumnFamily(name))
}
}
fn reject_unsupported_column_families(path: &Path) -> StorageResult<()> {
// An empty directory is a valid first-run state root. A RocksDB directory
// has CURRENT; inspect its column families before opening so an older
// private schema cannot be silently ignored or partially loaded.
if !path.join("CURRENT").exists() {
return Ok(());
}
let options = Options::default();
let existing = DB::list_cf(&options, path)
.map_err(|error| StorageError::RocksDb(format!("inspect state schema: {error}")))?;
let supported: std::collections::HashSet<&str> = std::iter::once("default")
.chain(ALL_COLUMN_FAMILY_NAMES.iter().copied())
.collect();
let unsupported: Vec<_> = existing
.iter()
.filter(|name| !supported.contains(name.as_str()))
.cloned()
.collect();
if unsupported.is_empty() {
return Ok(());
}
Err(StorageError::InvalidData {
entity: "state_schema",
detail: format!(
"unsupported column families {:?}; create a new state directory instead of reusing this database",
unsupported
),
})
}

View File

@@ -0,0 +1,33 @@
// Manifest rollback metadata is protocol state retained independently from
// repository objects and validation outputs.
impl RocksStore {
pub fn get_manifest_anti_rollback_meta(
&self,
manifest_rsync_uri: &str,
) -> StorageResult<Option<ManifestAntiRollbackMeta>> {
let cf = self.cf(CF_MANIFEST_ANTI_ROLLBACK)?;
let key = manifest_anti_rollback_key(manifest_rsync_uri);
let Some(bytes) = self
.db
.get_cf(cf, key.as_bytes())
.map_err(|error| StorageError::RocksDb(error.to_string()))?
else {
return Ok(None);
};
let metadata = decode_cbor::<ManifestAntiRollbackMeta>(&bytes, "manifest_anti_rollback")?;
metadata.validate_internal()?;
Ok(Some(metadata))
}
pub fn put_manifest_anti_rollback_meta(&self, metadata: &ManifestAntiRollbackMeta) -> StorageResult<()> {
metadata.validate_internal()?;
let cf = self.cf(CF_MANIFEST_ANTI_ROLLBACK)?;
let key = manifest_anti_rollback_key(&metadata.manifest_rsync_uri);
let value = encode_cbor(metadata, "manifest_anti_rollback")?;
self.db
.put_cf(cf, key.as_bytes(), value)
.map_err(|error| StorageError::RocksDb(error.to_string()))?;
Ok(())
}
}

View File

@@ -0,0 +1,363 @@
// Repository-view and raw-object/blob storage operations.
impl RocksStore {
pub fn put_repository_view_entry(&self, entry: &RepositoryViewEntry) -> StorageResult<()> {
entry.validate_internal()?;
let cf = self.cf(CF_REPOSITORY_VIEW)?;
let key = repository_view_key(&entry.rsync_uri);
let value = encode_cbor(entry, "repository_view")?;
self.db
.put_cf(cf, key.as_bytes(), value)
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(())
}
pub fn get_repository_view_entry(
&self,
rsync_uri: &str,
) -> StorageResult<Option<RepositoryViewEntry>> {
let cf = self.cf(CF_REPOSITORY_VIEW)?;
let key = repository_view_key(rsync_uri);
let Some(bytes) = self
.db
.get_cf(cf, key.as_bytes())
.map_err(|e| StorageError::RocksDb(e.to_string()))?
else {
return Ok(None);
};
let entry = decode_cbor::<RepositoryViewEntry>(&bytes, "repository_view")?;
entry.validate_internal()?;
Ok(Some(entry))
}
pub fn delete_repository_view_entry(&self, rsync_uri: &str) -> StorageResult<()> {
let cf = self.cf(CF_REPOSITORY_VIEW)?;
let key = repository_view_key(rsync_uri);
self.db
.delete_cf(cf, key.as_bytes())
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(())
}
pub fn put_projection_batch(
&self,
repository_view_entries: &[RepositoryViewEntry],
member_records: &[RrdpSourceMemberRecord],
owner_records: &[RrdpUriOwnerRecord],
) -> StorageResult<()> {
if repository_view_entries.is_empty()
&& member_records.is_empty()
&& owner_records.is_empty()
{
return Ok(());
}
let repo_cf = self.cf(CF_REPOSITORY_VIEW)?;
let member_cf = self.cf(CF_RRDP_SOURCE_MEMBER)?;
let owner_cf = self.cf(CF_RRDP_URI_OWNER)?;
let mut batch = WriteBatch::default();
for entry in repository_view_entries {
entry.validate_internal()?;
let key = repository_view_key(&entry.rsync_uri);
let value = encode_cbor(entry, "repository_view")?;
batch.put_cf(repo_cf, key.as_bytes(), value);
}
for record in member_records {
record.validate_internal()?;
let key = rrdp_source_member_key(&record.notify_uri, &record.rsync_uri);
let value = encode_cbor(record, "rrdp_source_member")?;
batch.put_cf(member_cf, key.as_bytes(), value);
}
for record in owner_records {
record.validate_internal()?;
let key = rrdp_uri_owner_key(&record.rsync_uri);
let value = encode_cbor(record, "rrdp_uri_owner")?;
batch.put_cf(owner_cf, key.as_bytes(), value);
}
self.write_batch(batch)
}
pub fn list_repository_view_entries_with_prefix(
&self,
rsync_uri_prefix: &str,
) -> StorageResult<Vec<RepositoryViewEntry>> {
let cf = self.cf(CF_REPOSITORY_VIEW)?;
let prefix = repository_view_prefix(rsync_uri_prefix);
let mode = IteratorMode::From(prefix.as_bytes(), Direction::Forward);
self.db
.iterator_cf(cf, mode)
.take_while(|res| match res {
Ok((key, _)) => key.starts_with(prefix.as_bytes()),
Err(_) => false,
})
.map(|res| {
let (_key, value) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?;
let entry = decode_cbor::<RepositoryViewEntry>(&value, "repository_view")?;
entry.validate_internal()?;
Ok(entry)
})
.collect()
}
pub fn verify_current_repository_blobs(
&self,
batch_size: usize,
) -> StorageResult<RepositoryBlobVerificationSummary> {
if batch_size == 0 {
return Err(StorageError::InvalidData {
entity: "repository_blob_verification",
detail: "batch_size must be greater than zero".to_string(),
});
}
let cf = self.cf(CF_REPOSITORY_VIEW)?;
let prefix = repository_view_prefix("");
let mode = IteratorMode::From(prefix.as_bytes(), Direction::Forward);
let mut summary = RepositoryBlobVerificationSummary::default();
let mut batch = Vec::with_capacity(batch_size);
for item in self.db.iterator_cf(cf, mode) {
let (key, value) = item.map_err(|error| StorageError::RocksDb(error.to_string()))?;
if !key.starts_with(prefix.as_bytes()) {
break;
}
let entry = decode_cbor::<RepositoryViewEntry>(&value, "repository_view")?;
entry.validate_internal()?;
if !matches!(
entry.state,
RepositoryViewState::Present | RepositoryViewState::Replaced
) {
continue;
}
let hash = entry
.current_hash
.clone()
.ok_or(StorageError::InvalidData {
entity: "repository_blob_verification",
detail: format!("current hash missing for {}", entry.rsync_uri),
})?;
batch.push((entry.rsync_uri, hash));
if batch.len() >= batch_size {
verify_repository_blob_batch(self, &batch, &mut summary)?;
batch.clear();
}
}
if !batch.is_empty() {
verify_repository_blob_batch(self, &batch, &mut summary)?;
}
Ok(summary)
}
pub fn put_raw_by_hash_entry(&self, entry: &RawByHashEntry) -> StorageResult<()> {
entry.validate_internal()?;
if let Some(raw_store) = self.external_raw_store.as_ref() {
return raw_store.put_raw_entry(entry);
}
let cf = self.cf(CF_RAW_BY_HASH)?;
let key = raw_by_hash_key(&entry.sha256_hex);
let value = encode_cbor(entry, "raw_by_hash")?;
self.db
.put_cf(cf, key.as_bytes(), value)
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(())
}
pub fn put_raw_by_hash_entries_batch(&self, entries: &[RawByHashEntry]) -> StorageResult<()> {
if entries.is_empty() {
return Ok(());
}
if let Some(raw_store) = self.external_raw_store.as_ref() {
return raw_store.put_raw_entries_batch(entries);
}
let cf = self.cf(CF_RAW_BY_HASH)?;
let mut batch = WriteBatch::default();
for entry in entries {
entry.validate_internal()?;
let key = raw_by_hash_key(&entry.sha256_hex);
let value = encode_cbor(entry, "raw_by_hash")?;
batch.put_cf(cf, key.as_bytes(), value);
}
self.write_batch(batch)
}
pub fn put_raw_by_hash_entries_batch_unchecked(
&self,
entries: &[RawByHashEntry],
) -> StorageResult<()> {
if entries.is_empty() {
return Ok(());
}
if let Some(raw_store) = self.external_raw_store.as_ref() {
return raw_store.put_raw_entries_batch(entries);
}
let cf = self.cf(CF_RAW_BY_HASH)?;
let mut batch = WriteBatch::default();
for entry in entries {
let key = raw_by_hash_key(&entry.sha256_hex);
let value = encode_cbor(entry, "raw_by_hash")?;
batch.put_cf(cf, key.as_bytes(), value);
}
self.write_batch(batch)
}
pub fn put_blob_bytes_batch(&self, blobs: &[(String, Vec<u8>)]) -> StorageResult<()> {
if blobs.is_empty() {
return Ok(());
}
if let Some(repo_bytes) = self.external_repo_bytes.as_ref() {
if repo_bytes.is_read_only() {
return repo_bytes.require_existing_blob_bytes_batch(blobs);
}
return repo_bytes.put_blob_bytes_batch(blobs);
}
if let Some(raw_store) = self.external_raw_store.as_ref() {
return raw_store.put_blob_bytes_batch(blobs);
}
let cf = self.cf(CF_RAW_BLOB)?;
let mut batch = WriteBatch::default();
for (sha256_hex, bytes) in blobs {
validate_sha256_hex("raw_blob.sha256_hex", sha256_hex)?;
if bytes.is_empty() {
return Err(StorageError::InvalidData {
entity: "raw_blob",
detail: "bytes must not be empty".to_string(),
});
}
let key = raw_blob_key(sha256_hex);
batch.put_cf(cf, key.as_bytes(), bytes.as_slice());
}
self.write_batch(batch)
}
pub fn delete_raw_by_hash_entry(&self, sha256_hex: &str) -> StorageResult<()> {
validate_sha256_hex("raw_by_hash.sha256_hex", sha256_hex)?;
if let Some(raw_store) = self.external_raw_store.as_ref() {
return raw_store.delete_raw_entry(sha256_hex);
}
let cf = self.cf(CF_RAW_BY_HASH)?;
let key = raw_by_hash_key(sha256_hex);
self.db
.delete_cf(cf, key.as_bytes())
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(())
}
pub fn get_raw_by_hash_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>> {
if let Some(raw_store) = self.external_raw_store.as_ref() {
return raw_store.get_raw_entry(sha256_hex);
}
let cf = self.cf(CF_RAW_BY_HASH)?;
let key = raw_by_hash_key(sha256_hex);
let Some(bytes) = self
.db
.get_cf(cf, key.as_bytes())
.map_err(|e| StorageError::RocksDb(e.to_string()))?
else {
return Ok(None);
};
let entry = decode_cbor::<RawByHashEntry>(&bytes, "raw_by_hash")?;
entry.validate_internal()?;
Ok(Some(entry))
}
pub fn get_raw_by_hash_entries_batch(
&self,
sha256_hexes: &[String],
) -> StorageResult<Vec<Option<RawByHashEntry>>> {
if sha256_hexes.is_empty() {
return Ok(Vec::new());
}
if let Some(raw_store) = self.external_raw_store.as_ref() {
return raw_store.get_raw_entries_batch(sha256_hexes);
}
let cf = self.cf(CF_RAW_BY_HASH)?;
let keys: Vec<String> = sha256_hexes
.iter()
.map(|hash| raw_by_hash_key(hash))
.collect();
self.db
.multi_get_cf(keys.iter().map(|key| (cf, key.as_bytes())))
.into_iter()
.map(|res| {
let maybe = res.map_err(|e| StorageError::RocksDb(e.to_string()))?;
match maybe {
Some(bytes) => {
let entry = decode_cbor::<RawByHashEntry>(&bytes, "raw_by_hash")?;
entry.validate_internal()?;
Ok(Some(entry))
}
None => Ok(None),
}
})
.collect()
}
pub fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult<Option<Vec<u8>>> {
if let Some(repo_bytes) = self.external_repo_bytes.as_ref() {
return repo_bytes.get_blob_bytes(sha256_hex);
}
if let Some(raw_store) = self.external_raw_store.as_ref() {
return raw_store.get_blob_bytes(sha256_hex);
}
validate_sha256_hex("raw_blob.sha256_hex", sha256_hex)?;
let cf = self.cf(CF_RAW_BLOB)?;
let key = raw_blob_key(sha256_hex);
if let Some(bytes) = self
.db
.get_cf(cf, key.as_bytes())
.map_err(|e| StorageError::RocksDb(e.to_string()))?
{
return Ok(Some(bytes));
}
self.get_raw_by_hash_entry(sha256_hex)
.map(|entry| entry.map(|entry| entry.bytes))
}
pub fn get_blob_bytes_batch(
&self,
sha256_hexes: &[String],
) -> StorageResult<Vec<Option<Vec<u8>>>> {
if sha256_hexes.is_empty() {
return Ok(Vec::new());
}
if let Some(repo_bytes) = self.external_repo_bytes.as_ref() {
return repo_bytes.get_blob_bytes_batch(sha256_hexes);
}
if let Some(raw_store) = self.external_raw_store.as_ref() {
return raw_store.get_blob_bytes_batch(sha256_hexes);
}
let cf = self.cf(CF_RAW_BLOB)?;
let keys: Vec<String> = sha256_hexes
.iter()
.map(|hash| {
validate_sha256_hex("raw_blob.sha256_hex", hash)?;
Ok::<String, StorageError>(raw_blob_key(hash))
})
.collect::<Result<_, _>>()?;
let blob_results: Vec<Option<Vec<u8>>> = self
.db
.multi_get_cf(keys.iter().map(|key| (cf, key.as_bytes())))
.into_iter()
.map(|res| res.map_err(|e| StorageError::RocksDb(e.to_string())))
.collect::<Result<_, _>>()?;
let mut out = Vec::with_capacity(sha256_hexes.len());
for (sha256_hex, maybe_blob) in sha256_hexes.iter().zip(blob_results.into_iter()) {
if maybe_blob.is_some() {
out.push(maybe_blob);
} else {
out.push(
self.get_raw_by_hash_entry(sha256_hex)?
.map(|entry| entry.bytes),
);
}
}
Ok(out)
}
}

View File

@@ -0,0 +1,108 @@
// RRDP source/session state and repository-view reads.
impl RocksStore {
pub fn put_rrdp_source_record(&self, record: &RrdpSourceRecord) -> StorageResult<()> {
record.validate_internal()?;
let cf = self.cf(CF_RRDP_SOURCE)?;
let key = rrdp_source_key(&record.notify_uri);
let value = encode_cbor(record, "rrdp_source")?;
self.db.put_cf(cf, key.as_bytes(), value).map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(())
}
pub fn get_rrdp_source_record(&self, notify_uri: &str) -> StorageResult<Option<RrdpSourceRecord>> {
let cf = self.cf(CF_RRDP_SOURCE)?;
let key = rrdp_source_key(notify_uri);
let Some(bytes) = self.db.get_cf(cf, key.as_bytes()).map_err(|e| StorageError::RocksDb(e.to_string()))? else { return Ok(None) };
let record = decode_cbor::<RrdpSourceRecord>(&bytes, "rrdp_source")?;
record.validate_internal()?;
Ok(Some(record))
}
pub fn put_rrdp_source_member_record(&self, record: &RrdpSourceMemberRecord) -> StorageResult<()> {
record.validate_internal()?;
let cf = self.cf(CF_RRDP_SOURCE_MEMBER)?;
let key = rrdp_source_member_key(&record.notify_uri, &record.rsync_uri);
let value = encode_cbor(record, "rrdp_source_member")?;
self.db.put_cf(cf, key.as_bytes(), value).map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(())
}
pub fn get_rrdp_source_member_record(&self, notify_uri: &str, rsync_uri: &str) -> StorageResult<Option<RrdpSourceMemberRecord>> {
let cf = self.cf(CF_RRDP_SOURCE_MEMBER)?;
let key = rrdp_source_member_key(notify_uri, rsync_uri);
let Some(bytes) = self.db.get_cf(cf, key.as_bytes()).map_err(|e| StorageError::RocksDb(e.to_string()))? else { return Ok(None) };
let record = decode_cbor::<RrdpSourceMemberRecord>(&bytes, "rrdp_source_member")?;
record.validate_internal()?;
Ok(Some(record))
}
pub fn list_rrdp_source_member_records(&self, notify_uri: &str) -> StorageResult<Vec<RrdpSourceMemberRecord>> {
let cf = self.cf(CF_RRDP_SOURCE_MEMBER)?;
let prefix = rrdp_source_member_prefix(notify_uri);
let mode = IteratorMode::From(prefix.as_bytes(), Direction::Forward);
self.db.iterator_cf(cf, mode).take_while(|res| match res { Ok((key, _)) => key.starts_with(prefix.as_bytes()), Err(_) => false }).map(|res| {
let (_key, value) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?;
let record = decode_cbor::<RrdpSourceMemberRecord>(&value, "rrdp_source_member")?;
record.validate_internal()?;
Ok(record)
}).collect()
}
pub fn list_current_rrdp_source_members(&self, notify_uri: &str) -> StorageResult<Vec<RrdpSourceMemberRecord>> {
let mut records = self.list_rrdp_source_member_records(notify_uri)?;
records.retain(|record| record.present);
records.sort_by(|a, b| a.rsync_uri.cmp(&b.rsync_uri));
Ok(records)
}
pub fn is_current_rrdp_source_member(&self, notify_uri: &str, rsync_uri: &str) -> StorageResult<bool> {
Ok(matches!(self.get_rrdp_source_member_record(notify_uri, rsync_uri)?, Some(record) if record.present))
}
pub fn load_current_object_bytes_by_uri(&self, rsync_uri: &str) -> StorageResult<Option<Vec<u8>>> {
Ok(self.load_current_object_with_hash_by_uri(rsync_uri)?.map(|object| object.bytes))
}
pub fn load_current_object_with_hash_by_uri(&self, rsync_uri: &str) -> StorageResult<Option<CurrentObjectWithHash>> {
let Some(view) = self.get_repository_view_entry(rsync_uri)? else { return Ok(None) };
match view.state {
RepositoryViewState::Withdrawn => Ok(None),
RepositoryViewState::Present | RepositoryViewState::Replaced => {
let hash = view.current_hash.as_deref().ok_or(StorageError::InvalidData { entity: "repository_view", detail: format!("current_hash missing for current object URI: {rsync_uri}") })?;
let bytes = self.get_blob_bytes(hash)?.ok_or(StorageError::InvalidData { entity: "repository_view", detail: format!("blob bytes missing for current object URI: {rsync_uri} (hash={hash})") })?;
Ok(Some(CurrentObjectWithHash { current_hash_hex: hash.to_ascii_lowercase(), current_hash: decode_sha256_hex_32("repository_view.current_hash", hash)?, bytes }))
}
}
}
pub fn put_rrdp_uri_owner_record(&self, record: &RrdpUriOwnerRecord) -> StorageResult<()> {
record.validate_internal()?;
let cf = self.cf(CF_RRDP_URI_OWNER)?;
let key = rrdp_uri_owner_key(&record.rsync_uri);
let value = encode_cbor(record, "rrdp_uri_owner")?;
self.db.put_cf(cf, key.as_bytes(), value).map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(())
}
pub fn get_rrdp_uri_owner_record(&self, rsync_uri: &str) -> StorageResult<Option<RrdpUriOwnerRecord>> {
let cf = self.cf(CF_RRDP_URI_OWNER)?;
let key = rrdp_uri_owner_key(rsync_uri);
let Some(bytes) = self.db.get_cf(cf, key.as_bytes()).map_err(|e| StorageError::RocksDb(e.to_string()))? else { return Ok(None) };
let record = decode_cbor::<RrdpUriOwnerRecord>(&bytes, "rrdp_uri_owner")?;
record.validate_internal()?;
Ok(Some(record))
}
pub fn delete_rrdp_uri_owner_record(&self, rsync_uri: &str) -> StorageResult<()> {
let cf = self.cf(CF_RRDP_URI_OWNER)?;
let key = rrdp_uri_owner_key(rsync_uri);
self.db.delete_cf(cf, key.as_bytes()).map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(())
}
pub fn write_batch(&self, batch: WriteBatch) -> StorageResult<()> {
self.db.write(batch).map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(())
}
}

View File

@@ -0,0 +1,52 @@
// Storage tests are grouped by the public storage surface they protect.
use super::*;
use rocksdb::{DB, Options};
use sha2::Digest;
fn sha256_hex(bytes: &[u8]) -> String {
hex::encode(sha2::Sha256::digest(bytes))
}
fn pack_time(seconds: i64) -> PackTime {
PackTime::from_utc_offset_datetime(
time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(seconds),
)
}
fn sample_repository_view_entry(uri: &str, bytes: &[u8]) -> RepositoryViewEntry {
RepositoryViewEntry {
rsync_uri: uri.to_string(),
current_hash: Some(sha256_hex(bytes)),
repository_source: Some("fixture".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
}
}
fn sample_raw_by_hash_entry(bytes: Vec<u8>) -> RawByHashEntry {
RawByHashEntry::from_bytes(sha256_hex(&bytes), bytes)
}
#[test]
fn opening_an_unsupported_state_schema_fails_without_rewriting() {
let td = tempfile::tempdir().expect("tempdir");
let mut options = Options::default();
options.create_if_missing(true);
options.create_missing_column_families(true);
let legacy = DB::open_cf(&options, td.path(), ["legacy_private_state"])
.expect("create legacy state database");
drop(legacy);
match RocksStore::open(td.path()) {
Err(StorageError::InvalidData {
entity: "state_schema",
detail,
}) => assert!(detail.contains("legacy_private_state")),
Err(error) => panic!("unexpected schema error: {error}"),
Ok(_) => panic!("legacy schema must be rejected"),
}
}
include!("tests_parts/repository.rs");
include!("tests_parts/rrdp.rs");
include!("tests_parts/object_loading.rs");

View File

@@ -0,0 +1,234 @@
// Storage test group: object loading.
#[test]
fn load_current_object_with_hash_by_uri_uses_internal_blob_cf_without_raw_entry() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let rsync_uri = "rsync://example.test/repo/blob-only.roa";
let bytes = b"blob-only-current-object".to_vec();
let hash = sha256_hex(&bytes);
store
.put_blob_bytes_batch(&[(hash.clone(), bytes.clone())])
.expect("put blob bytes");
store
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: rsync_uri.to_string(),
current_hash: Some(hash.clone()),
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
})
.expect("put view");
let got = store
.load_current_object_with_hash_by_uri(rsync_uri)
.expect("load current object")
.expect("current object exists");
assert_eq!(got.current_hash_hex, hash);
assert_eq!(got.current_hash, compute_sha256_32(&bytes));
assert_eq!(got.bytes, bytes);
assert!(
store
.get_raw_by_hash_entry(&got.current_hash_hex)
.expect("get raw entry")
.is_none()
);
}
#[test]
fn load_current_object_bytes_by_uri_uses_internal_blob_cf_without_raw_entry() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let rsync_uri = "rsync://example.test/repo/blob-only-bytes.roa";
let bytes = b"blob-only-current-object-bytes".to_vec();
let hash = sha256_hex(&bytes);
store
.put_blob_bytes_batch(&[(hash, bytes.clone())])
.expect("put blob bytes");
store
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: rsync_uri.to_string(),
current_hash: Some(sha256_hex(&bytes)),
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
})
.expect("put view");
assert_eq!(
store
.load_current_object_bytes_by_uri(rsync_uri)
.expect("load current object bytes"),
Some(bytes)
);
}
#[test]
fn pack_file_can_lazy_load_bytes_from_external_raw_store() {
let td = tempfile::tempdir().expect("tempdir");
let raw_store = std::sync::Arc::new(
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store"),
);
let bytes = b"lazy-pack-file".to_vec();
let sha256_hex = sha256_hex(&bytes);
raw_store
.put_raw_entry(&RawByHashEntry::from_bytes(
sha256_hex.clone(),
bytes.clone(),
))
.expect("put raw entry");
let file = PackFile::from_lazy_external_raw_store(
"rsync://example.test/repo/a.roa",
sha256_hex,
compute_sha256_32(&bytes),
raw_store,
);
assert_eq!(file.bytes().expect("lazy bytes"), bytes.as_slice());
assert_eq!(file.bytes_cloned().expect("cloned bytes"), bytes);
}
#[test]
fn pack_file_can_lazy_load_bytes_from_external_repo_bytes_store() {
let td = tempfile::tempdir().expect("tempdir");
let repo_bytes_store = std::sync::Arc::new(
ExternalRepoBytesDb::open(td.path().join("repo-bytes.db")).expect("open repo bytes"),
);
let bytes = b"repo-object-pack-file".to_vec();
let sha256_hex = sha256_hex(&bytes);
repo_bytes_store
.put_blob_bytes_batch(&[(sha256_hex.clone(), bytes.clone())])
.expect("put repo bytes");
let file = PackFile::from_lazy_repo_bytes(
"rsync://example.test/repo/a.roa",
sha256_hex,
compute_sha256_32(&bytes),
repo_bytes_store,
);
assert_eq!(file.bytes().expect("lazy repo bytes"), bytes.as_slice());
assert_eq!(file.bytes_cloned().expect("cloned repo bytes"), bytes);
assert_eq!(file.compute_sha256().expect("compute sha256"), file.sha256);
}
#[test]
fn read_only_checkpoint_isolated_from_source_work_db() {
let td = tempfile::tempdir().expect("tempdir");
let source_path = td.path().join("source-work-db");
let checkpoint_path = td.path().join("checkpoint-work-db");
let uri = "rsync://example.test/repo/a.roa";
let bytes = b"checkpoint-object";
let hash = sha256_hex(bytes);
{
let source = RocksStore::open(&source_path).expect("open source");
source
.put_blob_bytes_batch(&[(hash.clone(), bytes.to_vec())])
.expect("put blob");
source
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: uri.to_string(),
current_hash: Some(hash),
repository_source: Some("fixture".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
})
.expect("put view");
}
RocksStore::create_read_only_checkpoint(&source_path, &checkpoint_path)
.expect("create checkpoint");
let checkpoint = RocksStore::open(&checkpoint_path).expect("open checkpoint");
checkpoint
.delete_repository_view_entry(uri)
.expect("delete checkpoint entry");
drop(checkpoint);
let source = RocksStore::open(&source_path).expect("reopen source");
assert!(
source
.get_repository_view_entry(uri)
.expect("get source entry")
.is_some()
);
}
#[test]
fn read_only_external_repo_bytes_rejects_writes() {
let td = tempfile::tempdir().expect("tempdir");
let path = td.path().join("repo-bytes.db");
let hash = sha256_hex(b"repo-object");
{
let writable = ExternalRepoBytesDb::open(&path).expect("open writable repo bytes");
writable
.put_blob_bytes_batch(&[(hash.clone(), b"repo-object".to_vec())])
.expect("seed repo bytes");
}
let read_only = ExternalRepoBytesDb::open_read_only(&path).expect("open read-only repo bytes");
assert_eq!(
read_only
.get_blob_bytes(&hash)
.expect("read repo bytes")
.expect("blob"),
b"repo-object"
);
assert!(
read_only
.put_blob_bytes_batch(&[(hash, b"repo-object".to_vec())])
.is_err()
);
}
#[test]
fn read_only_store_accepts_only_idempotent_existing_blob_apply() {
let td = tempfile::tempdir().expect("tempdir");
let work_db_path = td.path().join("work-db");
let repo_bytes_path = td.path().join("repo-bytes.db");
let bytes = b"repo-object".to_vec();
let hash = sha256_hex(&bytes);
{
let repo_bytes = ExternalRepoBytesDb::open(&repo_bytes_path).expect("open repo bytes");
repo_bytes
.put_blob_bytes_batch(&[(hash.clone(), bytes.clone())])
.expect("seed repo bytes");
}
let store =
RocksStore::open_with_external_repo_bytes_read_only(&work_db_path, &repo_bytes_path)
.expect("open read-only store");
store
.put_blob_bytes_batch(&[(hash.clone(), bytes)])
.expect("idempotent apply");
assert!(
store
.put_blob_bytes_batch(&[(hash, b"different".to_vec())])
.is_err()
);
}
#[test]
fn repository_blob_verification_detects_missing_external_blob() {
let td = tempfile::tempdir().expect("tempdir");
let repo_bytes_path = td.path().join("repo-bytes.db");
ExternalRepoBytesDb::open(&repo_bytes_path).expect("create repo bytes");
let store = RocksStore::open_with_external_repo_bytes_read_only(
&td.path().join("work-db"),
&repo_bytes_path,
)
.expect("open work db");
store
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/missing.roa".to_string(),
current_hash: Some(sha256_hex(b"missing")),
repository_source: Some("fixture".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
})
.expect("put view");
let error = store
.verify_current_repository_blobs(16)
.expect_err("missing blob must fail");
assert!(error.to_string().contains("missing.roa"), "{error}");
}

View File

@@ -0,0 +1,391 @@
// Storage test group: repository.
fn sample_rrdp_source_member_record(
notify_uri: &str,
rsync_uri: &str,
serial: u64,
) -> RrdpSourceMemberRecord {
RrdpSourceMemberRecord {
notify_uri: notify_uri.to_string(),
rsync_uri: rsync_uri.to_string(),
current_hash: Some(sha256_hex(rsync_uri.as_bytes())),
object_type: Some("cer".to_string()),
present: true,
last_confirmed_session_id: "session-1".to_string(),
last_confirmed_serial: serial,
last_changed_at: pack_time(serial as i64),
}
}
#[test]
fn repository_view_and_raw_by_hash_roundtrip() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let entry1 = sample_repository_view_entry("rsync://example.test/repo/a.cer", b"object-a");
let entry2 = sample_repository_view_entry("rsync://example.test/repo/sub/b.roa", b"object-b");
store
.put_repository_view_entry(&entry1)
.expect("put repository view entry1");
store
.put_repository_view_entry(&entry2)
.expect("put repository view entry2");
let got1 = store
.get_repository_view_entry(&entry1.rsync_uri)
.expect("get repository view entry1")
.expect("entry1 exists");
assert_eq!(got1, entry1);
let got_prefix = store
.list_repository_view_entries_with_prefix("rsync://example.test/repo/sub/")
.expect("list repository view prefix");
assert_eq!(got_prefix, vec![entry2.clone()]);
store
.delete_repository_view_entry(&entry1.rsync_uri)
.expect("delete repository view entry1");
assert!(
store
.get_repository_view_entry(&entry1.rsync_uri)
.expect("get deleted repository view entry1")
.is_none()
);
let raw = sample_raw_by_hash_entry(b"raw-der-object".to_vec());
store
.put_raw_by_hash_entry(&raw)
.expect("put raw_by_hash entry");
let got_raw = store
.get_raw_by_hash_entry(&raw.sha256_hex)
.expect("get raw_by_hash entry")
.expect("raw entry exists");
assert_eq!(got_raw, raw);
}
#[test]
fn raw_by_hash_routes_to_external_raw_store_when_configured() {
let td = tempfile::tempdir().expect("tempdir");
let main_db = td.path().join("main-db");
let raw_db = td.path().join("raw-store.db");
let raw = sample_raw_by_hash_entry(b"external-raw".to_vec());
{
let store =
RocksStore::open_with_external_raw_store(&main_db, &raw_db).expect("open store");
store.put_raw_by_hash_entry(&raw).expect("put external raw");
let got = store
.get_raw_by_hash_entry(&raw.sha256_hex)
.expect("get external raw")
.expect("raw exists");
assert_eq!(got, raw);
}
let main_store = RocksStore::open(&main_db).expect("open main only");
assert!(
main_store
.get_raw_by_hash_entry(&raw.sha256_hex)
.expect("read main store")
.is_none(),
"main db should not contain raw entry when external raw store is configured"
);
}
#[test]
fn put_blob_bytes_batch_uses_internal_blob_cf_without_raw_entry() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let bytes = b"internal-blob-only".to_vec();
let hash = sha256_hex(&bytes);
store
.put_blob_bytes_batch(&[(hash.clone(), bytes.clone())])
.expect("put blob bytes");
assert_eq!(
store.get_blob_bytes(&hash).expect("get blob bytes"),
Some(bytes.clone())
);
assert!(
store
.get_raw_by_hash_entry(&hash)
.expect("get raw entry")
.is_none()
);
}
#[test]
fn put_blob_bytes_batch_routes_to_external_raw_store_without_raw_entry() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open_with_external_raw_store(
&td.path().join("main-db"),
&td.path().join("raw-store.db"),
)
.expect("open store");
let bytes = b"external-blob-only".to_vec();
let hash = sha256_hex(&bytes);
store
.put_blob_bytes_batch(&[(hash.clone(), bytes.clone())])
.expect("put external blob bytes");
assert_eq!(store.get_blob_bytes(&hash).unwrap(), Some(bytes));
assert!(store.get_raw_by_hash_entry(&hash).unwrap().is_none());
}
#[test]
fn repo_bytes_db_is_physically_separate_from_external_raw_store() {
let td = tempfile::tempdir().expect("tempdir");
let main_db = td.path().join("main-db");
let raw_db = td.path().join("raw-store.db");
let repo_bytes_db = td.path().join("repo-bytes.db");
let store =
RocksStore::open_with_external_stores(&main_db, Some(&raw_db), Some(&repo_bytes_db))
.expect("open store");
let repo_bytes = b"repo-object".to_vec();
let repo_hash = sha256_hex(&repo_bytes);
let raw = sample_raw_by_hash_entry(b"raw-evidence".to_vec());
store
.put_blob_bytes_batch(&[(repo_hash.clone(), repo_bytes.clone())])
.expect("put repo bytes");
store.put_raw_by_hash_entry(&raw).expect("put raw evidence");
assert_eq!(store.get_blob_bytes(&repo_hash).unwrap(), Some(repo_bytes));
assert_eq!(
store.get_raw_by_hash_entry(&raw.sha256_hex).unwrap(),
Some(raw.clone())
);
drop(store);
let raw_only = RocksStore::open_with_external_raw_store(&td.path().join("raw-reader"), &raw_db)
.expect("open raw only");
assert!(
raw_only.get_blob_bytes(&repo_hash).unwrap().is_none(),
"repo object bytes must not be written into raw-store.db"
);
let repo_only =
RocksStore::open_with_external_repo_bytes(&td.path().join("repo-reader"), &repo_bytes_db)
.expect("open repo bytes only");
assert_eq!(
repo_only.get_blob_bytes(&repo_hash).unwrap(),
Some(b"repo-object".to_vec())
);
assert!(
repo_only.get_blob_bytes(&raw.sha256_hex).unwrap().is_none(),
"raw evidence bytes must not be written into repo-bytes.db"
);
}
#[test]
fn memory_snapshot_includes_work_db_and_external_stores() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open_with_external_stores(
&td.path().join("main-db"),
Some(&td.path().join("raw-store.db")),
Some(&td.path().join("repo-bytes.db")),
)
.expect("open store");
let snapshot = store.memory_snapshot();
let labels: Vec<&str> = snapshot
.databases
.iter()
.map(|db| db.label.as_str())
.collect();
assert_eq!(labels, vec!["work-db", "raw-store.db", "repo-bytes.db"]);
assert!(
snapshot.databases[0]
.column_families
.iter()
.any(|cf| cf.name == CF_REPOSITORY_VIEW)
);
serde_json::to_value(&snapshot).expect("serialize memory snapshot");
}
#[test]
fn put_blob_bytes_batch_accepts_empty_batch_with_external_raw_store() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open_with_external_raw_store(
&td.path().join("main-db"),
&td.path().join("raw-store.db"),
)
.expect("open store");
store
.put_blob_bytes_batch(&[])
.expect("empty external blob batch should be a no-op");
}
#[test]
fn get_blob_bytes_internal_falls_back_to_raw_entry_when_blob_missing() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let raw = sample_raw_by_hash_entry(b"raw-fallback".to_vec());
store.put_raw_by_hash_entry(&raw).expect("put raw entry");
assert_eq!(
store
.get_blob_bytes(&raw.sha256_hex)
.expect("get blob bytes via raw fallback"),
Some(raw.bytes.clone())
);
}
#[test]
fn get_blob_bytes_batch_internal_prefers_blob_cf_and_falls_back_to_raw_entry() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let blob_bytes = b"blob-cf-object".to_vec();
let blob_hash = sha256_hex(&blob_bytes);
store
.put_blob_bytes_batch(&[(blob_hash.clone(), blob_bytes.clone())])
.expect("put blob bytes");
let raw = sample_raw_by_hash_entry(b"raw-fallback-batch".to_vec());
store.put_raw_by_hash_entry(&raw).expect("put raw fallback");
let batch = store
.get_blob_bytes_batch(&[blob_hash.clone(), raw.sha256_hex.clone(), "00".repeat(32)])
.expect("get blob bytes batch");
assert_eq!(batch, vec![Some(blob_bytes), Some(raw.bytes.clone()), None]);
}
#[test]
fn get_blob_bytes_batch_routes_to_external_raw_store_without_raw_entry() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open_with_external_raw_store(
&td.path().join("main-db"),
&td.path().join("raw-store.db"),
)
.expect("open store");
let bytes = b"external-batch-blob".to_vec();
let hash = sha256_hex(&bytes);
store
.put_blob_bytes_batch(&[(hash.clone(), bytes.clone())])
.expect("put external blob bytes");
assert_eq!(
store
.get_blob_bytes_batch(&[hash, "00".repeat(32)])
.expect("get external blob batch"),
vec![Some(bytes), None]
);
}
#[test]
fn get_blob_bytes_rejects_invalid_hash_for_internal_store() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let err = store
.get_blob_bytes("not-a-valid-hash")
.expect_err("invalid hash must fail");
assert!(matches!(err, StorageError::InvalidData { .. }));
}
#[test]
fn get_blob_bytes_batch_rejects_invalid_hash_for_internal_store() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let err = store
.get_blob_bytes_batch(&["not-a-valid-hash".to_string()])
.expect_err("invalid hash must fail");
assert!(matches!(err, StorageError::InvalidData { .. }));
}
#[test]
fn get_blob_bytes_batch_returns_empty_for_empty_request_internal() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
assert!(
store
.get_blob_bytes_batch(&[])
.expect("empty blob batch request")
.is_empty()
);
}
#[test]
fn put_blob_bytes_batch_accepts_empty_batch() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
store
.put_blob_bytes_batch(&[])
.expect("empty blob batch should be a no-op");
}
#[test]
fn put_blob_bytes_batch_rejects_empty_bytes() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let err = store
.put_blob_bytes_batch(&[(sha256_hex(b"valid"), Vec::new())])
.expect_err("empty bytes must fail");
assert!(matches!(err, StorageError::InvalidData { .. }));
}
#[test]
fn delete_raw_by_hash_entry_internal_preserves_blob_bytes() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let bytes = b"blob-persists-after-raw-delete".to_vec();
let hash = sha256_hex(&bytes);
let raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone());
store
.put_blob_bytes_batch(&[(hash.clone(), bytes.clone())])
.expect("put blob bytes");
store.put_raw_by_hash_entry(&raw).expect("put raw entry");
store
.delete_raw_by_hash_entry(&hash)
.expect("delete raw entry only");
assert!(store.get_raw_by_hash_entry(&hash).unwrap().is_none());
assert_eq!(store.get_blob_bytes(&hash).unwrap(), Some(bytes));
}
#[test]
fn delete_raw_by_hash_entry_rejects_invalid_hash() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let err = store
.delete_raw_by_hash_entry("not-a-valid-hash")
.expect_err("invalid hash must fail");
assert!(matches!(err, StorageError::InvalidData { .. }));
}
#[test]
fn delete_raw_by_hash_entry_routes_to_external_raw_store() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open_with_external_raw_store(
&td.path().join("main-db"),
&td.path().join("raw-store.db"),
)
.expect("open store");
let raw = sample_raw_by_hash_entry(b"external-delete".to_vec());
store.put_raw_by_hash_entry(&raw).expect("put raw entry");
store
.delete_raw_by_hash_entry(&raw.sha256_hex)
.expect("delete external raw entry");
assert!(
store
.get_raw_by_hash_entry(&raw.sha256_hex)
.unwrap()
.is_none()
);
assert!(store.get_blob_bytes(&raw.sha256_hex).unwrap().is_none());
}

View File

@@ -0,0 +1,262 @@
// Storage test group: rrdp.
#[test]
fn projection_batch_roundtrip_writes_repository_view_member_and_owner_records() {
let dir = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(dir.path()).expect("open store");
let view = RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
current_hash: Some(hex::encode([1u8; 32])),
repository_source: Some("https://example.test/notify.xml".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
};
let member = RrdpSourceMemberRecord {
notify_uri: "https://example.test/notify.xml".to_string(),
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
current_hash: Some(hex::encode([1u8; 32])),
object_type: Some("roa".to_string()),
present: true,
last_confirmed_session_id: "session-1".to_string(),
last_confirmed_serial: 7,
last_changed_at: pack_time(1),
};
let owner = RrdpUriOwnerRecord {
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
notify_uri: "https://example.test/notify.xml".to_string(),
current_hash: Some(hex::encode([1u8; 32])),
last_confirmed_session_id: "session-1".to_string(),
last_confirmed_serial: 7,
last_changed_at: pack_time(1),
owner_state: RrdpUriOwnerState::Active,
};
store
.put_projection_batch(
std::slice::from_ref(&view),
std::slice::from_ref(&member),
std::slice::from_ref(&owner),
)
.expect("write projection batch");
assert_eq!(
store
.get_repository_view_entry(&view.rsync_uri)
.expect("get view")
.expect("present view"),
view
);
assert_eq!(
store
.get_rrdp_source_member_record(&member.notify_uri, &member.rsync_uri)
.expect("get member")
.expect("present member"),
member
);
assert_eq!(
store
.get_rrdp_uri_owner_record(&owner.rsync_uri)
.expect("get owner")
.expect("present owner"),
owner
);
}
#[test]
fn current_rrdp_source_member_helpers_filter_present_records() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let notify_uri = "https://rrdp.example.test/notification.xml";
let mut present_a =
sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/a.cer", 1);
let mut withdrawn_b =
sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/b.roa", 2);
withdrawn_b.present = false;
let present_c =
sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/c.crl", 3);
let other_source = sample_rrdp_source_member_record(
"https://other.example.test/notification.xml",
"rsync://other.example.test/repo/x.cer",
4,
);
present_a.last_confirmed_serial = 10;
store
.put_rrdp_source_member_record(&present_a)
.expect("put present a");
store
.put_rrdp_source_member_record(&withdrawn_b)
.expect("put withdrawn b");
store
.put_rrdp_source_member_record(&present_c)
.expect("put present c");
store
.put_rrdp_source_member_record(&other_source)
.expect("put other source");
let members = store
.list_current_rrdp_source_members(notify_uri)
.expect("list current members");
assert_eq!(
members
.iter()
.map(|record| record.rsync_uri.as_str())
.collect::<Vec<_>>(),
vec![
"rsync://example.test/repo/a.cer",
"rsync://example.test/repo/c.crl",
]
);
assert!(
store
.is_current_rrdp_source_member(notify_uri, &present_a.rsync_uri)
.expect("current a")
);
assert!(
!store
.is_current_rrdp_source_member(notify_uri, &withdrawn_b.rsync_uri)
.expect("withdrawn b")
);
assert!(
!store
.is_current_rrdp_source_member(notify_uri, &other_source.rsync_uri)
.expect("other source")
);
}
#[test]
fn load_current_object_bytes_by_uri_uses_repository_view_and_raw_by_hash() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let present_bytes = b"present-object".to_vec();
let present_hash = sha256_hex(&present_bytes);
let mut present_raw = RawByHashEntry::from_bytes(present_hash.clone(), present_bytes.clone());
present_raw
.origin_uris
.push("rsync://example.test/repo/present.roa".to_string());
present_raw.object_type = Some("roa".to_string());
store
.put_raw_by_hash_entry(&present_raw)
.expect("put present raw");
store
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/present.roa".to_string(),
current_hash: Some(present_hash),
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
})
.expect("put present view");
let replaced_bytes = b"replaced-object".to_vec();
let replaced_hash = sha256_hex(&replaced_bytes);
let mut replaced_raw =
RawByHashEntry::from_bytes(replaced_hash.clone(), replaced_bytes.clone());
replaced_raw
.origin_uris
.push("rsync://example.test/repo/replaced.cer".to_string());
replaced_raw.object_type = Some("cer".to_string());
store
.put_raw_by_hash_entry(&replaced_raw)
.expect("put replaced raw");
store
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/replaced.cer".to_string(),
current_hash: Some(replaced_hash),
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
object_type: Some("cer".to_string()),
state: RepositoryViewState::Replaced,
})
.expect("put replaced view");
store
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/withdrawn.crl".to_string(),
current_hash: Some(sha256_hex(b"withdrawn")),
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
object_type: Some("crl".to_string()),
state: RepositoryViewState::Withdrawn,
})
.expect("put withdrawn view");
assert_eq!(
store
.load_current_object_bytes_by_uri("rsync://example.test/repo/present.roa")
.expect("load present"),
Some(present_bytes)
);
assert_eq!(
store
.load_current_object_bytes_by_uri("rsync://example.test/repo/replaced.cer")
.expect("load replaced"),
Some(replaced_bytes)
);
assert_eq!(
store
.load_current_object_bytes_by_uri("rsync://example.test/repo/withdrawn.crl")
.expect("load withdrawn"),
None
);
assert_eq!(
store
.load_current_object_bytes_by_uri("rsync://example.test/repo/missing.roa")
.expect("load missing"),
None
);
}
#[test]
fn load_current_object_bytes_by_uri_errors_when_raw_by_hash_is_missing() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let rsync_uri = "rsync://example.test/repo/missing.cer";
store
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: rsync_uri.to_string(),
current_hash: Some(hex::encode([0x11; 32])),
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
object_type: Some("cer".to_string()),
state: RepositoryViewState::Present,
})
.expect("put view");
let err = store
.load_current_object_bytes_by_uri(rsync_uri)
.expect_err("missing raw_by_hash should error");
assert!(matches!(err, StorageError::InvalidData { .. }));
}
#[test]
fn load_current_object_with_hash_by_uri_returns_hash_and_bytes() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let rsync_uri = "rsync://example.test/repo/present.roa";
let bytes = b"present-object".to_vec();
let hash = sha256_hex(&bytes);
let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone());
raw.origin_uris.push(rsync_uri.to_string());
raw.object_type = Some("roa".to_string());
store.put_raw_by_hash_entry(&raw).expect("put raw");
store
.put_repository_view_entry(&RepositoryViewEntry {
rsync_uri: rsync_uri.to_string(),
current_hash: Some(hash.clone()),
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
})
.expect("put view");
let got = store
.load_current_object_with_hash_by_uri(rsync_uri)
.expect("load current object")
.expect("current object exists");
assert_eq!(got.current_hash_hex, hash);
assert_eq!(got.current_hash, compute_sha256_32(&bytes));
assert_eq!(got.bytes, bytes);
}

View File

@@ -0,0 +1,32 @@
// Repository blob integrity verification helper.
fn verify_repository_blob_batch(
store: &RocksStore,
batch: &[(String, String)],
summary: &mut RepositoryBlobVerificationSummary,
) -> StorageResult<()> {
let hashes = batch
.iter()
.map(|(_, hash)| hash.clone())
.collect::<Vec<_>>();
let blobs = store.get_blob_bytes_batch(&hashes)?;
for ((uri, expected_hash), blob) in batch.iter().zip(blobs.into_iter()) {
let bytes = blob.as_ref().ok_or(StorageError::InvalidData {
entity: "repository_blob_verification",
detail: format!("blob missing for URI {uri} (hash={expected_hash})"),
})?;
let actual_hash = hex::encode(compute_sha256_32(bytes));
if !actual_hash.eq_ignore_ascii_case(expected_hash) {
return Err(StorageError::InvalidData {
entity: "repository_blob_verification",
detail: format!(
"blob hash mismatch for URI {uri}: expected={expected_hash}, actual={actual_hash}"
),
});
}
summary.current_objects += 1;
summary.bytes_verified += bytes.len() as u64;
}
summary.batches += 1;
Ok(())
}

View File

@@ -0,0 +1,3 @@
pub mod repo;
pub mod rrdp;
pub(crate) mod store_projection;

516
src/repository/sync/repo.rs Normal file
View File

@@ -0,0 +1,516 @@
#![allow(clippy::too_many_arguments)]
use crate::output::analysis::timing::TimingHandle;
use crate::output::audit::AuditDownloadKind;
use crate::output::audit_downloads::DownloadLogHandle;
use crate::output::report::{RfcRef, Warning};
use crate::repository::current_repo_index::CurrentRepoIndexHandle;
use crate::repository::fetch::rsync::{RsyncFetchError, RsyncFetcher};
use crate::repository::storage::RocksStore;
use crate::repository::sync::rrdp::sync_from_notification_with_timing_and_download_log;
use crate::repository::sync::rrdp::{
Fetcher as HttpFetcher, RrdpFetchError, RrdpOrigin, RrdpResourceKind, RrdpSyncError,
};
use crate::repository::sync::store_projection::{
build_repository_view_present_entry, build_repository_view_withdrawn_entry,
prepare_repo_bytes_batch_owned,
};
use crate::validation::policy::{Policy, SyncPreference};
use std::collections::HashSet;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RepoSyncSource {
Rrdp,
Rsync,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RepoSyncPhase {
RrdpOk,
RrdpFailedRsyncOk,
RsyncOnlyOk,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RepoSyncResult {
pub source: RepoSyncSource,
pub phase: RepoSyncPhase,
pub objects_written: usize,
pub warnings: Vec<Warning>,
}
#[derive(Debug, thiserror::Error)]
pub enum RepoSyncError {
#[error("RRDP sync failed: {0}")]
Rrdp(#[from] RrdpSyncError),
#[error("rsync fallback failed: {0}")]
Rsync(#[from] RsyncFetchError),
#[error("storage error: {0}")]
Storage(String),
}
/// Sync a publication point into the current repository view.
///
/// v1 behavior:
/// - If `rrdp_notification_uri` is present and `policy.sync_preference` is `rrdp_then_rsync`,
/// try RRDP snapshot sync first (RFC 8182 §3.4.1-§3.4.3).
/// - On RRDP failure, fall back to rsync (RFC 8182 §3.4.5).
/// - If `sync_preference` is `rsync_only` or there is no RRDP URI, use rsync.
pub fn sync_publication_point(
store: &RocksStore,
policy: &Policy,
rrdp_notification_uri: Option<&str>,
rsync_base_uri: &str,
http_fetcher: &dyn HttpFetcher,
rsync_fetcher: &dyn RsyncFetcher,
timing: Option<&TimingHandle>,
download_log: Option<&DownloadLogHandle>,
) -> Result<RepoSyncResult, RepoSyncError> {
match (policy.sync_preference, rrdp_notification_uri) {
(SyncPreference::RrdpThenRsync, Some(notification_uri)) => {
match try_rrdp_sync_with_retry(
store,
notification_uri,
None,
http_fetcher,
timing,
download_log,
) {
Ok(written) => {
if let Some(t) = timing.as_ref() {
t.record_count("repo_sync_rrdp_ok_total", 1);
t.record_count("repo_sync_rrdp_objects_written_total", written as u64);
}
crate::logging::progress::emit!(
"repo_sync_rrdp_ok",
serde_json::json!({
"notify_uri": notification_uri,
"objects_written": written,
}),
);
Ok(RepoSyncResult {
source: RepoSyncSource::Rrdp,
phase: RepoSyncPhase::RrdpOk,
objects_written: written,
warnings: Vec::new(),
})
}
Err(err) => {
if let Some(t) = timing.as_ref() {
t.record_count("repo_sync_rrdp_failed_total", 1);
}
crate::logging::progress::emit!(
"rrdp_fallback_rsync",
serde_json::json!({
"notify_uri": notification_uri,
"rsync_base_uri": rsync_base_uri,
"rrdp_error": err.to_string(),
}),
);
crate::logging::progress::emit!(
"rrdp_failed_fallback_rsync",
serde_json::json!({
"notify_uri": notification_uri,
"rsync_base_uri": rsync_base_uri,
"rrdp_error": err.to_string(),
}),
);
let mut rfc_refs = vec![RfcRef("RFC 8182 §3.4.5")];
if matches!(&err, RrdpSyncError::Rrdp(e) if e.is_cross_origin_violation()) {
rfc_refs.push(RfcRef("RFC 9674 §3.2"));
}
let warnings = vec![
Warning::new(format!("RRDP failed; falling back to rsync: {err}"))
.with_rfc_refs(&rfc_refs)
.with_context(notification_uri),
];
let written = rsync_sync_into_current_store(
store,
rsync_base_uri,
None,
rsync_fetcher,
timing,
download_log,
)?;
if let Some(t) = timing.as_ref() {
t.record_count("repo_sync_rsync_peer_aligned_profile_total", 1);
}
if let Some(t) = timing.as_ref() {
t.record_count("repo_sync_rsync_fallback_ok_total", 1);
t.record_count("repo_sync_rsync_objects_written_total", written as u64);
}
Ok(RepoSyncResult {
source: RepoSyncSource::Rsync,
phase: RepoSyncPhase::RrdpFailedRsyncOk,
objects_written: written,
warnings,
})
}
}
}
_ => {
let written = rsync_sync_into_current_store(
store,
rsync_base_uri,
None,
rsync_fetcher,
timing,
download_log,
)?;
if let Some(t) = timing.as_ref() {
t.record_count("repo_sync_rsync_peer_aligned_profile_total", 1);
}
crate::logging::progress::emit!(
"repo_sync_rsync_direct",
serde_json::json!({
"rsync_base_uri": rsync_base_uri,
"objects_written": written,
}),
);
if let Some(t) = timing.as_ref() {
t.record_count("repo_sync_rsync_direct_total", 1);
t.record_count("repo_sync_rsync_objects_written_total", written as u64);
}
Ok(RepoSyncResult {
source: RepoSyncSource::Rsync,
phase: RepoSyncPhase::RsyncOnlyOk,
objects_written: written,
warnings: Vec::new(),
})
}
}
}
fn try_rrdp_sync(
store: &RocksStore,
notification_uri: &str,
current_repo_index: Option<&CurrentRepoIndexHandle>,
http_fetcher: &dyn HttpFetcher,
timing: Option<&TimingHandle>,
download_log: Option<&DownloadLogHandle>,
) -> Result<usize, RrdpSyncError> {
let notification_origin = RrdpOrigin::parse(notification_uri, RrdpResourceKind::Notification)?;
let notification_xml = {
let _step = timing
.as_ref()
.map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_notification"));
let _total = timing
.as_ref()
.map(|t| t.span_phase("rrdp_fetch_notification_total"));
let mut dl_span = download_log
.map(|dl| dl.span_download(AuditDownloadKind::RrdpNotification, notification_uri));
match http_fetcher.fetch_rrdp(
RrdpResourceKind::Notification,
notification_uri,
&notification_origin,
) {
Ok(v) => {
if let Some(t) = timing.as_ref() {
t.record_count("rrdp_notification_fetch_ok_total", 1);
}
if let Some(s) = dl_span.as_mut() {
s.set_bytes(v.len() as u64);
s.set_ok();
}
v
}
Err(RrdpFetchError::Fetch(e)) => {
if let Some(t) = timing.as_ref() {
t.record_count("rrdp_notification_fetch_fail_total", 1);
}
if let Some(s) = dl_span.as_mut() {
s.set_err(e.clone());
}
return Err(RrdpSyncError::Fetch(e));
}
Err(RrdpFetchError::Rrdp(e)) => {
if let Some(t) = timing.as_ref() {
t.record_count("rrdp_notification_fetch_fail_total", 1);
if e.is_cross_origin_violation() {
t.record_count("rrdp_rejected_cross_origin_total", 1);
}
}
if let Some(s) = dl_span.as_mut() {
s.set_err(e.to_string());
}
return Err(e.into());
}
}
};
if let Some(t) = timing.as_ref() {
t.record_count(
"rrdp_notification_bytes_total",
notification_xml.len() as u64,
);
}
sync_from_notification_with_timing_and_download_log(
store,
notification_uri,
current_repo_index,
&notification_xml,
http_fetcher,
timing,
download_log,
)
}
fn is_retryable_http_fetch_error(msg: &str) -> bool {
if msg.contains("http request failed:") || msg.contains("http read body failed:") {
return true;
}
let Some(rest) = msg.strip_prefix("http status ") else {
return false;
};
let code = rest
.split_whitespace()
.next()
.and_then(|s| s.parse::<u16>().ok())
.unwrap_or(0);
code == 408 || code == 429 || (500..600).contains(&code)
}
fn try_rrdp_sync_with_retry(
store: &RocksStore,
notification_uri: &str,
current_repo_index: Option<&CurrentRepoIndexHandle>,
http_fetcher: &dyn HttpFetcher,
timing: Option<&TimingHandle>,
download_log: Option<&DownloadLogHandle>,
) -> Result<usize, RrdpSyncError> {
let attempt = 1usize;
crate::logging::progress::emit!(
"rrdp_sync_attempt",
serde_json::json!({
"notify_uri": notification_uri,
"attempt": attempt,
}),
);
if let Some(t) = timing.as_ref() {
t.record_count("rrdp_retry_attempt_total", 1);
}
match try_rrdp_sync(
store,
notification_uri,
current_repo_index,
http_fetcher,
timing,
download_log,
) {
Ok(written) => {
crate::logging::progress::emit!(
"rrdp_sync_success",
serde_json::json!({
"notify_uri": notification_uri,
"attempt": attempt,
"objects_written": written,
}),
);
Ok(written)
}
Err(err) => {
let retryable = match &err {
RrdpSyncError::Fetch(msg) => is_retryable_http_fetch_error(msg),
_ => false,
};
crate::logging::progress::emit!(
"rrdp_sync_failed",
serde_json::json!({
"notify_uri": notification_uri,
"attempt": attempt,
"retryable": retryable,
"error": err.to_string(),
}),
);
if let Some(t) = timing.as_ref() {
match &err {
RrdpSyncError::Fetch(_) => t.record_count("rrdp_failed_fetch_total", 1),
RrdpSyncError::Rrdp(_) => t.record_count("rrdp_failed_protocol_total", 1),
RrdpSyncError::Storage(_) => t.record_count("rrdp_failed_storage_total", 1),
}
}
Err(err)
}
}
}
pub(crate) fn run_rrdp_transport(
store: &RocksStore,
notification_uri: &str,
current_repo_index: Option<&CurrentRepoIndexHandle>,
http_fetcher: &dyn HttpFetcher,
timing: Option<&TimingHandle>,
download_log: Option<&DownloadLogHandle>,
) -> Result<usize, RrdpSyncError> {
try_rrdp_sync_with_retry(
store,
notification_uri,
current_repo_index,
http_fetcher,
timing,
download_log,
)
}
fn rsync_sync_into_current_store(
store: &RocksStore,
rsync_base_uri: &str,
current_repo_index: Option<&CurrentRepoIndexHandle>,
rsync_fetcher: &dyn RsyncFetcher,
timing: Option<&TimingHandle>,
download_log: Option<&DownloadLogHandle>,
) -> Result<usize, RepoSyncError> {
let started = std::time::Instant::now();
let sync_scope_uri = rsync_fetcher.dedup_key(rsync_base_uri);
crate::logging::progress::emit!(
"rsync_sync_start",
serde_json::json!({
"rsync_base_uri": rsync_base_uri,
"sync_scope_uri": &sync_scope_uri,
}),
);
let _s = timing
.as_ref()
.map(|t| t.span_rrdp_repo_step(rsync_base_uri, "rsync_fetch_objects"));
let _p = timing.as_ref().map(|t| t.span_phase("rsync_fetch_total"));
let mut dl_span =
download_log.map(|dl| dl.span_download(AuditDownloadKind::Rsync, rsync_base_uri));
let mut new_set: HashSet<String> = HashSet::new();
let mut fetched_objects: Vec<(String, Vec<u8>)> = Vec::new();
let (object_count, bytes_total) =
match rsync_fetcher.visit_objects(rsync_base_uri, &mut |uri, bytes| {
new_set.insert(uri.clone());
fetched_objects.push((uri, bytes));
Ok(())
}) {
Ok(v) => {
if let Some(s) = dl_span.as_mut() {
s.set_objects(v.0 as u64, v.1);
s.set_bytes(v.1);
s.set_ok();
}
v
}
Err(e) => {
if let Some(s) = dl_span.as_mut() {
s.set_err(e.to_string());
}
return Err(e.into());
}
};
crate::logging::progress::emit!(
"rsync_sync_fetch_done",
serde_json::json!({
"rsync_base_uri": rsync_base_uri,
"sync_scope_uri": &sync_scope_uri,
"object_count": object_count,
"bytes_total": bytes_total,
"duration_ms": started.elapsed().as_millis() as u64,
}),
);
if let Some(t) = timing.as_ref() {
t.record_count("rsync_objects_fetched_total", object_count as u64);
t.record_count("rsync_objects_bytes_total", bytes_total);
}
drop(_p);
let existing_view = store
.list_repository_view_entries_with_prefix(&sync_scope_uri)
.map_err(|e| RepoSyncError::Storage(e.to_string()))?;
let _proj = timing
.as_ref()
.map(|t| t.span_phase("rsync_write_current_store_total"));
let prepared_bytes =
prepare_repo_bytes_batch_owned(fetched_objects).map_err(RepoSyncError::Storage)?;
let mut repository_view_entries = Vec::new();
for entry in existing_view {
if !new_set.contains(&entry.rsync_uri) {
repository_view_entries.push(build_repository_view_withdrawn_entry(
&sync_scope_uri,
&entry.rsync_uri,
entry.current_hash,
));
}
}
for uri in &new_set {
let current_hash = prepared_bytes
.uri_to_hash
.get(uri)
.cloned()
.ok_or_else(|| {
RepoSyncError::Storage(format!("missing raw_by_hash mapping for {uri}"))
})?;
repository_view_entries.push(build_repository_view_present_entry(
&sync_scope_uri,
uri,
&current_hash,
));
}
store
.put_blob_bytes_batch(&prepared_bytes.blobs_to_write)
.map_err(|e| RepoSyncError::Storage(e.to_string()))?;
store
.put_projection_batch(&repository_view_entries, &[], &[])
.map_err(|e| RepoSyncError::Storage(e.to_string()))?;
if let Some(index) = current_repo_index {
index
.write()
.map_err(|_| RepoSyncError::Storage("current repo index lock poisoned".to_string()))?
.apply_repository_view_entries(&repository_view_entries)
.map_err(RepoSyncError::Storage)?;
}
let total_duration_ms = started.elapsed().as_millis() as u64;
crate::logging::progress::emit!(
"rsync_sync_done",
serde_json::json!({
"rsync_base_uri": rsync_base_uri,
"sync_scope_uri": &sync_scope_uri,
"object_count": object_count,
"bytes_total": bytes_total,
"duration_ms": total_duration_ms,
}),
);
if (total_duration_ms as f64) / 1000.0 >= crate::logging::progress::slow_threshold_secs() {
crate::logging::progress::emit!(
"rsync_sync_slow",
serde_json::json!({
"rsync_base_uri": rsync_base_uri,
"sync_scope_uri": &sync_scope_uri,
"object_count": object_count,
"bytes_total": bytes_total,
"duration_ms": total_duration_ms,
}),
);
}
Ok(object_count)
}
pub(crate) fn run_rsync_transport(
store: &RocksStore,
rsync_base_uri: &str,
current_repo_index: Option<&CurrentRepoIndexHandle>,
rsync_fetcher: &dyn RsyncFetcher,
timing: Option<&TimingHandle>,
download_log: Option<&DownloadLogHandle>,
) -> Result<usize, RepoSyncError> {
rsync_sync_into_current_store(
store,
rsync_base_uri,
current_repo_index,
rsync_fetcher,
timing,
download_log,
)
}
#[cfg(test)]
#[path = "repo/tests.rs"]
mod tests;

View File

@@ -0,0 +1,2 @@
// Repository sync tests cover public RRDP/rsync transport behavior.
include!("tests_parts/setup_and_sync.rs");

View File

@@ -0,0 +1,371 @@
// Repository sync test group: setup and sync.
use super::*;
use crate::output::analysis::timing::{TimingHandle, TimingMeta};
use crate::repository::fetch::rsync::LocalDirRsyncFetcher;
use crate::repository::storage::RepositoryViewState;
use crate::repository::sync::rrdp::Fetcher as HttpFetcher;
use crate::repository::sync::store_projection::{build_repository_view_present_entry, compute_sha256_hex};
use sha2::Digest;
use std::sync::atomic::{AtomicUsize, Ordering};
struct DummyHttpFetcher;
impl HttpFetcher for DummyHttpFetcher {
fn fetch(&self, _url: &str) -> Result<Vec<u8>, String> {
panic!("http fetcher must not be used in rsync-only mode")
}
}
fn assert_current_object(store: &RocksStore, uri: &str, expected: &[u8]) {
assert_eq!(
store
.load_current_object_bytes_by_uri(uri)
.expect("load current object"),
Some(expected.to_vec())
);
}
#[test]
fn rsync_sync_uses_fetcher_dedup_scope_for_repository_view_projection() {
struct ScopeFetcher;
impl RsyncFetcher for ScopeFetcher {
fn fetch_objects(
&self,
_rsync_base_uri: &str,
) -> Result<Vec<(String, Vec<u8>)>, RsyncFetchError> {
Ok(vec![(
"rsync://example.net/repo/child/a.mft".to_string(),
b"manifest".to_vec(),
)])
}
fn dedup_key(&self, _rsync_base_uri: &str) -> String {
"rsync://example.net/repo/".to_string()
}
}
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(td.path()).expect("open rocksdb");
let seeded = build_repository_view_present_entry(
"rsync://example.net/repo/",
"rsync://example.net/repo/sibling/old.roa",
&compute_sha256_hex(b"old"),
);
store
.put_projection_batch(&[seeded], &[], &[])
.expect("seed repository view");
let fetcher = ScopeFetcher;
let written = rsync_sync_into_current_store(
&store,
"rsync://example.net/repo/child/",
None,
&fetcher,
None,
None,
)
.expect("sync ok");
assert_eq!(written, 1);
let entries = store
.list_repository_view_entries_with_prefix("rsync://example.net/repo/")
.expect("list repository view");
let sibling = entries
.iter()
.find(|entry| entry.rsync_uri == "rsync://example.net/repo/sibling/old.roa")
.expect("sibling entry exists");
assert_eq!(sibling.state, RepositoryViewState::Withdrawn);
let child = entries
.iter()
.find(|entry| entry.rsync_uri == "rsync://example.net/repo/child/a.mft")
.expect("child entry exists");
assert_eq!(child.state, RepositoryViewState::Present);
}
fn timing_to_json(temp_dir: &std::path::Path, timing: &TimingHandle) -> serde_json::Value {
let timing_path = temp_dir.join("timing_retry.json");
timing.write_json(&timing_path, 50).expect("write json");
serde_json::from_slice(&std::fs::read(&timing_path).expect("read json")).expect("parse json")
}
#[test]
fn rsync_sync_writes_current_store_and_records_counts() {
let temp = tempfile::tempdir().expect("tempdir");
let repo_dir = temp.path().join("repo");
std::fs::create_dir_all(repo_dir.join("sub")).expect("mkdir");
std::fs::write(repo_dir.join("a.mft"), b"mft").expect("write");
std::fs::write(repo_dir.join("sub").join("b.roa"), b"roa").expect("write");
std::fs::write(repo_dir.join("sub").join("c.cer"), b"cer").expect("write");
let store_dir = temp.path().join("db");
let store = RocksStore::open(&store_dir).expect("open rocksdb");
let timing = TimingHandle::new(TimingMeta {
recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(),
validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(),
tal_url: None,
db_path: Some(store_dir.to_string_lossy().into_owned()),
});
let policy = Policy {
sync_preference: SyncPreference::RsyncOnly,
..Policy::default()
};
let http = DummyHttpFetcher;
let rsync = LocalDirRsyncFetcher::new(&repo_dir);
let download_log = DownloadLogHandle::new();
let out = sync_publication_point(
&store,
&policy,
None,
"rsync://example.test/repo/",
&http,
&rsync,
Some(&timing),
Some(&download_log),
)
.expect("sync ok");
assert_eq!(out.source, RepoSyncSource::Rsync);
assert_eq!(out.objects_written, 3);
let events = download_log.snapshot_events();
assert_eq!(events.len(), 1);
assert_eq!(events[0].kind, AuditDownloadKind::Rsync);
assert!(events[0].success);
assert_eq!(events[0].bytes, Some(9));
let objects = events[0].objects.as_ref().expect("objects stat");
assert_eq!(objects.objects_count, 3);
assert_eq!(objects.objects_bytes_total, 9);
assert_current_object(&store, "rsync://example.test/repo/a.mft", b"mft");
assert_current_object(&store, "rsync://example.test/repo/sub/b.roa", b"roa");
assert_current_object(&store, "rsync://example.test/repo/sub/c.cer", b"cer");
let view = store
.get_repository_view_entry("rsync://example.test/repo/a.mft")
.expect("get repository view")
.expect("repository view entry present");
assert_eq!(
view.current_hash.as_deref(),
Some(hex::encode(sha2::Sha256::digest(b"mft")).as_str())
);
assert_eq!(
view.repository_source.as_deref(),
Some("rsync://example.test/repo/")
);
let current_bytes = store
.load_current_object_bytes_by_uri("rsync://example.test/repo/sub/b.roa")
.expect("load current bytes")
.expect("current object bytes exist");
assert_eq!(current_bytes, b"roa".to_vec());
assert!(
store
.get_raw_by_hash_entry(hex::encode(sha2::Sha256::digest(b"roa")).as_str())
.expect("get raw_by_hash")
.is_none()
);
let timing_path = temp.path().join("timing.json");
timing.write_json(&timing_path, 5).expect("write json");
let v: serde_json::Value =
serde_json::from_slice(&std::fs::read(&timing_path).expect("read json"))
.expect("parse json");
let counts = v.get("counts").expect("counts");
assert_eq!(
counts
.get("rsync_objects_fetched_total")
.and_then(|v| v.as_u64()),
Some(3)
);
assert_eq!(
counts
.get("rsync_objects_bytes_total")
.and_then(|v| v.as_u64()),
Some(3 * 3)
);
}
#[test]
fn rsync_second_sync_marks_missing_repository_view_entries_withdrawn() {
let temp = tempfile::tempdir().expect("tempdir");
let repo_dir = temp.path().join("repo");
std::fs::create_dir_all(repo_dir.join("sub")).expect("mkdir");
std::fs::write(repo_dir.join("a.mft"), b"mft-v1").expect("write a");
std::fs::write(repo_dir.join("sub").join("b.roa"), b"roa-v1").expect("write b");
let store_dir = temp.path().join("db");
let store = RocksStore::open(&store_dir).expect("open rocksdb");
let policy = Policy {
sync_preference: SyncPreference::RsyncOnly,
..Policy::default()
};
let http = DummyHttpFetcher;
let rsync = LocalDirRsyncFetcher::new(&repo_dir);
sync_publication_point(
&store,
&policy,
None,
"rsync://example.test/repo/",
&http,
&rsync,
None,
None,
)
.expect("first sync ok");
std::fs::remove_file(repo_dir.join("sub").join("b.roa")).expect("remove b");
std::fs::write(repo_dir.join("c.crl"), b"crl-v2").expect("write c");
sync_publication_point(
&store,
&policy,
None,
"rsync://example.test/repo/",
&http,
&rsync,
None,
None,
)
.expect("second sync ok");
let withdrawn = store
.get_repository_view_entry("rsync://example.test/repo/sub/b.roa")
.expect("get withdrawn repo view")
.expect("withdrawn entry exists");
assert_eq!(
withdrawn.state,
crate::repository::storage::RepositoryViewState::Withdrawn
);
assert_eq!(
withdrawn.repository_source.as_deref(),
Some("rsync://example.test/repo/")
);
let added = store
.get_repository_view_entry("rsync://example.test/repo/c.crl")
.expect("get added repo view")
.expect("added entry exists");
assert_eq!(added.state, crate::repository::storage::RepositoryViewState::Present);
}
#[test]
fn rrdp_fetch_error_falls_back_to_rsync_without_retry() {
let temp = tempfile::tempdir().expect("tempdir");
let store_dir = temp.path().join("db");
let store = RocksStore::open(&store_dir).expect("open rocksdb");
let timing = TimingHandle::new(TimingMeta {
recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(),
validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(),
tal_url: None,
db_path: Some(store_dir.to_string_lossy().into_owned()),
});
let notification_uri = "https://example.test/notification.xml";
let published_uri = "rsync://example.test/repo/a.mft";
let published_bytes = b"x";
struct AlwaysFailHttp {
notification_calls: AtomicUsize,
}
impl HttpFetcher for AlwaysFailHttp {
fn fetch(&self, _uri: &str) -> Result<Vec<u8>, String> {
self.notification_calls.fetch_add(1, Ordering::SeqCst);
Err("http request failed: simulated transient".to_string())
}
}
struct SingleObjectRsync {
uri: String,
bytes: Vec<u8>,
}
impl RsyncFetcher for SingleObjectRsync {
fn fetch_objects(
&self,
_rsync_base_uri: &str,
) -> Result<Vec<(String, Vec<u8>)>, RsyncFetchError> {
Ok(vec![(self.uri.clone(), self.bytes.clone())])
}
}
let http = AlwaysFailHttp {
notification_calls: AtomicUsize::new(0),
};
let policy = Policy {
sync_preference: SyncPreference::RrdpThenRsync,
..Policy::default()
};
let download_log = DownloadLogHandle::new();
let out = sync_publication_point(
&store,
&policy,
Some(notification_uri),
"rsync://example.test/repo/",
&http,
&SingleObjectRsync {
uri: published_uri.to_string(),
bytes: published_bytes.to_vec(),
},
Some(&timing),
Some(&download_log),
)
.expect("sync ok");
assert_eq!(out.source, RepoSyncSource::Rsync);
assert_current_object(&store, published_uri, published_bytes);
assert_eq!(http.notification_calls.load(Ordering::SeqCst), 1);
let events = download_log.snapshot_events();
assert_eq!(events.len(), 2, "expected 1x notification + 1x rsync");
assert_eq!(
events
.iter()
.filter(|e| e.kind == AuditDownloadKind::RrdpNotification)
.count(),
1
);
assert_eq!(
events
.iter()
.filter(|e| e.kind == AuditDownloadKind::RrdpNotification && !e.success)
.count(),
1
);
assert_eq!(
events
.iter()
.filter(|e| e.kind == AuditDownloadKind::Rsync)
.count(),
1
);
let v = timing_to_json(temp.path(), &timing);
let counts = v.get("counts").expect("counts");
assert_eq!(
counts
.get("rrdp_retry_attempt_total")
.and_then(|v| v.as_u64()),
Some(1)
);
assert_eq!(
counts
.get("repo_sync_rrdp_failed_total")
.and_then(|v| v.as_u64()),
Some(1)
);
assert_eq!(
counts
.get("repo_sync_rsync_fallback_ok_total")
.and_then(|v| v.as_u64()),
Some(1)
);
}

View File

@@ -0,0 +1,56 @@
use crate::output::analysis::timing::TimingHandle;
use crate::output::audit::AuditDownloadKind;
use crate::output::audit_downloads::DownloadLogHandle;
use crate::repository::current_repo_index::CurrentRepoIndexHandle;
use crate::repository::storage::{
RepositoryViewEntry, RepositoryViewState, RocksStore, RrdpDeltaOp, RrdpSourceSyncState,
};
mod snapshot_apply;
use crate::repository::sync::store_projection::{
build_repository_view_present_entry, build_repository_view_withdrawn_entry,
current_rrdp_owner_is, ensure_rrdp_uri_can_be_owned_by, put_repository_view_present,
put_repository_view_withdrawn, put_rrdp_source_member_present,
put_rrdp_source_member_withdrawn, put_rrdp_uri_owner_active, put_rrdp_uri_owner_withdrawn,
update_rrdp_source_record_on_success, upsert_repo_blob_bytes,
};
use base64::Engine;
use serde::{Deserialize, Serialize};
use sha2::Digest;
use std::fmt;
use std::io::Write;
use url::Url;
use uuid::Uuid;
const RRDP_XMLNS: &str = "http://www.ripe.net/rpki/rrdp";
const RRDP_SNAPSHOT_APPLY_BATCH_SIZE: usize = 1024;
/// The RRDP object currently being fetched. Keeping this in protocol errors
/// makes an origin rejection actionable without treating it as a transient
/// transport failure.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RrdpResourceKind {
Notification,
Snapshot,
Delta,
}
impl fmt::Display for RrdpResourceKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
Self::Notification => "notification",
Self::Snapshot => "snapshot",
Self::Delta => "delta",
})
}
}
include!("rrdp/models_and_parsing.rs");
include!("rrdp/snapshot_sync.rs");
include!("rrdp/notification_sync.rs");
include!("rrdp/delta.rs");
include!("rrdp/parse_helpers.rs");
#[cfg(test)]
#[path = "rrdp/tests.rs"]
mod tests;

View File

@@ -0,0 +1,272 @@
// Current-repository hydration and delta application.
fn hydrate_current_repo_index_from_rrdp_members(
store: &RocksStore,
notification_uri: &str,
current_repo_index: Option<&CurrentRepoIndexHandle>,
) -> Result<usize, RrdpSyncError> {
let Some(index) = current_repo_index else {
return Ok(0);
};
let members = store
.list_current_rrdp_source_members(notification_uri)
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
if members.is_empty() {
return Ok(0);
}
let mut entries = Vec::with_capacity(members.len());
for member in members {
let current_hash = member.current_hash.ok_or_else(|| {
RrdpSyncError::Storage(format!(
"rrdp source member missing current_hash for current object {}",
member.rsync_uri
))
})?;
entries.push(RepositoryViewEntry {
rsync_uri: member.rsync_uri,
current_hash: Some(current_hash),
repository_source: Some(notification_uri.to_string()),
object_type: member.object_type,
state: RepositoryViewState::Present,
});
}
index
.write()
.map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))?
.apply_repository_view_entries(&entries)
.map_err(RrdpSyncError::Storage)?;
Ok(entries.len())
}
fn apply_delta(
store: &RocksStore,
notification_uri: &str,
current_repo_index: Option<&CurrentRepoIndexHandle>,
delta_xml: &[u8],
expected_hash_sha256: [u8; 32],
expected_session_id: Uuid,
expected_serial: u64,
) -> Result<usize, RrdpSyncError> {
let computed = sha2::Sha256::digest(delta_xml);
if computed.as_slice() != expected_hash_sha256.as_slice() {
return Err(RrdpError::DeltaHashMismatch.into());
}
let delta = parse_delta_file(delta_xml)?;
if delta.session_id != expected_session_id {
return Err(RrdpError::DeltaSessionIdMismatch {
expected: expected_session_id.to_string(),
got: delta.session_id.to_string(),
}
.into());
}
if delta.serial != expected_serial {
return Err(RrdpError::DeltaSerialMismatch {
expected: expected_serial,
got: delta.serial,
}
.into());
}
enum DeltaProjectionEffect {
Upsert {
rsync_uri: String,
bytes: Vec<u8>,
},
Delete {
rsync_uri: String,
previous_hash: String,
},
}
let session_id = expected_session_id.to_string();
let mut ops: Vec<RrdpDeltaOp> = Vec::with_capacity(delta.elements.len());
let mut projection: Vec<DeltaProjectionEffect> = Vec::with_capacity(delta.elements.len());
for e in delta.elements {
match e {
DeltaElement::Publish {
uri,
hash_sha256: Some(old_hash),
bytes,
} => {
let is_member = store
.is_current_rrdp_source_member(notification_uri, uri.as_str())
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
if !is_member {
return Err(RrdpError::DeltaTargetNotFromRepository { rsync_uri: uri }.into());
}
ensure_rrdp_uri_can_be_owned_by(store, notification_uri, uri.as_str())
.map_err(RrdpSyncError::Storage)?;
let old_bytes = store
.load_current_object_bytes_by_uri(uri.as_str())
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?
.ok_or_else(|| RrdpError::DeltaTargetMissing {
rsync_uri: uri.clone(),
})?;
let old_computed = sha2::Sha256::digest(old_bytes.as_slice());
if old_computed.as_slice() != old_hash.as_slice() {
return Err(RrdpError::DeltaTargetHashMismatch { rsync_uri: uri }.into());
}
ops.push(RrdpDeltaOp::Upsert {
rsync_uri: uri.clone(),
bytes: bytes.clone(),
});
projection.push(DeltaProjectionEffect::Upsert {
rsync_uri: uri,
bytes,
});
}
DeltaElement::Publish {
uri,
hash_sha256: None,
bytes,
} => {
let is_member = store
.is_current_rrdp_source_member(notification_uri, uri.as_str())
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
if is_member {
return Err(
RrdpError::DeltaPublishWithoutHashForExisting { rsync_uri: uri }.into(),
);
}
ensure_rrdp_uri_can_be_owned_by(store, notification_uri, uri.as_str())
.map_err(RrdpSyncError::Storage)?;
ops.push(RrdpDeltaOp::Upsert {
rsync_uri: uri.clone(),
bytes: bytes.clone(),
});
projection.push(DeltaProjectionEffect::Upsert {
rsync_uri: uri,
bytes,
});
}
DeltaElement::Withdraw { uri, hash_sha256 } => {
let is_member = store
.is_current_rrdp_source_member(notification_uri, uri.as_str())
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
if !is_member {
return Err(RrdpError::DeltaTargetNotFromRepository { rsync_uri: uri }.into());
}
ensure_rrdp_uri_can_be_owned_by(store, notification_uri, uri.as_str())
.map_err(RrdpSyncError::Storage)?;
let old_bytes = store
.load_current_object_bytes_by_uri(uri.as_str())
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?
.ok_or_else(|| RrdpError::DeltaTargetMissing {
rsync_uri: uri.clone(),
})?;
let old_computed = sha2::Sha256::digest(old_bytes.as_slice());
if old_computed.as_slice() != hash_sha256.as_slice() {
return Err(RrdpError::DeltaTargetHashMismatch { rsync_uri: uri }.into());
}
let previous_hash = hex::encode(old_computed);
ops.push(RrdpDeltaOp::Delete {
rsync_uri: uri.clone(),
});
projection.push(DeltaProjectionEffect::Delete {
rsync_uri: uri,
previous_hash,
});
}
}
}
for effect in projection {
match effect {
DeltaProjectionEffect::Upsert { rsync_uri, bytes } => {
let current_hash =
upsert_repo_blob_bytes(store, &bytes).map_err(RrdpSyncError::Storage)?;
put_repository_view_present(store, notification_uri, &rsync_uri, &current_hash)
.map_err(RrdpSyncError::Storage)?;
if let Some(index) = current_repo_index {
let entry = build_repository_view_present_entry(
notification_uri,
&rsync_uri,
&current_hash,
);
index
.write()
.map_err(|_| {
RrdpSyncError::Storage("current repo index lock poisoned".to_string())
})?
.apply_repository_view_entries(&[entry])
.map_err(RrdpSyncError::Storage)?;
}
put_rrdp_source_member_present(
store,
notification_uri,
&session_id,
expected_serial,
&rsync_uri,
&current_hash,
)
.map_err(RrdpSyncError::Storage)?;
put_rrdp_uri_owner_active(
store,
notification_uri,
&session_id,
expected_serial,
&rsync_uri,
&current_hash,
)
.map_err(RrdpSyncError::Storage)?;
}
DeltaProjectionEffect::Delete {
rsync_uri,
previous_hash,
} => {
put_rrdp_source_member_withdrawn(
store,
notification_uri,
&session_id,
expected_serial,
&rsync_uri,
Some(previous_hash.clone()),
)
.map_err(RrdpSyncError::Storage)?;
if current_rrdp_owner_is(store, notification_uri, &rsync_uri)
.map_err(RrdpSyncError::Storage)?
{
put_repository_view_withdrawn(
store,
notification_uri,
&rsync_uri,
Some(previous_hash.clone()),
)
.map_err(RrdpSyncError::Storage)?;
if let Some(index) = current_repo_index {
let entry = build_repository_view_withdrawn_entry(
notification_uri,
&rsync_uri,
Some(previous_hash.clone()),
);
index
.write()
.map_err(|_| {
RrdpSyncError::Storage(
"current repo index lock poisoned".to_string(),
)
})?
.apply_repository_view_entries(&[entry])
.map_err(RrdpSyncError::Storage)?;
}
put_rrdp_uri_owner_withdrawn(
store,
notification_uri,
&session_id,
expected_serial,
&rsync_uri,
Some(previous_hash),
)
.map_err(RrdpSyncError::Storage)?;
}
}
}
}
Ok(ops.len())
}

View File

@@ -0,0 +1,639 @@
// RRDP error/state models and notification/delta parsing.
#[derive(Debug, thiserror::Error)]
pub enum RrdpError {
#[error("RRDP XML must be US-ASCII encoded (RFC 8182 §3.5.1.3, §3.5.2.3), got non-ASCII bytes")]
NotAscii,
#[error("RRDP XML parse error: {0} (RFC 8182 §3.5.1.3, §3.5.2.3)")]
Xml(String),
#[error(
"RRDP root element must be <notification>, <snapshot>, or <delta>, got <{0}> (RFC 8182 §3.5.1.3, §3.5.2.3, §3.5.3.3)"
)]
UnexpectedRoot(String),
#[error("RRDP XML namespace must be {RRDP_XMLNS}, got {0} (RFC 8182 §3.5.1.3, §3.5.2.3)")]
InvalidNamespace(String),
#[error("RRDP version must be 1, got {0} (RFC 8182 §3.5.1.3, §3.5.2.3)")]
InvalidVersion(String),
#[error("RRDP session_id invalid UUID: {0} (RFC 8182 §3.5.1.3)")]
InvalidSessionId(String),
#[error("RRDP serial invalid unsigned integer: {0} (RFC 8182 §3.5.1.3)")]
InvalidSerial(String),
#[error("notification must contain exactly one <snapshot> element (RFC 8182 §3.5.1.3)")]
SnapshotCountInvalid,
#[error("snapshot/@uri missing (RFC 8182 §3.5.1.3)")]
SnapshotUriMissing,
#[error("snapshot/@hash missing (RFC 8182 §3.5.1.3)")]
SnapshotHashMissing,
#[error("snapshot/@hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.1.3)")]
SnapshotHashInvalid(String),
#[error("delta/@serial missing in notification (RFC 8182 §3.5.1.3)")]
DeltaRefSerialMissing,
#[error("delta/@uri missing in notification (RFC 8182 §3.5.1.3)")]
DeltaRefUriMissing,
#[error("delta/@hash missing in notification (RFC 8182 §3.5.1.3)")]
DeltaRefHashMissing,
#[error("delta/@hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.1.3)")]
DeltaRefHashInvalid(String),
#[error("delta/@serial duplicates in notification: {0} (RFC 8182 §3.5.1.3)")]
DeltaRefSerialDuplicate(u64),
#[error(
"delta/@serial must be <= notification/@serial: delta={delta_serial} notification={notification_serial} (RFC 8182 §3.5.1.3)"
)]
DeltaRefSerialTooHigh {
delta_serial: u64,
notification_serial: u64,
},
#[error(
"notification contains deltas but does not end at notification/@serial: max_delta={max_delta_serial} notification={notification_serial} (RFC 8182 §3.5.1.3)"
)]
DeltaRefChainDoesNotEndAtNotificationSerial {
max_delta_serial: u64,
notification_serial: u64,
},
#[error(
"notification delta chain not contiguous: missing serial={missing_serial} (range {min_serial}..{notification_serial}) (RFC 8182 §3.5.1.3)"
)]
DeltaRefChainNotContiguous {
min_serial: u64,
notification_serial: u64,
missing_serial: u64,
},
#[error("snapshot file hash mismatch (RFC 8182 §3.5.1.3)")]
SnapshotHashMismatch,
#[error("snapshot session_id mismatch: expected {expected}, got {got} (RFC 8182 §3.5.2.3)")]
SnapshotSessionIdMismatch { expected: String, got: String },
#[error("snapshot serial mismatch: expected {expected}, got {got} (RFC 8182 §3.5.2.3)")]
SnapshotSerialMismatch { expected: u64, got: u64 },
#[error("delta file hash mismatch (RFC 8182 §3.4.2; RFC 8182 §3.5.1.3)")]
DeltaHashMismatch,
#[error("delta session_id mismatch: expected {expected}, got {got} (RFC 8182 §3.5.3.3)")]
DeltaSessionIdMismatch { expected: String, got: String },
#[error("delta serial mismatch: expected {expected}, got {got} (RFC 8182 §3.5.3.3)")]
DeltaSerialMismatch { expected: u64, got: u64 },
#[error("notification serial moved backwards: old={old} new={new} (RFC 8182 §3.4.1)")]
NotificationSerialRollback { old: u64, new: u64 },
#[error("delta publish without @hash for existing object: {rsync_uri} (RFC 8182 §3.4.2)")]
DeltaPublishWithoutHashForExisting { rsync_uri: String },
#[error(
"delta withdraw/replace target not from this repository server: {rsync_uri} (RFC 8182 §3.4.2)"
)]
DeltaTargetNotFromRepository { rsync_uri: String },
#[error("delta withdraw/replace target missing in local cache: {rsync_uri} (RFC 8182 §3.4.2)")]
DeltaTargetMissing { rsync_uri: String },
#[error("delta withdraw/replace target hash mismatch: {rsync_uri} (RFC 8182 §3.4.2)")]
DeltaTargetHashMismatch { rsync_uri: String },
#[error("publish/@uri missing (RFC 8182 §3.5.2.3)")]
PublishUriMissing,
#[error("publish element missing base64 content (RFC 8182 §3.5.2.3)")]
PublishContentMissing,
#[error("publish base64 decode failed (RFC 8182 §3.5.2.3): {0}")]
PublishBase64(String),
#[error("delta file <publish> missing @uri (RFC 8182 §3.5.3.3)")]
DeltaPublishUriMissing,
#[error("delta file <publish> base64 content missing (RFC 8182 §3.5.3.3)")]
DeltaPublishContentMissing,
#[error("delta file <publish> base64 decode failed (RFC 8182 §3.5.3.3): {0}")]
DeltaPublishBase64(String),
#[error(
"delta file <publish> @hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.3.3)"
)]
DeltaPublishHashInvalid(String),
#[error("delta file <withdraw> missing @uri (RFC 8182 §3.5.3.3)")]
DeltaWithdrawUriMissing,
#[error("delta file <withdraw> missing @hash (RFC 8182 §3.5.3.3)")]
DeltaWithdrawHashMissing,
#[error(
"delta file <withdraw> @hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.3.3)"
)]
DeltaWithdrawHashInvalid(String),
#[error("delta file <withdraw> must not contain text content (RFC 8182 §3.5.3.3)")]
DeltaWithdrawUnexpectedContent,
#[error("delta file must contain at least one publish/withdraw element (RFC 8182 §3.5.3.3)")]
DeltaNoElements,
#[error("delta file contains unexpected element <{0}> (RFC 8182 §3.5.3.3)")]
DeltaUnexpectedElement(String),
#[error("invalid RRDP {resource} URI: {detail} (RFC 9674 §3.2)")]
InvalidRrdpUri {
resource: RrdpResourceKind,
detail: String,
},
#[error(
"RRDP {resource} URI crosses notification origin: notification={notification_origin}, resource={resource_origin} (RFC 9674 §3.2)"
)]
CrossOriginReference {
resource: RrdpResourceKind,
notification_origin: String,
resource_origin: String,
},
#[error(
"RRDP {resource} redirect crosses notification origin: notification={notification_origin}, redirect={redirect_origin} (RFC 9674 §3.2)"
)]
CrossOriginRedirect {
resource: RrdpResourceKind,
notification_origin: String,
redirect_origin: String,
},
#[error("RRDP {resource} redirect has no Location header (RFC 9674 §3.2)")]
RedirectLocationMissing { resource: RrdpResourceKind },
#[error("RRDP {resource} redirect Location is invalid: {location} (RFC 9674 §3.2)")]
InvalidRedirectLocation {
resource: RrdpResourceKind,
location: String,
},
#[error("RRDP {resource} redirect limit exceeded ({max}) (RFC 9674 §3.2)")]
RedirectLimitExceeded {
resource: RrdpResourceKind,
max: usize,
},
}
impl RrdpError {
pub fn is_cross_origin_violation(&self) -> bool {
matches!(
self,
Self::CrossOriginReference { .. } | Self::CrossOriginRedirect { .. }
)
}
}
/// RFC 9674 §3.2 compares scheme, host name, and port. `Url` normalizes an
/// omitted well-known port, so `https://example/` and `https://example:443/`
/// intentionally have the same effective origin.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RrdpOrigin {
scheme: String,
host: String,
port: u16,
}
impl fmt::Display for RrdpOrigin {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}://{}:{}", self.scheme, self.host, self.port)
}
}
impl RrdpOrigin {
pub fn parse(uri: &str, resource: RrdpResourceKind) -> Result<Self, RrdpError> {
let url = Url::parse(uri).map_err(|e| RrdpError::InvalidRrdpUri {
resource,
detail: e.to_string(),
})?;
Self::from_url(&url, resource)
}
pub fn from_url(url: &Url, resource: RrdpResourceKind) -> Result<Self, RrdpError> {
if !matches!(url.scheme(), "http" | "https") {
return Err(RrdpError::InvalidRrdpUri {
resource,
detail: format!("unsupported scheme {}", url.scheme()),
});
}
if !url.username().is_empty() || url.password().is_some() {
return Err(RrdpError::InvalidRrdpUri {
resource,
detail: "userinfo is not permitted".to_string(),
});
}
let host = url.host_str().ok_or_else(|| RrdpError::InvalidRrdpUri {
resource,
detail: "host is missing".to_string(),
})?;
let port = url
.port_or_known_default()
.ok_or_else(|| RrdpError::InvalidRrdpUri {
resource,
detail: "port is missing or unknown".to_string(),
})?;
Ok(Self {
scheme: url.scheme().to_ascii_lowercase(),
host: host.to_ascii_lowercase(),
port,
})
}
}
#[derive(Debug, thiserror::Error)]
pub enum RrdpSyncError {
#[error("{0}")]
Rrdp(#[from] RrdpError),
#[error("fetch failed: {0}")]
Fetch(String),
#[error("storage error: {0}")]
Storage(String),
}
#[derive(Debug, thiserror::Error)]
pub enum RrdpFetchError {
#[error("{0}")]
Rrdp(#[from] RrdpError),
#[error("{0}")]
Fetch(String),
}
pub type RrdpSyncResult<T> = Result<T, RrdpSyncError>;
pub trait Fetcher: Send + Sync {
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String>;
fn fetch_to_writer(&self, uri: &str, out: &mut dyn Write) -> Result<u64, String> {
let bytes = self.fetch(uri)?;
out.write_all(&bytes)
.map_err(|e| format!("write sink failed: {e}"))?;
Ok(bytes.len() as u64)
}
/// RRDP-only fetch path. Implementations that do not perform HTTP may use
/// the compatibility default; HTTP implementations must enforce redirect
/// origin checks before issuing the next request.
fn fetch_rrdp(
&self,
_resource: RrdpResourceKind,
uri: &str,
_notification_origin: &RrdpOrigin,
) -> Result<Vec<u8>, RrdpFetchError> {
self.fetch(uri).map_err(RrdpFetchError::Fetch)
}
fn fetch_rrdp_to_writer(
&self,
_resource: RrdpResourceKind,
uri: &str,
_notification_origin: &RrdpOrigin,
out: &mut dyn Write,
) -> Result<u64, RrdpFetchError> {
self.fetch_to_writer(uri, out)
.map_err(RrdpFetchError::Fetch)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RrdpState {
pub session_id: String,
pub serial: u64,
}
impl RrdpState {
pub fn encode(&self) -> Result<Vec<u8>, String> {
serde_cbor::to_vec(self).map_err(|e| e.to_string())
}
pub fn decode(bytes: &[u8]) -> Result<Self, String> {
serde_cbor::from_slice(bytes).map_err(|e| e.to_string())
}
}
pub(crate) fn load_rrdp_local_state(
store: &RocksStore,
notification_uri: &str,
) -> Result<Option<RrdpState>, String> {
if let Some(record) = store
.get_rrdp_source_record(notification_uri)
.map_err(|e| e.to_string())?
&& let (Some(session_id), Some(serial)) = (record.last_session_id, record.last_serial) {
return Ok(Some(RrdpState { session_id, serial }));
}
Ok(None)
}
pub(crate) fn persist_rrdp_local_state(
store: &RocksStore,
notification_uri: &str,
state: &RrdpState,
sync_state: RrdpSourceSyncState,
last_snapshot_uri: Option<&str>,
last_snapshot_hash_hex: Option<&str>,
) -> Result<(), String> {
update_rrdp_source_record_on_success(
store,
notification_uri,
state.session_id.as_str(),
state.serial,
sync_state,
last_snapshot_uri,
last_snapshot_hash_hex,
)
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct NotificationSnapshot {
pub session_id: Uuid,
pub serial: u64,
pub snapshot_uri: String,
pub snapshot_hash_sha256: [u8; 32],
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct NotificationDeltaRef {
pub serial: u64,
pub uri: String,
pub hash_sha256: [u8; 32],
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Notification {
pub session_id: Uuid,
pub serial: u64,
pub snapshot_uri: String,
pub snapshot_hash_sha256: [u8; 32],
/// Deltas referenced by the notification file, sorted by serial ascending.
///
/// If present, this list is guaranteed to be contiguous and to end at `serial`
/// (RFC 8182 §3.5.1.3).
pub deltas: Vec<NotificationDeltaRef>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum DeltaElement {
Publish {
uri: String,
hash_sha256: Option<[u8; 32]>,
bytes: Vec<u8>,
},
Withdraw {
uri: String,
hash_sha256: [u8; 32],
},
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DeltaFile {
pub session_id: Uuid,
pub serial: u64,
pub elements: Vec<DeltaElement>,
}
pub fn parse_notification(xml: &[u8]) -> Result<Notification, RrdpError> {
let doc = parse_rrdp_xml(xml)?;
let root = doc.root_element();
if root.tag_name().name() != "notification" {
return Err(RrdpError::UnexpectedRoot(
root.tag_name().name().to_string(),
));
}
validate_root_common(&root)?;
let session_id = parse_uuid_attr(&root, "session_id")?;
let serial = parse_u64_attr(&root, "serial")?;
let snapshots: Vec<_> = root
.children()
.filter(|n| n.is_element() && n.tag_name().name() == "snapshot")
.collect();
if snapshots.len() != 1 {
return Err(RrdpError::SnapshotCountInvalid);
}
let snapshot = snapshots[0];
let snapshot_uri = snapshot
.attribute("uri")
.ok_or(RrdpError::SnapshotUriMissing)?
.to_string();
let snapshot_hash_hex = snapshot
.attribute("hash")
.ok_or(RrdpError::SnapshotHashMissing)?;
let snapshot_hash_sha256 = parse_sha256_hex_snapshot(snapshot_hash_hex)?;
let mut deltas: Vec<NotificationDeltaRef> = Vec::new();
for d in root
.children()
.filter(|n| n.is_element() && n.tag_name().name() == "delta")
{
let delta_serial = d
.attribute("serial")
.ok_or(RrdpError::DeltaRefSerialMissing)?;
let delta_serial = parse_u64_str(delta_serial)?;
if delta_serial > serial {
return Err(RrdpError::DeltaRefSerialTooHigh {
delta_serial,
notification_serial: serial,
});
}
let uri = d.attribute("uri").ok_or(RrdpError::DeltaRefUriMissing)?;
let hash = d.attribute("hash").ok_or(RrdpError::DeltaRefHashMissing)?;
let hash_sha256 = parse_sha256_hex_delta_ref(hash)?;
deltas.push(NotificationDeltaRef {
serial: delta_serial,
uri: uri.to_string(),
hash_sha256,
});
}
if !deltas.is_empty() {
let mut seen: std::collections::HashSet<u64> = std::collections::HashSet::new();
let mut min_serial = u64::MAX;
let mut max_serial = 0u64;
for d in &deltas {
if !seen.insert(d.serial) {
return Err(RrdpError::DeltaRefSerialDuplicate(d.serial));
}
min_serial = min_serial.min(d.serial);
max_serial = max_serial.max(d.serial);
}
if max_serial != serial {
return Err(RrdpError::DeltaRefChainDoesNotEndAtNotificationSerial {
max_delta_serial: max_serial,
notification_serial: serial,
});
}
for s in min_serial..=serial {
if !seen.contains(&s) {
return Err(RrdpError::DeltaRefChainNotContiguous {
min_serial,
notification_serial: serial,
missing_serial: s,
});
}
}
deltas.sort_by_key(|d| d.serial);
}
Ok(Notification {
session_id,
serial,
snapshot_uri,
snapshot_hash_sha256,
deltas,
})
}
pub fn parse_notification_snapshot(xml: &[u8]) -> Result<NotificationSnapshot, RrdpError> {
let n = parse_notification(xml)?;
Ok(NotificationSnapshot {
session_id: n.session_id,
serial: n.serial,
snapshot_uri: n.snapshot_uri,
snapshot_hash_sha256: n.snapshot_hash_sha256,
})
}
/// Validate every URI embedded in a notification before reading mutable local
/// RRDP state or fetching a snapshot/delta. RFC 9674 §3.2 requires the
/// notification, snapshot, and all delta URIs to share one origin.
pub fn validate_notification_references(
notification_uri: &str,
notification: &Notification,
) -> Result<RrdpOrigin, RrdpError> {
let notification_origin = RrdpOrigin::parse(notification_uri, RrdpResourceKind::Notification)?;
validate_reference_origin(
&notification_origin,
&notification.snapshot_uri,
RrdpResourceKind::Snapshot,
)?;
for delta in &notification.deltas {
validate_reference_origin(&notification_origin, &delta.uri, RrdpResourceKind::Delta)?;
}
Ok(notification_origin)
}
fn validate_reference_origin(
notification_origin: &RrdpOrigin,
uri: &str,
resource: RrdpResourceKind,
) -> Result<(), RrdpError> {
let resource_origin = RrdpOrigin::parse(uri, resource)?;
if &resource_origin != notification_origin {
crate::logging::progress::emit!(
"rrdp_cross_origin_rejected",
serde_json::json!({
"resource": resource.to_string(),
"notification_origin": notification_origin.to_string(),
"resource_origin": resource_origin.to_string(),
"reason": "direct_reference",
"rfc": "RFC 9674 §3.2",
}),
);
return Err(RrdpError::CrossOriginReference {
resource,
notification_origin: notification_origin.to_string(),
resource_origin: resource_origin.to_string(),
});
}
Ok(())
}
pub fn parse_delta_file(xml: &[u8]) -> Result<DeltaFile, RrdpError> {
let doc = parse_rrdp_xml(xml)?;
let root = doc.root_element();
if root.tag_name().name() != "delta" {
return Err(RrdpError::UnexpectedRoot(
root.tag_name().name().to_string(),
));
}
validate_root_common(&root)?;
let session_id = parse_uuid_attr(&root, "session_id")?;
let serial = parse_u64_attr(&root, "serial")?;
let mut elements: Vec<DeltaElement> = Vec::new();
for child in root.children().filter(|n| n.is_element()) {
match child.tag_name().name() {
"publish" => {
let uri = child
.attribute("uri")
.ok_or(RrdpError::DeltaPublishUriMissing)?
.to_string();
let hash_sha256 = child
.attribute("hash")
.map(parse_sha256_hex_delta_publish)
.transpose()?;
let content_b64 =
collect_element_text(&child).ok_or(RrdpError::DeltaPublishContentMissing)?;
let content_b64 = strip_all_ascii_whitespace(&content_b64);
if content_b64.is_empty() {
return Err(RrdpError::DeltaPublishContentMissing);
}
let bytes = base64::engine::general_purpose::STANDARD
.decode(content_b64.as_bytes())
.map_err(|e| RrdpError::DeltaPublishBase64(e.to_string()))?;
elements.push(DeltaElement::Publish {
uri,
hash_sha256,
bytes,
});
}
"withdraw" => {
let uri = child
.attribute("uri")
.ok_or(RrdpError::DeltaWithdrawUriMissing)?
.to_string();
let hash = child
.attribute("hash")
.ok_or(RrdpError::DeltaWithdrawHashMissing)?;
let hash_sha256 = parse_sha256_hex_delta_withdraw(hash)?;
if let Some(s) = collect_element_text(&child)
&& !strip_all_ascii_whitespace(&s).is_empty() {
return Err(RrdpError::DeltaWithdrawUnexpectedContent);
}
elements.push(DeltaElement::Withdraw { uri, hash_sha256 });
}
other => return Err(RrdpError::DeltaUnexpectedElement(other.to_string())),
}
}
if elements.is_empty() {
return Err(RrdpError::DeltaNoElements);
}
Ok(DeltaFile {
session_id,
serial,
elements,
})
}

View File

@@ -0,0 +1,375 @@
// RRDP notification, delta-chain, and state synchronization.
pub fn sync_from_notification(
store: &RocksStore,
notification_uri: &str,
notification_xml: &[u8],
fetcher: &dyn Fetcher,
) -> RrdpSyncResult<usize> {
sync_from_notification_inner(
store,
notification_uri,
None,
notification_xml,
fetcher,
None,
None,
)
}
pub fn sync_from_notification_with_timing(
store: &RocksStore,
notification_uri: &str,
notification_xml: &[u8],
fetcher: &dyn Fetcher,
timing: Option<&TimingHandle>,
) -> RrdpSyncResult<usize> {
sync_from_notification_inner(
store,
notification_uri,
None,
notification_xml,
fetcher,
timing,
None,
)
}
pub fn sync_from_notification_with_timing_and_download_log(
store: &RocksStore,
notification_uri: &str,
current_repo_index: Option<&CurrentRepoIndexHandle>,
notification_xml: &[u8],
fetcher: &dyn Fetcher,
timing: Option<&TimingHandle>,
download_log: Option<&DownloadLogHandle>,
) -> RrdpSyncResult<usize> {
sync_from_notification_inner(
store,
notification_uri,
current_repo_index,
notification_xml,
fetcher,
timing,
download_log,
)
}
fn sync_from_notification_inner(
store: &RocksStore,
notification_uri: &str,
current_repo_index: Option<&CurrentRepoIndexHandle>,
notification_xml: &[u8],
fetcher: &dyn Fetcher,
timing: Option<&TimingHandle>,
download_log: Option<&DownloadLogHandle>,
) -> RrdpSyncResult<usize> {
let _parse_step = timing
.as_ref()
.map(|t| t.span_rrdp_repo_step(notification_uri, "parse_notification"));
let _parse_total = timing
.as_ref()
.map(|t| t.span_phase("rrdp_parse_notification_total"));
let notif = parse_notification(notification_xml)?;
let notification_origin = match validate_notification_references(notification_uri, &notif) {
Ok(origin) => origin,
Err(e) => {
if let Some(t) = timing.as_ref() {
t.record_count(
"rrdp_rejected_cross_origin_total",
e.is_cross_origin_violation() as u64,
);
}
return Err(e.into());
}
};
drop(_parse_step);
drop(_parse_total);
if let Some(t) = timing.as_ref() {
t.record_count(
"rrdp_notification_delta_refs_total",
notif.deltas.len() as u64,
);
}
let _read_state_step = timing
.as_ref()
.map(|t| t.span_rrdp_repo_step(notification_uri, "read_state"));
let _read_state_total = timing
.as_ref()
.map(|t| t.span_phase("rrdp_read_state_total"));
let state = load_rrdp_local_state(store, notification_uri).map_err(RrdpSyncError::Storage)?;
drop(_read_state_step);
drop(_read_state_total);
let same_session_state = state
.as_ref()
.filter(|s| s.session_id == notif.session_id.to_string());
if let Some(s) = same_session_state {
if s.serial == notif.serial {
let _hydrate_step = timing
.as_ref()
.map(|t| t.span_rrdp_repo_step(notification_uri, "hydrate_current_index"));
let _hydrate_total = timing
.as_ref()
.map(|t| t.span_phase("rrdp_hydrate_current_index_total"));
let hydrated = hydrate_current_repo_index_from_rrdp_members(
store,
notification_uri,
current_repo_index,
)?;
drop(_hydrate_step);
drop(_hydrate_total);
if let Some(t) = timing.as_ref() {
t.record_count("rrdp_current_index_hydrated_objects_total", hydrated as u64);
}
return Ok(0);
}
if s.serial > notif.serial {
return Err(RrdpError::NotificationSerialRollback {
old: s.serial,
new: notif.serial,
}
.into());
}
}
if let Some(s) = same_session_state {
// RFC 8182 §3.4.1: if session matches, MAY use deltas when a contiguous chain from the
// last processed serial to the current serial can be processed (i.e. deltas cover the gap).
let want_first = s.serial + 1;
let want_last = notif.serial;
if want_first <= want_last && !notif.deltas.is_empty() {
let min_serial = notif.deltas[0].serial;
let max_serial = notif.deltas[notif.deltas.len() - 1].serial;
// `parse_notification` guarantees contiguity and max==notif.serial when deltas exist.
if max_serial == notif.serial && want_first >= min_serial {
// Fetch all required delta files first so a network failure doesn't leave us with
// partially applied deltas and no snapshot fallback.
let _fetch_d_step = timing
.as_ref()
.map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_deltas"));
let _fetch_d_total = timing
.as_ref()
.map(|t| t.span_phase("rrdp_fetch_deltas_total"));
let mut fetched: Vec<(u64, [u8; 32], Vec<u8>)> =
Vec::with_capacity((want_last - want_first + 1) as usize);
let mut fetch_ok = true;
for serial in want_first..=want_last {
if let Some(t) = timing.as_ref() {
t.record_count("rrdp_delta_fetch_attempted_total", 1);
}
let idx = (serial - min_serial) as usize;
let dref = match notif.deltas.get(idx) {
Some(v) if v.serial == serial => v,
_ => {
fetch_ok = false;
break;
}
};
let mut dl_span = download_log
.map(|dl| dl.span_download(AuditDownloadKind::RrdpDelta, &dref.uri));
match fetcher.fetch_rrdp(
RrdpResourceKind::Delta,
&dref.uri,
&notification_origin,
) {
Ok(bytes) => {
if let Some(t) = timing.as_ref() {
t.record_count("rrdp_delta_fetch_ok_total", 1);
t.record_count("rrdp_delta_bytes_total", bytes.len() as u64);
}
if let Some(s) = dl_span.as_mut() {
s.set_bytes(bytes.len() as u64);
s.set_ok();
}
fetched.push((serial, dref.hash_sha256, bytes))
}
Err(RrdpFetchError::Fetch(e)) => {
if let Some(t) = timing.as_ref() {
t.record_count("rrdp_delta_fetch_fail_total", 1);
}
if let Some(s) = dl_span.as_mut() {
s.set_err(e.clone());
}
fetch_ok = false;
break;
}
Err(RrdpFetchError::Rrdp(e)) => return Err(e.into()),
}
}
drop(_fetch_d_step);
drop(_fetch_d_total);
if fetch_ok {
let _apply_d_step = timing
.as_ref()
.map(|t| t.span_rrdp_repo_step(notification_uri, "apply_deltas"));
let _apply_d_total = timing
.as_ref()
.map(|t| t.span_phase("rrdp_apply_deltas_total"));
let mut applied_total = 0usize;
let mut ok = true;
for (serial, expected_hash, bytes) in &fetched {
match apply_delta(
store,
notification_uri,
current_repo_index,
bytes.as_slice(),
*expected_hash,
notif.session_id,
*serial,
) {
Ok(n) => applied_total += n,
Err(_) => {
ok = false;
break;
}
}
}
drop(_apply_d_step);
drop(_apply_d_total);
if ok {
let _write_state_step = timing
.as_ref()
.map(|t| t.span_rrdp_repo_step(notification_uri, "write_state"));
let _write_state_total = timing
.as_ref()
.map(|t| t.span_phase("rrdp_write_state_total"));
let new_state = RrdpState {
session_id: notif.session_id.to_string(),
serial: notif.serial,
};
persist_rrdp_local_state(
store,
notification_uri,
&new_state,
RrdpSourceSyncState::DeltaReady,
Some(&notif.snapshot_uri),
Some(&hex::encode(notif.snapshot_hash_sha256)),
)
.map_err(RrdpSyncError::Storage)?;
drop(_write_state_step);
drop(_write_state_total);
if let Some(t) = timing.as_ref() {
t.record_count("rrdp_delta_ops_applied_total", applied_total as u64);
}
let _hydrate_step = timing.as_ref().map(|t| {
t.span_rrdp_repo_step(notification_uri, "hydrate_current_index")
});
let _hydrate_total = timing
.as_ref()
.map(|t| t.span_phase("rrdp_hydrate_current_index_total"));
let hydrated = hydrate_current_repo_index_from_rrdp_members(
store,
notification_uri,
current_repo_index,
)?;
drop(_hydrate_step);
drop(_hydrate_total);
if let Some(t) = timing.as_ref() {
t.record_count(
"rrdp_current_index_hydrated_objects_total",
hydrated as u64,
);
}
return Ok(applied_total);
}
}
}
}
}
// Snapshot fallback (RFC 8182 §3.4.3).
let _fetch_step = timing
.as_ref()
.map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_snapshot"));
let _fetch_total = timing
.as_ref()
.map(|t| t.span_phase("rrdp_fetch_snapshot_total"));
let mut dl_span = download_log
.map(|dl| dl.span_download(AuditDownloadKind::RrdpSnapshot, &notif.snapshot_uri));
let (snapshot_file, _snapshot_bytes) = match fetch_snapshot_into_tempfile(
fetcher,
&notif.snapshot_uri,
&notif.snapshot_hash_sha256,
&notification_origin,
) {
Ok(v) => {
if let Some(t) = timing.as_ref() {
t.record_count("rrdp_snapshot_fetch_ok_total", 1);
t.record_count("rrdp_snapshot_bytes_total", v.1);
}
if let Some(s) = dl_span.as_mut() {
s.set_bytes(v.1);
s.set_ok();
}
v
}
Err(RrdpSyncError::Fetch(e)) => {
if let Some(t) = timing.as_ref() {
t.record_count("rrdp_snapshot_fetch_fail_total", 1);
}
if let Some(s) = dl_span.as_mut() {
s.set_err(e.clone());
}
return Err(RrdpSyncError::Fetch(e));
}
Err(e) => return Err(e),
};
drop(_fetch_step);
drop(_fetch_total);
let _apply_step = timing
.as_ref()
.map(|t| t.span_rrdp_repo_step(notification_uri, "apply_snapshot"));
let _apply_total = timing
.as_ref()
.map(|t| t.span_phase("rrdp_apply_snapshot_total"));
let published = apply_snapshot_from_bufread(
store,
notification_uri,
current_repo_index,
std::io::BufReader::new(
snapshot_file
.reopen()
.map_err(|e| RrdpSyncError::Fetch(format!("tempfile reopen failed: {e}")))?,
),
notif.session_id,
notif.serial,
)?;
if let Some(t) = timing.as_ref() {
t.record_count("rrdp_snapshot_objects_applied_total", published as u64);
}
drop(_apply_step);
drop(_apply_total);
let _write_state_step = timing
.as_ref()
.map(|t| t.span_rrdp_repo_step(notification_uri, "write_state"));
let _write_state_total = timing
.as_ref()
.map(|t| t.span_phase("rrdp_write_state_total"));
let new_state = RrdpState {
session_id: notif.session_id.to_string(),
serial: notif.serial,
};
persist_rrdp_local_state(
store,
notification_uri,
&new_state,
RrdpSourceSyncState::SnapshotOnly,
Some(&notif.snapshot_uri),
Some(&hex::encode(notif.snapshot_hash_sha256)),
)
.map_err(RrdpSyncError::Storage)?;
drop(_write_state_step);
drop(_write_state_total);
Ok(published)
}

View File

@@ -0,0 +1,87 @@
// Shared XML, URI, and hash parsing helpers.
#[cfg(test)]
use snapshot_apply::apply_snapshot;
use snapshot_apply::{apply_snapshot_from_bufread, fetch_snapshot_into_tempfile};
fn parse_rrdp_xml(xml: &[u8]) -> Result<roxmltree::Document<'_>, RrdpError> {
if xml.iter().any(|&b| b > 0x7F) {
return Err(RrdpError::NotAscii);
}
let s = std::str::from_utf8(xml).map_err(|e| RrdpError::Xml(e.to_string()))?;
roxmltree::Document::parse(s).map_err(|e| RrdpError::Xml(e.to_string()))
}
fn validate_root_common(root: &roxmltree::Node<'_, '_>) -> Result<(), RrdpError> {
let ns = root.default_namespace().unwrap_or("").to_string();
if ns != RRDP_XMLNS {
return Err(RrdpError::InvalidNamespace(ns));
}
let version = root.attribute("version").unwrap_or("");
if version != "1" {
return Err(RrdpError::InvalidVersion(version.to_string()));
}
Ok(())
}
fn parse_uuid_attr(root: &roxmltree::Node<'_, '_>, name: &'static str) -> Result<Uuid, RrdpError> {
let s = root.attribute(name).unwrap_or("");
Uuid::parse_str(s).map_err(|_e| RrdpError::InvalidSessionId(s.to_string()))
}
fn parse_u64_attr(root: &roxmltree::Node<'_, '_>, name: &'static str) -> Result<u64, RrdpError> {
let s = root.attribute(name).unwrap_or("");
parse_u64_str(s)
}
fn parse_u64_str(s: &str) -> Result<u64, RrdpError> {
let v = s
.parse::<u64>()
.map_err(|_e| RrdpError::InvalidSerial(s.to_string()))?;
if v == 0 {
return Err(RrdpError::InvalidSerial(s.to_string()));
}
Ok(v)
}
fn parse_sha256_hex_impl(s: &str, invalid: fn(String) -> RrdpError) -> Result<[u8; 32], RrdpError> {
let bytes = hex::decode(s).map_err(|_e| invalid(s.to_string()))?;
if bytes.len() != 32 {
return Err(invalid(s.to_string()));
}
let mut out = [0u8; 32];
out.copy_from_slice(&bytes);
Ok(out)
}
fn parse_sha256_hex_snapshot(s: &str) -> Result<[u8; 32], RrdpError> {
parse_sha256_hex_impl(s, RrdpError::SnapshotHashInvalid)
}
fn parse_sha256_hex_delta_ref(s: &str) -> Result<[u8; 32], RrdpError> {
parse_sha256_hex_impl(s, RrdpError::DeltaRefHashInvalid)
}
fn parse_sha256_hex_delta_publish(s: &str) -> Result<[u8; 32], RrdpError> {
parse_sha256_hex_impl(s, RrdpError::DeltaPublishHashInvalid)
}
fn parse_sha256_hex_delta_withdraw(s: &str) -> Result<[u8; 32], RrdpError> {
parse_sha256_hex_impl(s, RrdpError::DeltaWithdrawHashInvalid)
}
fn collect_element_text(node: &roxmltree::Node<'_, '_>) -> Option<String> {
let mut out = String::new();
for child in node.children() {
if child.is_text() {
out.push_str(child.text().unwrap_or(""));
}
}
if out.is_empty() { None } else { Some(out) }
}
fn strip_all_ascii_whitespace(s: &str) -> String {
s.chars().filter(|c| !c.is_ascii_whitespace()).collect()
}

View File

@@ -0,0 +1,454 @@
use base64::Engine;
use quick_xml::Reader;
use quick_xml::events::Event;
use sha2::Digest;
use std::io::{BufRead, Seek, SeekFrom, Write};
use uuid::Uuid;
use crate::repository::current_repo_index::CurrentRepoIndexHandle;
use crate::repository::storage::RocksStore;
use crate::repository::sync::store_projection::{
build_repository_view_present_entry, build_repository_view_withdrawn_entry,
build_rrdp_source_member_present_record, build_rrdp_source_member_withdrawn_record,
build_rrdp_uri_owner_active_record, build_rrdp_uri_owner_withdrawn_record, compute_sha256_hex,
current_rrdp_owner_is, ensure_rrdp_uri_can_be_owned_by, prepare_repo_bytes_batch,
};
use super::{
Fetcher, RRDP_SNAPSHOT_APPLY_BATCH_SIZE, RRDP_XMLNS, RrdpError, RrdpFetchError, RrdpOrigin,
RrdpResourceKind, RrdpSyncError, parse_u64_str, strip_all_ascii_whitespace,
};
#[cfg(test)]
pub(super) fn apply_snapshot(
store: &RocksStore,
notification_uri: &str,
current_repo_index: Option<&CurrentRepoIndexHandle>,
snapshot_xml: &[u8],
expected_session_id: Uuid,
expected_serial: u64,
) -> Result<usize, RrdpSyncError> {
if snapshot_xml.iter().any(|&b| b > 0x7F) {
return Err(RrdpError::NotAscii.into());
}
apply_snapshot_from_bufread(
store,
notification_uri,
current_repo_index,
std::io::Cursor::new(snapshot_xml),
expected_session_id,
expected_serial,
)
}
pub(super) fn apply_snapshot_from_bufread<R: BufRead>(
store: &RocksStore,
notification_uri: &str,
current_repo_index: Option<&CurrentRepoIndexHandle>,
input: R,
expected_session_id: Uuid,
expected_serial: u64,
) -> Result<usize, RrdpSyncError> {
let previous_members: Vec<String> = store
.list_current_rrdp_source_members(notification_uri)
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?
.into_iter()
.map(|record| record.rsync_uri)
.collect();
let session_id = expected_session_id.to_string();
let mut new_set: std::collections::HashSet<String> = std::collections::HashSet::new();
let mut batch_published: Vec<(String, Vec<u8>)> =
Vec::with_capacity(RRDP_SNAPSHOT_APPLY_BATCH_SIZE);
let mut published_count = 0usize;
let mut reader = Reader::from_reader(input);
reader.config_mut().trim_text(false);
let mut buf = Vec::new();
let mut root_seen = false;
let mut in_publish = false;
let mut publish_nested_depth = 0usize;
let mut current_publish_uri: Option<String> = None;
let mut current_publish_text = String::new();
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(e)) => {
let local_name = e.local_name();
let local_name = local_name.as_ref();
if !root_seen {
root_seen = true;
if local_name != b"snapshot" {
let got = String::from_utf8_lossy(local_name).to_string();
return Err(RrdpError::UnexpectedRoot(got).into());
}
let mut xmlns = String::new();
let mut version = String::new();
let mut session_id_attr = String::new();
let mut serial_attr = String::new();
for attr in e.attributes().with_checks(false) {
let attr = match attr {
Ok(attr) => attr,
Err(e) => return Err(RrdpError::Xml(e.to_string()).into()),
};
let key = attr.key.as_ref();
let value = attr
.decode_and_unescape_value(reader.decoder())
.map_err(|e| RrdpError::Xml(e.to_string()))?
.into_owned();
match key {
b"xmlns" => xmlns = value,
b"version" => version = value,
b"session_id" => session_id_attr = value,
b"serial" => serial_attr = value,
_ => {}
}
}
if xmlns != RRDP_XMLNS {
return Err(RrdpError::InvalidNamespace(xmlns).into());
}
if version != "1" {
return Err(RrdpError::InvalidVersion(version).into());
}
let got_session_id = Uuid::parse_str(&session_id_attr)
.map_err(|_| RrdpError::InvalidSessionId(session_id_attr.clone()))?;
if got_session_id != expected_session_id {
return Err(RrdpError::SnapshotSessionIdMismatch {
expected: expected_session_id.to_string(),
got: got_session_id.to_string(),
}
.into());
}
let got_serial = parse_u64_str(&serial_attr)?;
if got_serial != expected_serial {
return Err(RrdpError::SnapshotSerialMismatch {
expected: expected_serial,
got: got_serial,
}
.into());
}
} else if in_publish {
publish_nested_depth += 1;
} else if local_name == b"publish" {
let mut uri = None;
for attr in e.attributes().with_checks(false) {
let attr = match attr {
Ok(attr) => attr,
Err(e) => return Err(RrdpError::Xml(e.to_string()).into()),
};
if attr.key.as_ref() == b"uri" {
uri = Some(
attr.decode_and_unescape_value(reader.decoder())
.map_err(|e| RrdpError::Xml(e.to_string()))?
.into_owned(),
);
}
}
let uri = uri.ok_or(RrdpError::PublishUriMissing)?;
ensure_rrdp_uri_can_be_owned_by(store, notification_uri, &uri)
.map_err(RrdpSyncError::Storage)?;
in_publish = true;
publish_nested_depth = 0;
current_publish_uri = Some(uri);
current_publish_text.clear();
}
}
Ok(Event::Empty(e)) => {
let local_name = e.local_name();
let local_name = local_name.as_ref();
if !root_seen {
let got = String::from_utf8_lossy(local_name).to_string();
return Err(RrdpError::UnexpectedRoot(got).into());
}
if local_name == b"publish" {
let mut has_uri = false;
for attr in e.attributes().with_checks(false) {
let attr = match attr {
Ok(attr) => attr,
Err(e) => return Err(RrdpError::Xml(e.to_string()).into()),
};
if attr.key.as_ref() == b"uri" {
has_uri = true;
break;
}
}
if !has_uri {
return Err(RrdpError::PublishUriMissing.into());
}
return Err(RrdpError::PublishContentMissing.into());
}
}
Ok(Event::Text(e)) => {
if in_publish && publish_nested_depth == 0 {
let text = reader
.decoder()
.decode(e.as_ref())
.map_err(|e| RrdpError::Xml(e.to_string()))?;
current_publish_text.push_str(&text);
}
}
Ok(Event::CData(e)) => {
if in_publish && publish_nested_depth == 0 {
let text = reader
.decoder()
.decode(e.as_ref())
.map_err(|e| RrdpError::Xml(e.to_string()))?;
current_publish_text.push_str(&text);
}
}
Ok(Event::End(e)) => {
let local_name = e.local_name();
let local_name = local_name.as_ref();
if in_publish {
if publish_nested_depth > 0 {
publish_nested_depth -= 1;
} else if local_name == b"publish" {
let uri = current_publish_uri
.take()
.ok_or_else(|| RrdpError::Xml("publish uri missing in state".into()))?;
let content_b64 = strip_all_ascii_whitespace(&current_publish_text);
current_publish_text.clear();
if content_b64.is_empty() {
return Err(RrdpError::PublishContentMissing.into());
}
let bytes = base64::engine::general_purpose::STANDARD
.decode(content_b64.as_bytes())
.map_err(|e| RrdpError::PublishBase64(e.to_string()))?;
new_set.insert(uri.clone());
batch_published.push((uri, bytes));
published_count += 1;
if batch_published.len() >= RRDP_SNAPSHOT_APPLY_BATCH_SIZE {
flush_snapshot_publish_batch(
store,
notification_uri,
current_repo_index,
&session_id,
expected_serial,
&batch_published,
)?;
batch_published.clear();
}
in_publish = false;
}
}
}
Ok(Event::Eof) => break,
Ok(Event::Decl(_) | Event::PI(_) | Event::Comment(_) | Event::DocType(_)) => {}
Err(e) => return Err(RrdpError::Xml(e.to_string()).into()),
}
buf.clear();
}
if !root_seen {
return Err(RrdpError::Xml("missing root element".to_string()).into());
}
if in_publish {
return Err(RrdpError::PublishContentMissing.into());
}
if !batch_published.is_empty() {
flush_snapshot_publish_batch(
store,
notification_uri,
current_repo_index,
&session_id,
expected_serial,
&batch_published,
)?;
batch_published.clear();
}
let mut withdrawn: Vec<(String, Option<String>)> = Vec::new();
for old_uri in &previous_members {
if new_set.contains(old_uri) {
continue;
}
let previous_hash = store
.get_repository_view_entry(old_uri)
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?
.and_then(|entry| entry.current_hash)
.or_else(|| {
store
.load_current_object_bytes_by_uri(old_uri)
.ok()
.flatten()
.map(|bytes| compute_sha256_hex(&bytes))
});
withdrawn.push((old_uri.clone(), previous_hash));
}
let mut repository_view_entries = Vec::with_capacity(withdrawn.len());
let mut member_records = Vec::with_capacity(withdrawn.len());
let mut owner_records = Vec::with_capacity(withdrawn.len());
for (uri, previous_hash) in withdrawn {
member_records.push(build_rrdp_source_member_withdrawn_record(
notification_uri,
&session_id,
expected_serial,
&uri,
previous_hash.clone(),
));
if current_rrdp_owner_is(store, notification_uri, &uri).map_err(RrdpSyncError::Storage)? {
repository_view_entries.push(build_repository_view_withdrawn_entry(
notification_uri,
&uri,
previous_hash.clone(),
));
owner_records.push(build_rrdp_uri_owner_withdrawn_record(
notification_uri,
&session_id,
expected_serial,
&uri,
previous_hash,
));
}
}
store
.put_projection_batch(&repository_view_entries, &member_records, &owner_records)
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
if let Some(index) = current_repo_index {
index
.write()
.map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))?
.apply_repository_view_entries(&repository_view_entries)
.map_err(RrdpSyncError::Storage)?;
}
Ok(published_count)
}
fn flush_snapshot_publish_batch(
store: &RocksStore,
notification_uri: &str,
current_repo_index: Option<&CurrentRepoIndexHandle>,
session_id: &str,
serial: u64,
published: &[(String, Vec<u8>)],
) -> Result<(), RrdpSyncError> {
let prepared_bytes = prepare_repo_bytes_batch(published).map_err(RrdpSyncError::Storage)?;
let mut repository_view_entries = Vec::with_capacity(published.len());
let mut member_records = Vec::with_capacity(published.len());
let mut owner_records = Vec::with_capacity(published.len());
for (uri, _bytes) in published {
let current_hash = prepared_bytes
.uri_to_hash
.get(uri)
.cloned()
.ok_or_else(|| {
RrdpSyncError::Storage(format!("missing raw_by_hash mapping for {uri}"))
})?;
repository_view_entries.push(build_repository_view_present_entry(
notification_uri,
uri,
&current_hash,
));
member_records.push(build_rrdp_source_member_present_record(
notification_uri,
session_id,
serial,
uri,
&current_hash,
));
owner_records.push(build_rrdp_uri_owner_active_record(
notification_uri,
session_id,
serial,
uri,
&current_hash,
));
}
store
.put_blob_bytes_batch(&prepared_bytes.blobs_to_write)
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
store
.put_projection_batch(&repository_view_entries, &member_records, &owner_records)
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
if let Some(index) = current_repo_index {
index
.write()
.map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))?
.apply_repository_view_entries(&repository_view_entries)
.map_err(RrdpSyncError::Storage)?;
}
Ok(())
}
const SNAPSHOT_NON_ASCII_ERROR: &str = "snapshot body contains non-ASCII bytes";
struct SnapshotSpoolWriter<'a, W: Write> {
inner: &'a mut W,
hasher: sha2::Sha256,
bytes: u64,
}
impl<'a, W: Write> SnapshotSpoolWriter<'a, W> {
fn new(inner: &'a mut W) -> Self {
Self {
inner,
hasher: sha2::Sha256::new(),
bytes: 0,
}
}
fn finalize_hash(self) -> [u8; 32] {
let digest = self.hasher.finalize();
let mut out = [0u8; 32];
out.copy_from_slice(&digest);
out
}
}
impl<W: Write> Write for SnapshotSpoolWriter<'_, W> {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if buf.iter().any(|&b| b > 0x7F) {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
SNAPSHOT_NON_ASCII_ERROR,
));
}
let n = self.inner.write(buf)?;
self.hasher.update(&buf[..n]);
self.bytes += n as u64;
Ok(n)
}
fn flush(&mut self) -> std::io::Result<()> {
self.inner.flush()
}
}
pub(super) fn fetch_snapshot_into_tempfile(
fetcher: &dyn Fetcher,
snapshot_uri: &str,
expected_hash_sha256: &[u8; 32],
notification_origin: &RrdpOrigin,
) -> Result<(tempfile::NamedTempFile, u64), RrdpSyncError> {
let mut tmp = tempfile::NamedTempFile::new()
.map_err(|e| RrdpSyncError::Fetch(format!("tempfile create failed: {e}")))?;
let mut spool = SnapshotSpoolWriter::new(tmp.as_file_mut());
let bytes_written = match fetcher.fetch_rrdp_to_writer(
RrdpResourceKind::Snapshot,
snapshot_uri,
notification_origin,
&mut spool,
) {
Ok(bytes) => bytes,
Err(RrdpFetchError::Fetch(e)) if e.contains(SNAPSHOT_NON_ASCII_ERROR) => {
return Err(RrdpError::NotAscii.into());
}
Err(RrdpFetchError::Fetch(e)) => return Err(RrdpSyncError::Fetch(e)),
Err(RrdpFetchError::Rrdp(e)) => return Err(e.into()),
};
let computed = spool.finalize_hash();
if computed.as_slice() != expected_hash_sha256.as_slice() {
return Err(RrdpError::SnapshotHashMismatch.into());
}
tmp.as_file_mut()
.flush()
.map_err(|e| RrdpSyncError::Fetch(format!("tempfile flush failed: {e}")))?;
tmp.as_file_mut()
.seek(SeekFrom::Start(0))
.map_err(|e| RrdpSyncError::Fetch(format!("tempfile rewind failed: {e}")))?;
Ok((tmp, bytes_written))
}

View File

@@ -0,0 +1,170 @@
// RRDP snapshot synchronization and timing instrumentation.
pub fn sync_from_notification_snapshot(
store: &RocksStore,
notification_uri: &str,
notification_xml: &[u8],
fetcher: &dyn Fetcher,
) -> RrdpSyncResult<usize> {
sync_from_notification_snapshot_inner(
store,
notification_uri,
notification_xml,
fetcher,
None,
None,
)
}
pub fn sync_from_notification_snapshot_with_timing(
store: &RocksStore,
notification_uri: &str,
notification_xml: &[u8],
fetcher: &dyn Fetcher,
timing: Option<&TimingHandle>,
) -> RrdpSyncResult<usize> {
sync_from_notification_snapshot_inner(
store,
notification_uri,
notification_xml,
fetcher,
timing,
None,
)
}
pub fn sync_from_notification_snapshot_with_timing_and_download_log(
store: &RocksStore,
notification_uri: &str,
notification_xml: &[u8],
fetcher: &dyn Fetcher,
timing: Option<&TimingHandle>,
download_log: Option<&DownloadLogHandle>,
) -> RrdpSyncResult<usize> {
sync_from_notification_snapshot_inner(
store,
notification_uri,
notification_xml,
fetcher,
timing,
download_log,
)
}
fn sync_from_notification_snapshot_inner(
store: &RocksStore,
notification_uri: &str,
notification_xml: &[u8],
fetcher: &dyn Fetcher,
timing: Option<&TimingHandle>,
download_log: Option<&DownloadLogHandle>,
) -> RrdpSyncResult<usize> {
let _parse_step = timing
.as_ref()
.map(|t| t.span_rrdp_repo_step(notification_uri, "parse_notification_snapshot"));
let _parse_total = timing
.as_ref()
.map(|t| t.span_phase("rrdp_parse_notification_total"));
let notif = parse_notification(notification_xml)?;
let notification_origin = match validate_notification_references(notification_uri, &notif) {
Ok(origin) => origin,
Err(e) => {
if let Some(t) = timing.as_ref() {
t.record_count(
"rrdp_rejected_cross_origin_total",
e.is_cross_origin_violation() as u64,
);
}
return Err(e.into());
}
};
drop(_parse_step);
drop(_parse_total);
let _fetch_step = timing
.as_ref()
.map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_snapshot"));
let _fetch_total = timing
.as_ref()
.map(|t| t.span_phase("rrdp_fetch_snapshot_total"));
let mut dl_span = download_log
.map(|dl| dl.span_download(AuditDownloadKind::RrdpSnapshot, &notif.snapshot_uri));
let (snapshot_file, _snapshot_bytes) = match fetch_snapshot_into_tempfile(
fetcher,
&notif.snapshot_uri,
&notif.snapshot_hash_sha256,
&notification_origin,
) {
Ok(v) => {
if let Some(t) = timing.as_ref() {
t.record_count("rrdp_snapshot_fetch_ok_total", 1);
t.record_count("rrdp_snapshot_bytes_total", v.1);
}
if let Some(s) = dl_span.as_mut() {
s.set_bytes(v.1);
s.set_ok();
}
v
}
Err(RrdpSyncError::Fetch(e)) => {
if let Some(t) = timing.as_ref() {
t.record_count("rrdp_snapshot_fetch_fail_total", 1);
}
if let Some(s) = dl_span.as_mut() {
s.set_err(e.clone());
}
return Err(RrdpSyncError::Fetch(e));
}
Err(e) => return Err(e),
};
drop(_fetch_step);
drop(_fetch_total);
let _apply_step = timing
.as_ref()
.map(|t| t.span_rrdp_repo_step(notification_uri, "apply_snapshot"));
let _apply_total = timing
.as_ref()
.map(|t| t.span_phase("rrdp_apply_snapshot_total"));
let published = apply_snapshot_from_bufread(
store,
notification_uri,
None,
std::io::BufReader::new(
snapshot_file
.reopen()
.map_err(|e| RrdpSyncError::Fetch(format!("tempfile reopen failed: {e}")))?,
),
notif.session_id,
notif.serial,
)?;
if let Some(t) = timing.as_ref() {
t.record_count("rrdp_snapshot_objects_applied_total", published as u64);
}
drop(_apply_step);
drop(_apply_total);
let _write_state_step = timing
.as_ref()
.map(|t| t.span_rrdp_repo_step(notification_uri, "write_state"));
let _write_state_total = timing
.as_ref()
.map(|t| t.span_phase("rrdp_write_state_total"));
let state = RrdpState {
session_id: notif.session_id.to_string(),
serial: notif.serial,
};
persist_rrdp_local_state(
store,
notification_uri,
&state,
RrdpSourceSyncState::SnapshotOnly,
Some(&notif.snapshot_uri),
Some(&hex::encode(notif.snapshot_hash_sha256)),
)
.map_err(RrdpSyncError::Storage)?;
drop(_write_state_step);
drop(_write_state_total);
Ok(published)
}

View File

@@ -0,0 +1,5 @@
// RRDP tests are grouped by parsing, delta application, and synchronization.
include!("tests_parts/parsing.rs");
include!("tests_parts/delta_apply.rs");
include!("tests_parts/sync.rs");
include!("tests_parts/edge_cases.rs");

View File

@@ -0,0 +1,359 @@
// RRDP test group: delta apply.
#[test]
fn apply_delta_applies_publish_replace_and_withdraw_with_membership_checks() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let sid = "550e8400-e29b-41d4-a716-446655440000";
let notif_uri = "https://example.net/notification.xml";
// Start from snapshot state with a + b
let snapshot_uri = "https://example.net/snapshot.xml";
let snapshot = snapshot_xml(
sid,
1,
&[
("rsync://example.net/repo/a.mft", b"a1"),
("rsync://example.net/repo/b.roa", b"b1"),
],
);
let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot));
let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash);
let fetcher = MapFetcher {
map: HashMap::from([(snapshot_uri.to_string(), snapshot)]),
};
sync_from_notification_snapshot(&store, notif_uri, &notif, &fetcher).expect("sync snapshot");
let old_b = store
.load_current_object_bytes_by_uri("rsync://example.net/repo/b.roa")
.expect("load current b")
.expect("b present");
let old_b_hash = hex::encode(sha2::Sha256::digest(old_b.as_slice()));
let withdraw_a_hash = hex::encode(sha2::Sha256::digest(b"a1".as_slice()));
let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2");
let replace_b_b64 = base64::engine::general_purpose::STANDARD.encode(b"b2");
let delta = delta_xml(
sid,
2,
&[
&format!(
r#"<withdraw uri="rsync://example.net/repo/a.mft" hash="{withdraw_a_hash}"/>"#
),
&format!(
r#"<publish uri="rsync://example.net/repo/b.roa" hash="{old_b_hash}">{replace_b_b64}</publish>"#
),
&format!(r#"<publish uri="rsync://example.net/repo/c.crl">{publish_c_b64}</publish>"#),
],
);
let delta_hash = sha2::Sha256::digest(&delta);
let mut expected_hash = [0u8; 32];
expected_hash.copy_from_slice(delta_hash.as_slice());
let applied = apply_delta(
&store,
notif_uri,
None,
&delta,
expected_hash,
Uuid::parse_str(sid).unwrap(),
2,
)
.expect("apply delta");
assert_eq!(applied, 3);
assert_eq!(
store
.load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft")
.expect("load current a"),
None,
"a withdrawn"
);
let b = store
.load_current_object_bytes_by_uri("rsync://example.net/repo/b.roa")
.expect("load current b")
.expect("b present");
assert_eq!(b, b"b2");
let c = store
.load_current_object_bytes_by_uri("rsync://example.net/repo/c.crl")
.expect("load current c")
.expect("c present");
assert_eq!(c, b"c2");
assert!(
!store
.is_current_rrdp_source_member(notif_uri, "rsync://example.net/repo/a.mft")
.expect("is member"),
"a removed from rrdp repo index"
);
assert!(
store
.is_current_rrdp_source_member(notif_uri, "rsync://example.net/repo/c.crl")
.expect("is member"),
"c added to rrdp repo index"
);
let a_view = store
.get_repository_view_entry("rsync://example.net/repo/a.mft")
.expect("get a view")
.expect("a view exists");
assert_eq!(a_view.state, crate::repository::storage::RepositoryViewState::Withdrawn);
let b_view = store
.get_repository_view_entry("rsync://example.net/repo/b.roa")
.expect("get b view")
.expect("b view exists");
assert_eq!(b_view.state, crate::repository::storage::RepositoryViewState::Present);
assert_eq!(
b_view.current_hash.as_deref(),
Some(hex::encode(sha2::Sha256::digest(b"b2")).as_str())
);
let c_owner = store
.get_rrdp_uri_owner_record("rsync://example.net/repo/c.crl")
.expect("get c owner")
.expect("c owner exists");
assert_eq!(
c_owner.owner_state,
crate::repository::storage::RrdpUriOwnerState::Active
);
let a_member = store
.get_rrdp_source_member_record(notif_uri, "rsync://example.net/repo/a.mft")
.expect("get a member")
.expect("a member exists");
assert!(!a_member.present);
let current_members = store
.list_current_rrdp_source_members(notif_uri)
.expect("list current members");
assert_eq!(
current_members
.iter()
.map(|record| record.rsync_uri.as_str())
.collect::<Vec<_>>(),
vec![
"rsync://example.net/repo/b.roa",
"rsync://example.net/repo/c.crl",
]
);
}
#[test]
fn apply_delta_rejects_hash_mismatch() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
let notif_uri = "https://example.net/notification.xml";
let delta = delta_xml(
sid.to_string().as_str(),
1,
&[r#"<publish uri="rsync://example.net/repo/a.mft">QQ==</publish>"#],
);
let mut wrong = [0u8; 32];
wrong[0] = 1;
let err = apply_delta(&store, notif_uri, None, &delta, wrong, sid, 1).unwrap_err();
assert!(matches!(
err,
RrdpSyncError::Rrdp(RrdpError::DeltaHashMismatch)
));
}
#[test]
fn apply_delta_rejects_withdraw_of_non_member() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
let notif_uri = "https://example.net/notification.xml";
let withdraw_hash = "00".repeat(32);
let delta = delta_xml(
sid.to_string().as_str(),
1,
&[&format!(
r#"<withdraw uri="rsync://example.net/repo/a.mft" hash="{withdraw_hash}"/>"#
)],
);
let delta_hash = sha2::Sha256::digest(&delta);
let mut expected_hash = [0u8; 32];
expected_hash.copy_from_slice(delta_hash.as_slice());
let err = apply_delta(&store, notif_uri, None, &delta, expected_hash, sid, 1).unwrap_err();
assert!(matches!(
err,
RrdpSyncError::Rrdp(RrdpError::DeltaTargetNotFromRepository { .. })
));
}
#[test]
fn apply_delta_rejects_publish_without_hash_for_existing_object() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let sid = "550e8400-e29b-41d4-a716-446655440000";
let notif_uri = "https://example.net/notification.xml";
// Seed snapshot with a.mft.
let snapshot_uri = "https://example.net/snapshot.xml";
let snapshot = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]);
let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot));
let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash);
let fetcher = MapFetcher {
map: HashMap::from([(snapshot_uri.to_string(), snapshot)]),
};
sync_from_notification_snapshot(&store, notif_uri, &notif, &fetcher).expect("seed");
// Replace publish for an existing URI must have @hash.
let publish_b64 = base64::engine::general_purpose::STANDARD.encode(b"a2");
let delta = delta_xml(
sid,
2,
&[&format!(
r#"<publish uri="rsync://example.net/repo/a.mft">{publish_b64}</publish>"#
)],
);
let delta_hash = sha2::Sha256::digest(&delta);
let mut expected_hash = [0u8; 32];
expected_hash.copy_from_slice(delta_hash.as_slice());
let err = apply_delta(
&store,
notif_uri,
None,
&delta,
expected_hash,
Uuid::parse_str(sid).unwrap(),
2,
)
.unwrap_err();
assert!(matches!(
err,
RrdpSyncError::Rrdp(RrdpError::DeltaPublishWithoutHashForExisting { .. })
));
}
#[test]
fn apply_delta_rejects_target_missing_and_hash_mismatch() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let sid = "550e8400-e29b-41d4-a716-446655440000";
let notif_uri = "https://example.net/notification.xml";
// Seed snapshot with a.mft.
let snapshot_uri = "https://example.net/snapshot.xml";
let snapshot = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]);
let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot));
let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash);
let fetcher = MapFetcher {
map: HashMap::from([(snapshot_uri.to_string(), snapshot)]),
};
sync_from_notification_snapshot(&store, notif_uri, &notif, &fetcher).expect("seed");
let old_bytes = store
.load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft")
.expect("get")
.expect("present");
let old_hash = hex::encode(sha2::Sha256::digest(old_bytes.as_slice()));
// Hash mismatch on withdraw.
let wrong_hash = "11".repeat(32);
let delta = delta_xml(
sid,
2,
&[&format!(
r#"<withdraw uri="rsync://example.net/repo/a.mft" hash="{wrong_hash}"/>"#
)],
);
let delta_hash = sha2::Sha256::digest(&delta);
let mut expected_hash = [0u8; 32];
expected_hash.copy_from_slice(delta_hash.as_slice());
let err = apply_delta(
&store,
notif_uri,
None,
&delta,
expected_hash,
Uuid::parse_str(sid).unwrap(),
2,
)
.unwrap_err();
assert!(matches!(
err,
RrdpSyncError::Rrdp(RrdpError::DeltaTargetHashMismatch { .. })
));
// Target missing in local cache (index still says it's a member).
store
.delete_repository_view_entry("rsync://example.net/repo/a.mft")
.expect("delete current repository view entry");
let delta = delta_xml(
sid,
2,
&[&format!(
r#"<withdraw uri="rsync://example.net/repo/a.mft" hash="{old_hash}"/>"#
)],
);
let delta_hash = sha2::Sha256::digest(&delta);
let mut expected_hash = [0u8; 32];
expected_hash.copy_from_slice(delta_hash.as_slice());
let err = apply_delta(
&store,
notif_uri,
None,
&delta,
expected_hash,
Uuid::parse_str(sid).unwrap(),
2,
)
.unwrap_err();
assert!(matches!(
err,
RrdpSyncError::Rrdp(RrdpError::DeltaTargetMissing { .. })
));
}
#[test]
fn apply_delta_rejects_session_and_serial_mismatch() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let sid = "550e8400-e29b-41d4-a716-446655440000";
let notif_uri = "https://example.net/notification.xml";
let publish_b64 = base64::engine::general_purpose::STANDARD.encode(b"x");
let delta = delta_xml(
sid,
2,
&[&format!(
r#"<publish uri="rsync://example.net/repo/x.cer">{publish_b64}</publish>"#
)],
);
let delta_hash = sha2::Sha256::digest(&delta);
let mut expected_hash = [0u8; 32];
expected_hash.copy_from_slice(delta_hash.as_slice());
// Session mismatch.
let other_sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440001").unwrap();
let err =
apply_delta(&store, notif_uri, None, &delta, expected_hash, other_sid, 2).unwrap_err();
assert!(matches!(
err,
RrdpSyncError::Rrdp(RrdpError::DeltaSessionIdMismatch { .. })
));
// Serial mismatch.
let err = apply_delta(
&store,
notif_uri,
None,
&delta,
expected_hash,
Uuid::parse_str(sid).unwrap(),
3,
)
.unwrap_err();
assert!(matches!(
err,
RrdpSyncError::Rrdp(RrdpError::DeltaSerialMismatch { .. })
));
}

View File

@@ -0,0 +1,214 @@
// RRDP test group: edge cases.
#[test]
fn load_rrdp_local_state_uses_source_record_only() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let notif_uri = "https://example.net/notification.xml";
assert_eq!(
load_rrdp_local_state(&store, notif_uri).expect("load empty"),
None
);
update_rrdp_source_record_on_success(
&store,
notif_uri,
"source-session",
9,
crate::repository::storage::RrdpSourceSyncState::DeltaReady,
Some("https://example.net/snapshot.xml"),
Some(&hex::encode([0x11; 32])),
)
.expect("write source record");
let got = load_rrdp_local_state(&store, notif_uri)
.expect("load source preferred")
.expect("source present");
assert_eq!(
got,
RrdpState {
session_id: "source-session".to_string(),
serial: 9,
}
);
}
#[test]
fn sync_from_notification_falls_back_to_snapshot_if_missing_required_deltas() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let sid = "550e8400-e29b-41d4-a716-446655440000";
let notif_uri = "https://example.net/notification.xml";
// Seed state serial=1 with a only.
let snapshot_uri_1 = "https://example.net/snapshot-1.xml";
let snapshot_1 = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]);
let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1));
let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1);
let fetcher_1 = MapFetcher {
map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]),
};
sync_from_notification_snapshot(&store, notif_uri, &notif_1, &fetcher_1).expect("seed");
// Notification serial=3 only includes delta serial=3 (contiguous per RFC, but missing
// serial=2 relative to our local state, so we must use snapshot).
let snapshot_uri_3 = "https://example.net/snapshot-3.xml";
let snapshot_3 = snapshot_xml(sid, 3, &[("rsync://example.net/repo/z.roa", b"z3")]);
let snapshot_hash_3 = hex::encode(sha2::Sha256::digest(&snapshot_3));
let delta_3_hash_hex = "11".repeat(32);
let notif_3 = notification_xml_with_deltas(
sid,
3,
snapshot_uri_3,
&snapshot_hash_3,
&[(
"d3",
3,
"https://example.net/delta-3.xml",
&delta_3_hash_hex,
)],
);
let fetcher = MapFetcher {
map: HashMap::from([(snapshot_uri_3.to_string(), snapshot_3)]),
};
let published = sync_from_notification(&store, notif_uri, &notif_3, &fetcher).expect("sync");
assert_eq!(published, 1);
assert!(
store
.load_current_object_bytes_by_uri("rsync://example.net/repo/z.roa")
.expect("get current")
.is_some()
);
}
#[test]
fn sync_from_notification_snapshot_rejects_snapshot_hash_mismatch() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let sid = "550e8400-e29b-41d4-a716-446655440000";
let serial = 1u64;
let notif_uri = "https://example.net/notification.xml";
let snapshot_uri = "https://example.net/snapshot.xml";
let snapshot = snapshot_xml(sid, serial, &[("rsync://example.net/repo/a.mft", b"x")]);
let notif = notification_xml(sid, serial, snapshot_uri, &"00".repeat(32));
let fetcher = MapFetcher {
map: HashMap::from([(snapshot_uri.to_string(), snapshot)]),
};
let err = sync_from_notification_snapshot(&store, notif_uri, &notif, &fetcher).unwrap_err();
assert!(matches!(
err,
RrdpSyncError::Rrdp(RrdpError::SnapshotHashMismatch)
));
}
#[test]
fn apply_snapshot_rejects_session_id_and_serial_mismatch() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let notif_uri = "https://example.net/notification.xml";
let expected_sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
let got_sid = "550e8400-e29b-41d4-a716-446655440001";
let snapshot = snapshot_xml(got_sid, 2, &[("rsync://example.net/repo/a.mft", b"x")]);
let err = apply_snapshot(&store, notif_uri, None, &snapshot, expected_sid, 2).unwrap_err();
assert!(matches!(
err,
RrdpSyncError::Rrdp(RrdpError::SnapshotSessionIdMismatch { .. })
));
let snapshot = snapshot_xml(
expected_sid.to_string().as_str(),
3,
&[("rsync://example.net/repo/a.mft", b"x")],
);
let err = apply_snapshot(&store, notif_uri, None, &snapshot, expected_sid, 2).unwrap_err();
assert!(matches!(
err,
RrdpSyncError::Rrdp(RrdpError::SnapshotSerialMismatch { .. })
));
}
#[test]
fn strip_all_ascii_whitespace_removes_newlines_and_spaces() {
assert_eq!(strip_all_ascii_whitespace(" a \n b\tc "), "abc");
}
#[test]
fn apply_snapshot_reports_publish_errors() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let notif_uri = "https://example.net/notification.xml";
let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
// Missing publish/@uri
let xml = format!(
r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{sid}" serial="1"><publish>AA==</publish></snapshot>"#
)
.into_bytes();
let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err();
assert!(matches!(
err,
RrdpSyncError::Rrdp(RrdpError::PublishUriMissing)
));
// Missing base64 content (no text nodes).
let xml = format!(
r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{sid}" serial="1"><publish uri="rsync://example.net/repo/a.cer"></publish></snapshot>"#
)
.into_bytes();
let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err();
assert!(matches!(
err,
RrdpSyncError::Rrdp(RrdpError::PublishContentMissing)
));
// Invalid base64 content.
let xml = format!(
r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{sid}" serial="1"><publish uri="rsync://example.net/repo/a.cer">!!!</publish></snapshot>"#
)
.into_bytes();
let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err();
assert!(matches!(
err,
RrdpSyncError::Rrdp(RrdpError::PublishBase64(_))
));
}
#[test]
fn apply_snapshot_handles_multiple_publish_batches() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let notif_uri = "https://example.net/notification.xml";
let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
let total = RRDP_SNAPSHOT_APPLY_BATCH_SIZE + 7;
let mut xml =
format!(r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{sid}" serial="1">"#);
for i in 0..total {
let uri = format!("rsync://example.net/repo/{i:04}.roa");
let bytes = format!("payload-{i}").into_bytes();
let b64 = base64::engine::general_purpose::STANDARD.encode(bytes);
xml.push_str(&format!(r#"<publish uri="{uri}">{b64}</publish>"#));
}
xml.push_str("</snapshot>");
let published =
apply_snapshot(&store, notif_uri, None, xml.as_bytes(), sid, 1).expect("apply snapshot");
assert_eq!(published, total);
for idx in [0usize, RRDP_SNAPSHOT_APPLY_BATCH_SIZE - 1, total - 1] {
let uri = format!("rsync://example.net/repo/{idx:04}.roa");
let got = store
.load_current_object_bytes_by_uri(&uri)
.expect("load object")
.expect("object exists");
assert_eq!(got, format!("payload-{idx}").into_bytes());
}
}

View File

@@ -0,0 +1,374 @@
// RRDP test group: parsing.
use super::*;
use crate::output::analysis::timing::{TimingHandle, TimingMeta};
use crate::repository::current_repo_index::CurrentRepoIndex;
use crate::repository::storage::RocksStore;
use std::collections::HashMap;
use std::io::Read;
use std::time::Duration;
struct MapFetcher {
map: HashMap<String, Vec<u8>>,
}
impl Fetcher for MapFetcher {
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
self.map
.get(uri)
.cloned()
.ok_or_else(|| format!("not found: {uri}"))
}
}
struct SleepyFetcher {
inner: MapFetcher,
sleep_uri: String,
sleep: Duration,
}
impl Fetcher for SleepyFetcher {
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
if uri == self.sleep_uri {
std::thread::sleep(self.sleep);
}
self.inner.fetch(uri)
}
}
struct WriterOnlyFetcher {
map: HashMap<String, Vec<u8>>,
}
impl Fetcher for WriterOnlyFetcher {
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
Err(format!("unexpected buffered fetch: {uri}"))
}
fn fetch_to_writer(&self, uri: &str, out: &mut dyn std::io::Write) -> Result<u64, String> {
let bytes = self
.map
.get(uri)
.ok_or_else(|| format!("not found: {uri}"))?;
out.write_all(bytes)
.map_err(|e| format!("write sink failed: {e}"))?;
Ok(bytes.len() as u64)
}
}
struct NonAsciiWriterFetcher;
impl Fetcher for NonAsciiWriterFetcher {
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
Err(format!("unexpected buffered fetch: {uri}"))
}
fn fetch_to_writer(&self, _uri: &str, out: &mut dyn std::io::Write) -> Result<u64, String> {
out.write_all(&[0x80])
.map_err(|e| format!("write sink failed: {e}"))?;
Err("snapshot body contains non-ASCII bytes".to_string())
}
}
fn assert_current_object(store: &RocksStore, uri: &str, expected: &[u8]) {
assert_eq!(
store
.load_current_object_bytes_by_uri(uri)
.expect("load current object"),
Some(expected.to_vec())
);
}
fn notification_xml(
session_id: &str,
serial: u64,
snapshot_uri: &str,
snapshot_hash: &str,
) -> Vec<u8> {
format!(
r#"<notification xmlns="{RRDP_XMLNS}" version="1" session_id="{session_id}" serial="{serial}"><snapshot uri="{snapshot_uri}" hash="{snapshot_hash}"/></notification>"#
)
.into_bytes()
}
fn notification_xml_with_deltas(
session_id: &str,
serial: u64,
snapshot_uri: &str,
snapshot_hash: &str,
deltas: &[(&str, u64, &str, &str)],
) -> Vec<u8> {
let mut out = format!(
r#"<notification xmlns="{RRDP_XMLNS}" version="1" session_id="{session_id}" serial="{serial}"><snapshot uri="{snapshot_uri}" hash="{snapshot_hash}"/>"#
);
for (_name, delta_serial, uri, hash) in deltas {
out.push_str(&format!(
r#"<delta serial="{delta_serial}" uri="{uri}" hash="{hash}"/>"#
));
}
out.push_str("</notification>");
out.into_bytes()
}
fn snapshot_xml(session_id: &str, serial: u64, published: &[(&str, &[u8])]) -> Vec<u8> {
let mut out = format!(
r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{session_id}" serial="{serial}">"#
);
for (uri, bytes) in published {
let b64 = base64::engine::general_purpose::STANDARD.encode(bytes);
out.push_str(&format!(r#"<publish uri="{uri}">{b64}</publish>"#));
}
out.push_str("</snapshot>");
out.into_bytes()
}
#[test]
fn fetch_snapshot_into_tempfile_streams_and_validates_hash() {
let snapshot_uri = "https://example.test/snapshot.xml";
let snapshot = b"<snapshot xmlns=\"http://www.ripe.net/rpki/rrdp\" version=\"1\"/>".to_vec();
let mut expected_hash = [0u8; 32];
expected_hash.copy_from_slice(&sha2::Sha256::digest(&snapshot));
let fetcher = WriterOnlyFetcher {
map: HashMap::from([(snapshot_uri.to_string(), snapshot.clone())]),
};
let origin = RrdpOrigin::parse(snapshot_uri, RrdpResourceKind::Notification).expect("origin");
let (mut file, bytes_written) =
fetch_snapshot_into_tempfile(&fetcher, snapshot_uri, &expected_hash, &origin)
.expect("fetch snapshot into tempfile");
assert_eq!(bytes_written, snapshot.len() as u64);
let mut got = Vec::new();
file.as_file_mut()
.read_to_end(&mut got)
.expect("read tempfile");
assert_eq!(got, snapshot);
let mut wrong_hash = expected_hash;
wrong_hash[0] ^= 0xff;
let err =
fetch_snapshot_into_tempfile(&fetcher, snapshot_uri, &wrong_hash, &origin).unwrap_err();
assert!(matches!(
err,
RrdpSyncError::Rrdp(RrdpError::SnapshotHashMismatch)
));
}
#[test]
fn fetch_snapshot_into_tempfile_maps_stream_non_ascii_error() {
let err = fetch_snapshot_into_tempfile(
&NonAsciiWriterFetcher,
"https://example.test/snapshot.xml",
&[0u8; 32],
&RrdpOrigin::parse(
"https://example.test/notification.xml",
RrdpResourceKind::Notification,
)
.expect("origin"),
)
.unwrap_err();
assert!(matches!(err, RrdpSyncError::Rrdp(RrdpError::NotAscii)));
}
#[test]
fn timing_rrdp_repo_step_spans_cover_snapshot_fetch_duration() {
let temp = tempfile::tempdir().expect("tempdir");
let store_dir = temp.path().join("db");
let store = RocksStore::open(&store_dir).expect("open rocksdb");
let notification_uri = "https://example.test/notification.xml";
let snapshot_uri = "https://example.test/snapshot.xml";
let published_uri = "rsync://example.test/repo/a.mft";
let published_bytes = b"x";
let session_id = "550e8400-e29b-41d4-a716-446655440000";
let snapshot = snapshot_xml(session_id, 1, &[(published_uri, published_bytes)]);
let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot));
let notif = notification_xml(session_id, 1, snapshot_uri, &snapshot_hash);
let mut map = HashMap::new();
map.insert(snapshot_uri.to_string(), snapshot);
let fetcher = SleepyFetcher {
inner: MapFetcher { map },
sleep_uri: snapshot_uri.to_string(),
sleep: Duration::from_millis(25),
};
let timing = TimingHandle::new(TimingMeta {
recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(),
validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(),
tal_url: None,
db_path: Some(store_dir.to_string_lossy().into_owned()),
});
sync_from_notification_snapshot_with_timing(
&store,
notification_uri,
&notif,
&fetcher,
Some(&timing),
)
.expect("rrdp snapshot sync ok");
let timing_path = temp.path().join("timing.json");
timing.write_json(&timing_path, 200).expect("write timing");
let rep: serde_json::Value =
serde_json::from_slice(&std::fs::read(&timing_path).expect("read timing"))
.expect("parse timing");
let want = format!("{notification_uri}::fetch_snapshot");
let steps = rep
.get("top_rrdp_repo_steps")
.and_then(|v| v.as_array())
.expect("top_rrdp_repo_steps array");
let entry = steps
.iter()
.find(|e| e.get("key").and_then(|k| k.as_str()) == Some(want.as_str()))
.unwrap_or_else(|| panic!("missing timing step entry for {want}"));
let nanos = entry
.get("total_nanos")
.and_then(|v| v.as_u64())
.expect("total_nanos");
assert!(
nanos >= 20_000_000,
"expected fetch_snapshot timing to include the fetch duration; got {nanos}ns"
);
}
#[test]
fn parse_notification_snapshot_rejects_non_ascii() {
let mut xml = b"<notification/>".to_vec();
xml.push(0x80);
let err = parse_notification_snapshot(&xml).unwrap_err();
assert!(matches!(err, RrdpError::NotAscii));
}
#[test]
fn parse_notification_snapshot_parses_valid_minimal_notification() {
let sid = "550e8400-e29b-41d4-a716-446655440000";
let snapshot_uri = "https://example.net/snapshot.xml";
let hash = "00".repeat(32);
let xml = notification_xml(sid, 7, snapshot_uri, &hash);
let n = parse_notification_snapshot(&xml).expect("parse");
assert_eq!(n.session_id, Uuid::parse_str(sid).unwrap());
assert_eq!(n.serial, 7);
assert_eq!(n.snapshot_uri, snapshot_uri);
assert_eq!(hex::encode(n.snapshot_hash_sha256), hash);
}
#[test]
fn parse_notification_parses_deltas_and_validates_contiguity() {
let sid = "550e8400-e29b-41d4-a716-446655440000";
let snapshot_uri = "https://example.net/snapshot.xml";
let hash = "00".repeat(32);
let d_hash_2 = "11".repeat(32);
let d_hash_3 = "22".repeat(32);
// Provide deltas in reverse order to ensure we sort.
let xml = notification_xml_with_deltas(
sid,
3,
snapshot_uri,
&hash,
&[
("d3", 3, "https://example.net/delta-3.xml", &d_hash_3),
("d2", 2, "https://example.net/delta-2.xml", &d_hash_2),
],
);
let n = parse_notification(&xml).expect("parse notification");
assert_eq!(n.serial, 3);
assert_eq!(n.deltas.len(), 2);
assert_eq!(n.deltas[0].serial, 2);
assert_eq!(n.deltas[1].serial, 3);
assert_eq!(n.deltas[0].uri, "https://example.net/delta-2.xml");
assert_eq!(hex::encode(n.deltas[1].hash_sha256), d_hash_3);
}
#[test]
fn parse_notification_rejects_non_contiguous_deltas() {
let sid = "550e8400-e29b-41d4-a716-446655440000";
let snapshot_uri = "https://example.net/snapshot.xml";
let hash = "00".repeat(32);
let d_hash_1 = "11".repeat(32);
let d_hash_3 = "22".repeat(32);
// Missing delta serial 2.
let xml = notification_xml_with_deltas(
sid,
3,
snapshot_uri,
&hash,
&[
("d3", 3, "https://example.net/delta-3.xml", &d_hash_3),
("d1", 1, "https://example.net/delta-1.xml", &d_hash_1),
],
);
let err = parse_notification(&xml).unwrap_err();
assert!(matches!(err, RrdpError::DeltaRefChainNotContiguous { .. }));
}
fn delta_xml(session_id: &str, serial: u64, elements: &[&str]) -> Vec<u8> {
let mut out = format!(
r#"<delta xmlns="{RRDP_XMLNS}" version="1" session_id="{session_id}" serial="{serial}">"#
);
for e in elements {
out.push_str(e);
}
out.push_str("</delta>");
out.into_bytes()
}
#[test]
fn parse_delta_file_parses_publish_and_withdraw() {
let sid = "550e8400-e29b-41d4-a716-446655440000";
let serial = 3u64;
let publish_bytes = b"abc";
let publish_b64 = base64::engine::general_purpose::STANDARD.encode(publish_bytes);
let withdraw_hash = "33".repeat(32);
let xml = delta_xml(
sid,
serial,
&[
&format!(r#"<publish uri="rsync://example.net/repo/a.mft">{publish_b64}</publish>"#),
&format!(r#"<withdraw uri="rsync://example.net/repo/b.cer" hash="{withdraw_hash}"/>"#),
],
);
let d = parse_delta_file(&xml).expect("parse delta");
assert_eq!(d.session_id, Uuid::parse_str(sid).unwrap());
assert_eq!(d.serial, serial);
assert_eq!(d.elements.len(), 2);
match &d.elements[0] {
DeltaElement::Publish {
uri,
hash_sha256,
bytes,
} => {
assert_eq!(uri, "rsync://example.net/repo/a.mft");
assert_eq!(*hash_sha256, None);
assert_eq!(bytes, publish_bytes);
}
_ => panic!("expected publish"),
}
match &d.elements[1] {
DeltaElement::Withdraw { uri, hash_sha256 } => {
assert_eq!(uri, "rsync://example.net/repo/b.cer");
assert_eq!(hex::encode(hash_sha256), withdraw_hash);
}
_ => panic!("expected withdraw"),
}
}
#[test]
fn parse_delta_file_rejects_withdraw_with_content() {
let sid = "550e8400-e29b-41d4-a716-446655440000";
let serial = 1u64;
let withdraw_hash = "33".repeat(32);
let xml = delta_xml(
sid,
serial,
&[&format!(
r#"<withdraw uri="rsync://example.net/repo/b.cer" hash="{withdraw_hash}">AA==</withdraw>"#
)],
);
let err = parse_delta_file(&xml).unwrap_err();
assert!(matches!(err, RrdpError::DeltaWithdrawUnexpectedContent));
}

View File

@@ -0,0 +1,470 @@
// RRDP test group: sync.
#[test]
fn sync_from_notification_snapshot_rejects_cross_source_owner_conflict() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let sid_a = "550e8400-e29b-41d4-a716-446655440000";
let sid_b = "550e8400-e29b-41d4-a716-446655440001";
let uri = "rsync://example.net/repo/a.mft";
let notif_a_uri = "https://example.net/a/notification.xml";
let snapshot_a_uri = "https://example.net/a/snapshot.xml";
let snapshot_a = snapshot_xml(sid_a, 1, &[(uri, b"a1")]);
let snapshot_a_hash = hex::encode(sha2::Sha256::digest(&snapshot_a));
let notif_a = notification_xml(sid_a, 1, snapshot_a_uri, &snapshot_a_hash);
let fetcher_a = MapFetcher {
map: HashMap::from([(snapshot_a_uri.to_string(), snapshot_a)]),
};
sync_from_notification_snapshot(&store, notif_a_uri, &notif_a, &fetcher_a)
.expect("seed source a");
let notif_b_uri = "https://example.net/b/notification.xml";
let snapshot_b_uri = "https://example.net/b/snapshot.xml";
let snapshot_b = snapshot_xml(sid_b, 1, &[(uri, b"b1")]);
let snapshot_b_hash = hex::encode(sha2::Sha256::digest(&snapshot_b));
let notif_b = notification_xml(sid_b, 1, snapshot_b_uri, &snapshot_b_hash);
let fetcher_b = MapFetcher {
map: HashMap::from([(snapshot_b_uri.to_string(), snapshot_b)]),
};
let err = sync_from_notification_snapshot(&store, notif_b_uri, &notif_b, &fetcher_b)
.expect_err("cross-source overwrite must fail");
assert!(matches!(err, RrdpSyncError::Storage(_)));
assert!(err.to_string().contains("owner conflict"), "{err}");
}
struct PanicOnRrdpFetch;
impl Fetcher for PanicOnRrdpFetch {
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
panic!("cross-origin notification must be rejected before fetch: {uri}");
}
}
#[test]
fn cross_origin_snapshot_reference_is_rejected_before_fetch_or_state_write() {
let temp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(temp.path()).expect("open rocksdb");
let notification_uri = "https://origin.example.test/notification.xml";
let notification = notification_xml(
"550e8400-e29b-41d4-a716-446655440000",
1,
"https://foreign.example.test/snapshot.xml",
&"00".repeat(32),
);
let err =
sync_from_notification_snapshot(&store, notification_uri, &notification, &PanicOnRrdpFetch)
.expect_err("cross-origin snapshot must be rejected");
assert!(matches!(
err,
RrdpSyncError::Rrdp(RrdpError::CrossOriginReference {
resource: RrdpResourceKind::Snapshot,
..
})
));
assert!(
load_rrdp_local_state(&store, notification_uri)
.expect("read state")
.is_none(),
"rejection must precede RRDP state writes"
);
}
#[test]
fn cross_origin_delta_reference_is_rejected_even_when_delta_would_not_be_used() {
let temp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(temp.path()).expect("open rocksdb");
let notification_uri = "https://origin.example.test/notification.xml";
let notification = notification_xml_with_deltas(
"550e8400-e29b-41d4-a716-446655440000",
1,
"https://origin.example.test/snapshot.xml",
&"00".repeat(32),
&[(
"delta-1",
1,
"https://foreign.example.test/delta-1.xml",
&"11".repeat(32),
)],
);
let err = sync_from_notification(&store, notification_uri, &notification, &PanicOnRrdpFetch)
.expect_err("cross-origin delta must be rejected before state/fetch");
assert!(matches!(
err,
RrdpSyncError::Rrdp(RrdpError::CrossOriginReference {
resource: RrdpResourceKind::Delta,
..
})
));
assert!(
load_rrdp_local_state(&store, notification_uri)
.expect("read state")
.is_none(),
"all delta references are checked before state reads or writes"
);
}
#[test]
fn rrdp_origin_treats_https_default_port_as_equivalent() {
let notification = parse_notification(&notification_xml(
"550e8400-e29b-41d4-a716-446655440000",
1,
"https://origin.example.test:443/snapshot.xml",
&"00".repeat(32),
))
.expect("notification");
validate_notification_references(
"https://origin.example.test/notification.xml",
&notification,
)
.expect("explicit HTTPS default port must remain same-origin");
}
#[test]
fn sync_from_notification_snapshot_applies_snapshot_and_stores_state() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let sid = "550e8400-e29b-41d4-a716-446655440000";
let serial = 9u64;
let notif_uri = "https://example.net/notification.xml";
let snapshot_uri = "https://example.net/snapshot.xml";
let snapshot = snapshot_xml(
sid,
serial,
&[
("rsync://example.net/repo/a.mft", b"mft-bytes"),
("rsync://example.net/repo/b.roa", b"roa-bytes"),
],
);
let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot));
let notif = notification_xml(sid, serial, snapshot_uri, &snapshot_hash);
let fetcher = MapFetcher {
map: HashMap::from([(snapshot_uri.to_string(), snapshot.clone())]),
};
let published =
sync_from_notification_snapshot(&store, notif_uri, &notif, &fetcher).expect("sync");
assert_eq!(published, 2);
assert_current_object(&store, "rsync://example.net/repo/a.mft", b"mft-bytes");
assert_current_object(&store, "rsync://example.net/repo/b.roa", b"roa-bytes");
let state = load_rrdp_local_state(&store, notif_uri)
.expect("get rrdp state")
.expect("state present");
assert_eq!(state.session_id, sid);
assert_eq!(state.serial, serial);
let source = store
.get_rrdp_source_record(notif_uri)
.expect("get rrdp source")
.expect("rrdp source exists");
assert_eq!(source.last_session_id.as_deref(), Some(sid));
assert_eq!(source.last_serial, Some(serial));
assert_eq!(
source.sync_state,
crate::repository::storage::RrdpSourceSyncState::SnapshotOnly
);
let view = store
.get_repository_view_entry("rsync://example.net/repo/a.mft")
.expect("get repository view")
.expect("repository view exists");
assert_eq!(view.state, crate::repository::storage::RepositoryViewState::Present);
assert_eq!(view.repository_source.as_deref(), Some(notif_uri));
let current_bytes = store
.load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft")
.expect("load current bytes")
.expect("current object bytes exist");
assert_eq!(current_bytes, b"mft-bytes".to_vec());
assert!(
store
.get_raw_by_hash_entry(hex::encode(sha2::Sha256::digest(b"mft-bytes")).as_str())
.expect("get raw_by_hash")
.is_none()
);
let member = store
.get_rrdp_source_member_record(notif_uri, "rsync://example.net/repo/a.mft")
.expect("get member")
.expect("member exists");
assert!(member.present);
let owner = store
.get_rrdp_uri_owner_record("rsync://example.net/repo/a.mft")
.expect("get owner")
.expect("owner exists");
assert_eq!(owner.notify_uri, notif_uri);
assert_eq!(owner.owner_state, crate::repository::storage::RrdpUriOwnerState::Active);
}
#[test]
fn sync_from_notification_snapshot_deletes_objects_not_in_new_snapshot() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let sid = "550e8400-e29b-41d4-a716-446655440000";
let notif_uri = "https://example.net/notification.xml";
// serial 1: publish a + b
let snapshot_uri_1 = "https://example.net/snapshot-1.xml";
let snapshot_1 = snapshot_xml(
sid,
1,
&[
("rsync://example.net/repo/a.mft", b"a1"),
("rsync://example.net/repo/b.roa", b"b1"),
],
);
let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1));
let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1);
let fetcher_1 = MapFetcher {
map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]),
};
sync_from_notification_snapshot(&store, notif_uri, &notif_1, &fetcher_1).expect("sync 1");
// serial 2: publish b (new bytes) + c, and drop a
let snapshot_uri_2 = "https://example.net/snapshot-2.xml";
let snapshot_2 = snapshot_xml(
sid,
2,
&[
("rsync://example.net/repo/b.roa", b"b2"),
("rsync://example.net/repo/c.crl", b"c2"),
],
);
let snapshot_hash_2 = hex::encode(sha2::Sha256::digest(&snapshot_2));
let notif_2 = notification_xml(sid, 2, snapshot_uri_2, &snapshot_hash_2);
let fetcher_2 = MapFetcher {
map: HashMap::from([(snapshot_uri_2.to_string(), snapshot_2)]),
};
sync_from_notification_snapshot(&store, notif_uri, &notif_2, &fetcher_2).expect("sync 2");
assert!(
store
.load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft")
.expect("get current a")
.is_none(),
"a should be deleted by full-state snapshot apply"
);
assert_current_object(&store, "rsync://example.net/repo/b.roa", b"b2");
assert_current_object(&store, "rsync://example.net/repo/c.crl", b"c2");
}
#[test]
fn sync_from_notification_uses_deltas_when_available_for_local_state() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let sid = "550e8400-e29b-41d4-a716-446655440000";
let notif_uri = "https://example.net/notification.xml";
// Seed state with snapshot serial=1 containing a+b.
let snapshot_uri_1 = "https://example.net/snapshot-1.xml";
let snapshot_1 = snapshot_xml(
sid,
1,
&[
("rsync://example.net/repo/a.mft", b"a1"),
("rsync://example.net/repo/b.roa", b"b1"),
],
);
let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1));
let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1);
let fetcher_1 = MapFetcher {
map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]),
};
sync_from_notification_snapshot(&store, notif_uri, &notif_1, &fetcher_1).expect("seed");
// Notification serial=3 with deltas 2 and 3. Snapshot URI is intentionally not fetchable
// to assert we really use deltas.
let snapshot_uri_3 = "https://example.net/snapshot-3.xml";
let snapshot_hash_3 = "00".repeat(32);
let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2");
let delta_2 = delta_xml(
sid,
2,
&[&format!(
r#"<publish uri="rsync://example.net/repo/c.crl">{publish_c_b64}</publish>"#
)],
);
let delta_2_hash_hex = hex::encode(sha2::Sha256::digest(&delta_2));
let c_hash_hex = hex::encode(sha2::Sha256::digest(b"c2".as_slice()));
let delta_3 = delta_xml(
sid,
3,
&[&format!(
r#"<withdraw uri="rsync://example.net/repo/c.crl" hash="{c_hash_hex}"/>"#
)],
);
let delta_3_hash_hex = hex::encode(sha2::Sha256::digest(&delta_3));
let notif_3 = notification_xml_with_deltas(
sid,
3,
snapshot_uri_3,
&snapshot_hash_3,
&[
(
"d3",
3,
"https://example.net/delta-3.xml",
&delta_3_hash_hex,
),
(
"d2",
2,
"https://example.net/delta-2.xml",
&delta_2_hash_hex,
),
],
);
let fetcher = MapFetcher {
map: HashMap::from([
("https://example.net/delta-2.xml".to_string(), delta_2),
("https://example.net/delta-3.xml".to_string(), delta_3),
]),
};
let applied = sync_from_notification(&store, notif_uri, &notif_3, &fetcher).expect("sync");
assert!(applied > 0);
// Delta 2 publishes c then delta 3 withdraws it => final state should not contain c.
assert!(
store
.load_current_object_bytes_by_uri("rsync://example.net/repo/c.crl")
.expect("get current")
.is_none()
);
let state = load_rrdp_local_state(&store, notif_uri)
.expect("get rrdp state")
.expect("state present");
assert_eq!(state.session_id, Uuid::parse_str(sid).unwrap().to_string());
assert_eq!(state.serial, 3);
}
#[test]
fn sync_from_notification_same_serial_hydrates_current_repo_index() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let sid = "550e8400-e29b-41d4-a716-446655440000";
let notif_uri = "https://example.net/notification.xml";
let snapshot_uri = "https://example.net/snapshot.xml";
let uri_a = "rsync://example.net/repo/a.mft";
let uri_b = "rsync://example.net/repo/b.roa";
let snapshot = snapshot_xml(sid, 1, &[(uri_a, b"a1"), (uri_b, b"b1")]);
let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot));
let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash);
let fetcher_1 = MapFetcher {
map: HashMap::from([(snapshot_uri.to_string(), snapshot)]),
};
sync_from_notification_snapshot(&store, notif_uri, &notif, &fetcher_1).expect("seed");
let index = CurrentRepoIndex::shared();
let no_fetcher = MapFetcher {
map: HashMap::new(),
};
let applied = sync_from_notification_with_timing_and_download_log(
&store,
notif_uri,
Some(&index),
&notif,
&no_fetcher,
None,
None,
)
.expect("same serial no-op");
assert_eq!(applied, 0);
let index = index.read().expect("read-lock index");
assert_eq!(index.active_uri_count(), 2);
assert!(index.get_by_uri(uri_a).is_some());
assert!(index.get_by_uri(uri_b).is_some());
}
#[test]
fn sync_from_notification_delta_hydrates_unchanged_current_repo_entries() {
let tmp = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
let sid = "550e8400-e29b-41d4-a716-446655440000";
let notif_uri = "https://example.net/notification.xml";
let snapshot_uri_1 = "https://example.net/snapshot-1.xml";
let uri_a = "rsync://example.net/repo/a.mft";
let uri_b = "rsync://example.net/repo/b.roa";
let uri_c = "rsync://example.net/repo/c.crl";
let snapshot_1 = snapshot_xml(sid, 1, &[(uri_a, b"a1"), (uri_b, b"b1")]);
let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1));
let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1);
let fetcher_1 = MapFetcher {
map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]),
};
sync_from_notification_snapshot(&store, notif_uri, &notif_1, &fetcher_1).expect("seed");
let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2");
let delta_2 = delta_xml(
sid,
2,
&[&format!(
r#"<publish uri="{uri_c}">{publish_c_b64}</publish>"#
)],
);
let delta_2_hash_hex = hex::encode(sha2::Sha256::digest(&delta_2));
let notif_2 = notification_xml_with_deltas(
sid,
2,
"https://example.net/snapshot-2.xml",
&"00".repeat(32),
&[(
"d2",
2,
"https://example.net/delta-2.xml",
&delta_2_hash_hex,
)],
);
let fetcher_2 = MapFetcher {
map: HashMap::from([("https://example.net/delta-2.xml".to_string(), delta_2)]),
};
let index = CurrentRepoIndex::shared();
let applied = sync_from_notification_with_timing_and_download_log(
&store,
notif_uri,
Some(&index),
&notif_2,
&fetcher_2,
None,
None,
)
.expect("delta sync");
assert_eq!(applied, 1);
let index = index.read().expect("read-lock index");
assert_eq!(index.active_uri_count(), 3);
assert!(
index.get_by_uri(uri_a).is_some(),
"unchanged object from the previous serial must be visible"
);
assert!(
index.get_by_uri(uri_b).is_some(),
"unchanged object from the previous serial must be visible"
);
assert!(index.get_by_uri(uri_c).is_some(), "delta publish visible");
}

View File

@@ -0,0 +1,510 @@
use std::collections::BTreeMap;
use crate::repository::storage::{
PackTime, RawByHashEntry, RepositoryViewEntry, RepositoryViewState, RocksStore,
RrdpSourceMemberRecord, RrdpSourceRecord, RrdpSourceSyncState, RrdpUriOwnerRecord,
RrdpUriOwnerState,
};
use sha2::Digest;
#[allow(dead_code)]
pub struct PreparedRawByHashBatch {
pub uri_to_hash: BTreeMap<String, String>,
pub entries_to_write: Vec<RawByHashEntry>,
}
pub struct PreparedRepoBytesBatch {
pub uri_to_hash: BTreeMap<String, String>,
pub blobs_to_write: Vec<(String, Vec<u8>)>,
}
pub fn prepare_repo_bytes_batch(
objects: &[(String, Vec<u8>)],
) -> Result<PreparedRepoBytesBatch, String> {
prepare_repo_bytes_batch_owned(objects.to_vec())
}
pub fn prepare_repo_bytes_batch_owned(
objects: Vec<(String, Vec<u8>)>,
) -> Result<PreparedRepoBytesBatch, String> {
let mut uri_to_hash: BTreeMap<String, String> = BTreeMap::new();
let mut pending: BTreeMap<String, Vec<u8>> = BTreeMap::new();
for (uri, bytes) in objects {
if bytes.is_empty() {
return Err(format!("repo bytes for {uri} must not be empty"));
}
let sha256_hex = compute_sha256_hex(&bytes);
uri_to_hash.insert(uri.clone(), sha256_hex.clone());
match pending.entry(sha256_hex) {
std::collections::btree_map::Entry::Vacant(slot) => {
slot.insert(bytes);
}
std::collections::btree_map::Entry::Occupied(existing) => {
if existing.get() != &bytes {
return Err(format!(
"repo bytes collision for {uri}: same sha256 maps to different bytes"
));
}
}
}
}
Ok(PreparedRepoBytesBatch {
uri_to_hash,
blobs_to_write: pending.into_iter().collect(),
})
}
#[allow(dead_code)]
pub fn prepare_raw_by_hash_evidence_batch(
store: &RocksStore,
objects: &[(String, Vec<u8>)],
) -> Result<PreparedRawByHashBatch, String> {
let mut pending: BTreeMap<String, RawByHashEntry> = BTreeMap::new();
let mut uri_to_hash: BTreeMap<String, String> = BTreeMap::new();
for (uri, bytes) in objects {
let sha256_hex = compute_sha256_hex(bytes);
uri_to_hash.insert(uri.clone(), sha256_hex.clone());
let entry = pending
.entry(sha256_hex.clone())
.or_insert_with(|| RawByHashEntry::from_bytes(sha256_hex.clone(), bytes.clone()));
if entry.bytes != *bytes {
return Err(format!(
"raw_by_hash collision for {uri}: same sha256 maps to different bytes"
));
}
if !entry.origin_uris.iter().any(|existing| existing == uri) {
entry.origin_uris.push(uri.clone());
}
if entry.object_type.is_none() {
entry.object_type = infer_object_type_from_uri(uri);
}
}
let hashes: Vec<String> = pending.keys().cloned().collect();
let existing_entries = store
.get_raw_by_hash_entries_batch(&hashes)
.map_err(|e| e.to_string())?;
let mut entries_to_write = Vec::new();
for (hash, existing_opt) in hashes.into_iter().zip(existing_entries.into_iter()) {
let mut pending_entry = pending.remove(&hash).expect("pending raw_by_hash entry");
match existing_opt {
Some(mut existing) => {
if existing.bytes != pending_entry.bytes {
return Err(format!(
"raw_by_hash collision for hash {hash}: same sha256 maps to different bytes"
));
}
let mut changed = false;
for uri in pending_entry.origin_uris.drain(..) {
if !existing
.origin_uris
.iter()
.any(|existing_uri| existing_uri == &uri)
{
existing.origin_uris.push(uri);
changed = true;
}
}
if existing.object_type.is_none() && pending_entry.object_type.is_some() {
existing.object_type = pending_entry.object_type;
changed = true;
}
if changed {
entries_to_write.push(existing);
}
}
None => entries_to_write.push(pending_entry),
}
}
Ok(PreparedRawByHashBatch {
uri_to_hash,
entries_to_write,
})
}
pub fn upsert_repo_blob_bytes(store: &RocksStore, bytes: &[u8]) -> Result<String, String> {
if bytes.is_empty() {
return Err("repo bytes must not be empty".to_string());
}
let sha256_hex = compute_sha256_hex(bytes);
store
.put_blob_bytes_batch(&[(sha256_hex.clone(), bytes.to_vec())])
.map_err(|e| e.to_string())?;
Ok(sha256_hex)
}
pub fn infer_object_type_from_uri(uri: &str) -> Option<String> {
let ext = uri.rsplit('.').next()?;
let ext = ext.to_ascii_lowercase();
match ext.as_str() {
"cer" | "crl" | "mft" | "roa" | "asa" | "gbr" | "tal" | "xml" => Some(ext),
_ => None,
}
}
pub fn build_repository_view_present_entry(
repository_source: &str,
rsync_uri: &str,
current_hash: &str,
) -> RepositoryViewEntry {
RepositoryViewEntry {
rsync_uri: rsync_uri.to_string(),
current_hash: Some(current_hash.to_string()),
repository_source: Some(repository_source.to_string()),
object_type: infer_object_type_from_uri(rsync_uri),
state: RepositoryViewState::Present,
}
}
pub fn build_repository_view_withdrawn_entry(
repository_source: &str,
rsync_uri: &str,
current_hash: Option<String>,
) -> RepositoryViewEntry {
RepositoryViewEntry {
rsync_uri: rsync_uri.to_string(),
current_hash,
repository_source: Some(repository_source.to_string()),
object_type: infer_object_type_from_uri(rsync_uri),
state: RepositoryViewState::Withdrawn,
}
}
pub fn build_rrdp_source_member_present_record(
notification_uri: &str,
session_id: &str,
serial: u64,
rsync_uri: &str,
current_hash: &str,
) -> RrdpSourceMemberRecord {
RrdpSourceMemberRecord {
notify_uri: notification_uri.to_string(),
rsync_uri: rsync_uri.to_string(),
current_hash: Some(current_hash.to_string()),
object_type: infer_object_type_from_uri(rsync_uri),
present: true,
last_confirmed_session_id: session_id.to_string(),
last_confirmed_serial: serial,
last_changed_at: now_pack_time(),
}
}
pub fn build_rrdp_source_member_withdrawn_record(
notification_uri: &str,
session_id: &str,
serial: u64,
rsync_uri: &str,
current_hash: Option<String>,
) -> RrdpSourceMemberRecord {
RrdpSourceMemberRecord {
notify_uri: notification_uri.to_string(),
rsync_uri: rsync_uri.to_string(),
current_hash,
object_type: infer_object_type_from_uri(rsync_uri),
present: false,
last_confirmed_session_id: session_id.to_string(),
last_confirmed_serial: serial,
last_changed_at: now_pack_time(),
}
}
pub fn build_rrdp_uri_owner_active_record(
notification_uri: &str,
session_id: &str,
serial: u64,
rsync_uri: &str,
current_hash: &str,
) -> RrdpUriOwnerRecord {
RrdpUriOwnerRecord {
rsync_uri: rsync_uri.to_string(),
notify_uri: notification_uri.to_string(),
current_hash: Some(current_hash.to_string()),
last_confirmed_session_id: session_id.to_string(),
last_confirmed_serial: serial,
last_changed_at: now_pack_time(),
owner_state: RrdpUriOwnerState::Active,
}
}
pub fn build_rrdp_uri_owner_withdrawn_record(
notification_uri: &str,
session_id: &str,
serial: u64,
rsync_uri: &str,
current_hash: Option<String>,
) -> RrdpUriOwnerRecord {
RrdpUriOwnerRecord {
rsync_uri: rsync_uri.to_string(),
notify_uri: notification_uri.to_string(),
current_hash,
last_confirmed_session_id: session_id.to_string(),
last_confirmed_serial: serial,
last_changed_at: now_pack_time(),
owner_state: RrdpUriOwnerState::Withdrawn,
}
}
#[allow(dead_code)]
pub fn upsert_raw_by_hash_evidence(
store: &RocksStore,
rsync_uri: &str,
bytes: &[u8],
) -> Result<String, String> {
let prepared =
prepare_raw_by_hash_evidence_batch(store, &[(rsync_uri.to_string(), bytes.to_vec())])?;
store
.put_raw_by_hash_entries_batch_unchecked(&prepared.entries_to_write)
.map_err(|e| e.to_string())?;
let sha256_hex = prepared
.uri_to_hash
.get(rsync_uri)
.cloned()
.expect("raw_by_hash mapping for upsert input");
Ok(sha256_hex)
}
pub fn put_repository_view_present(
store: &RocksStore,
repository_source: &str,
rsync_uri: &str,
current_hash: &str,
) -> Result<(), String> {
let entry = build_repository_view_present_entry(repository_source, rsync_uri, current_hash);
store
.put_repository_view_entry(&entry)
.map_err(|e| e.to_string())
}
pub fn put_repository_view_withdrawn(
store: &RocksStore,
repository_source: &str,
rsync_uri: &str,
current_hash: Option<String>,
) -> Result<(), String> {
let entry = build_repository_view_withdrawn_entry(repository_source, rsync_uri, current_hash);
store
.put_repository_view_entry(&entry)
.map_err(|e| e.to_string())
}
pub fn ensure_rrdp_uri_can_be_owned_by(
store: &RocksStore,
notification_uri: &str,
rsync_uri: &str,
) -> Result<(), String> {
let Some(owner) = store
.get_rrdp_uri_owner_record(rsync_uri)
.map_err(|e| e.to_string())?
else {
return Ok(());
};
if owner.notify_uri != notification_uri && owner.owner_state == RrdpUriOwnerState::Active {
return Err(format!(
"RRDP source owner conflict for {rsync_uri}: current owner {} but incoming source {}",
owner.notify_uri, notification_uri
));
}
Ok(())
}
pub fn current_rrdp_owner_is(
store: &RocksStore,
notification_uri: &str,
rsync_uri: &str,
) -> Result<bool, String> {
Ok(matches!(
store
.get_rrdp_uri_owner_record(rsync_uri)
.map_err(|e| e.to_string())?,
Some(owner)
if owner.notify_uri == notification_uri && owner.owner_state == RrdpUriOwnerState::Active
))
}
pub fn put_rrdp_source_member_present(
store: &RocksStore,
notification_uri: &str,
session_id: &str,
serial: u64,
rsync_uri: &str,
current_hash: &str,
) -> Result<(), String> {
let record = build_rrdp_source_member_present_record(
notification_uri,
session_id,
serial,
rsync_uri,
current_hash,
);
store
.put_rrdp_source_member_record(&record)
.map_err(|e| e.to_string())
}
pub fn put_rrdp_source_member_withdrawn(
store: &RocksStore,
notification_uri: &str,
session_id: &str,
serial: u64,
rsync_uri: &str,
current_hash: Option<String>,
) -> Result<(), String> {
let record = build_rrdp_source_member_withdrawn_record(
notification_uri,
session_id,
serial,
rsync_uri,
current_hash,
);
store
.put_rrdp_source_member_record(&record)
.map_err(|e| e.to_string())
}
pub fn put_rrdp_uri_owner_active(
store: &RocksStore,
notification_uri: &str,
session_id: &str,
serial: u64,
rsync_uri: &str,
current_hash: &str,
) -> Result<(), String> {
let record = build_rrdp_uri_owner_active_record(
notification_uri,
session_id,
serial,
rsync_uri,
current_hash,
);
store
.put_rrdp_uri_owner_record(&record)
.map_err(|e| e.to_string())
}
pub fn put_rrdp_uri_owner_withdrawn(
store: &RocksStore,
notification_uri: &str,
session_id: &str,
serial: u64,
rsync_uri: &str,
current_hash: Option<String>,
) -> Result<(), String> {
let record = build_rrdp_uri_owner_withdrawn_record(
notification_uri,
session_id,
serial,
rsync_uri,
current_hash,
);
store
.put_rrdp_uri_owner_record(&record)
.map_err(|e| e.to_string())
}
pub fn update_rrdp_source_record_on_success(
store: &RocksStore,
notification_uri: &str,
session_id: &str,
serial: u64,
sync_state: RrdpSourceSyncState,
last_snapshot_uri: Option<&str>,
last_snapshot_hash_hex: Option<&str>,
) -> Result<(), String> {
let now = now_pack_time();
let mut record = match store
.get_rrdp_source_record(notification_uri)
.map_err(|e| e.to_string())?
{
Some(existing) => existing,
None => RrdpSourceRecord {
notify_uri: notification_uri.to_string(),
last_session_id: None,
last_serial: None,
first_seen_at: now.clone(),
last_seen_at: now.clone(),
last_sync_at: None,
sync_state,
last_snapshot_uri: None,
last_snapshot_hash: None,
last_error: None,
},
};
record.last_session_id = Some(session_id.to_string());
record.last_serial = Some(serial);
record.last_seen_at = now.clone();
record.last_sync_at = Some(now);
record.sync_state = sync_state;
record.last_snapshot_uri = last_snapshot_uri.map(str::to_string);
record.last_snapshot_hash = last_snapshot_hash_hex.map(str::to_string);
record.last_error = None;
store
.put_rrdp_source_record(&record)
.map_err(|e| e.to_string())
}
pub fn compute_sha256_hex(bytes: &[u8]) -> String {
hex::encode(sha2::Sha256::digest(bytes))
}
pub fn now_pack_time() -> PackTime {
PackTime::from_utc_offset_datetime(time::OffsetDateTime::now_utc())
}
#[cfg(test)]
mod tests {
use super::{prepare_repo_bytes_batch, prepare_repo_bytes_batch_owned};
use std::collections::BTreeSet;
#[test]
fn prepare_repo_bytes_batch_deduplicates_by_hash() {
let objects = vec![
(
"rsync://example.test/repo/a.roa".to_string(),
b"same".to_vec(),
),
(
"rsync://example.test/repo/b.roa".to_string(),
b"same".to_vec(),
),
(
"rsync://example.test/repo/c.roa".to_string(),
b"other".to_vec(),
),
];
let prepared = prepare_repo_bytes_batch(&objects).expect("prepare repo bytes");
assert_eq!(prepared.uri_to_hash.len(), 3);
assert_eq!(prepared.blobs_to_write.len(), 2);
let unique_hashes = prepared
.blobs_to_write
.iter()
.map(|(hash, _)| hash.clone())
.collect::<BTreeSet<_>>();
assert_eq!(unique_hashes.len(), 2);
}
#[test]
fn prepare_repo_bytes_batch_owned_deduplicates_without_borrowed_input() {
let objects = vec![
(
"rsync://example.test/repo/a.roa".to_string(),
b"same".to_vec(),
),
(
"rsync://example.test/repo/b.roa".to_string(),
b"same".to_vec(),
),
];
let prepared = prepare_repo_bytes_batch_owned(objects).expect("prepare repo bytes");
assert_eq!(prepared.uri_to_hash.len(), 2);
assert_eq!(prepared.blobs_to_write.len(), 1);
}
}