初始化 Panda RPKI v0.1.0 开源候选版本
This commit is contained in:
835
src/repository/blob_store.rs
Normal file
835
src/repository/blob_store.rs
Normal file
@@ -0,0 +1,835 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use rocksdb::{DB, Options, WriteBatch};
|
||||
|
||||
use crate::repository::storage::{
|
||||
RawByHashEntry, RocksDbMemoryDbSnapshot, RocksStore, StorageError, StorageResult,
|
||||
memory_db_snapshot_for_column_families,
|
||||
};
|
||||
|
||||
const RAW_BY_HASH_KEY_PREFIX: &str = "rawbyhash:";
|
||||
const RAW_BLOB_KEY_PREFIX: &str = "rawblob:";
|
||||
const REPO_BYTES_KEY_PREFIX: &str = "sha256:";
|
||||
|
||||
fn raw_by_hash_key(sha256_hex: &str) -> String {
|
||||
format!("{RAW_BY_HASH_KEY_PREFIX}{sha256_hex}")
|
||||
}
|
||||
|
||||
fn raw_blob_key(sha256_hex: &str) -> String {
|
||||
format!("{RAW_BLOB_KEY_PREFIX}{sha256_hex}")
|
||||
}
|
||||
|
||||
fn repo_bytes_key(sha256_hex: &str) -> String {
|
||||
format!("{REPO_BYTES_KEY_PREFIX}{sha256_hex}")
|
||||
}
|
||||
|
||||
fn validate_blob_sha256_hex(sha256_hex: &str) -> StorageResult<()> {
|
||||
if sha256_hex.len() != 64 || !sha256_hex.as_bytes().iter().all(u8::is_ascii_hexdigit) {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: "raw_blob",
|
||||
detail: format!("invalid sha256 hex: {sha256_hex}"),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_blob_bytes(bytes: &[u8]) -> StorageResult<()> {
|
||||
if bytes.is_empty() {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: "raw_blob",
|
||||
detail: "bytes must not be empty".to_string(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub trait RawObjectStore {
|
||||
fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>>;
|
||||
|
||||
fn get_raw_entries_batch(
|
||||
&self,
|
||||
sha256_hexes: &[String],
|
||||
) -> StorageResult<Vec<Option<RawByHashEntry>>>;
|
||||
|
||||
fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult<Option<Vec<u8>>> {
|
||||
self.get_raw_entry(sha256_hex)
|
||||
.map(|entry| entry.map(|entry| entry.bytes))
|
||||
}
|
||||
|
||||
fn get_blob_bytes_batch(&self, sha256_hexes: &[String]) -> StorageResult<Vec<Option<Vec<u8>>>> {
|
||||
self.get_raw_entries_batch(sha256_hexes).map(|entries| {
|
||||
entries
|
||||
.into_iter()
|
||||
.map(|entry| entry.map(|entry| entry.bytes))
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ExternalRawStoreDb {
|
||||
path: PathBuf,
|
||||
db: Arc<DB>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ExternalRepoBytesDb {
|
||||
path: PathBuf,
|
||||
db: Arc<DB>,
|
||||
read_only: bool,
|
||||
secondary: bool,
|
||||
}
|
||||
|
||||
impl ExternalRawStoreDb {
|
||||
pub fn open(path: impl Into<PathBuf>) -> StorageResult<Self> {
|
||||
let path = path.into();
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
}
|
||||
let mut opts = Options::default();
|
||||
opts.create_if_missing(true);
|
||||
opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
|
||||
opts.set_max_open_files(512);
|
||||
let db = DB::open(&opts, &path).map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(Self {
|
||||
path,
|
||||
db: Arc::new(db),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn put_raw_entry(&self, entry: &RawByHashEntry) -> StorageResult<()> {
|
||||
entry.validate_internal()?;
|
||||
let key = raw_by_hash_key(&entry.sha256_hex);
|
||||
let blob_key = raw_blob_key(&entry.sha256_hex);
|
||||
let value = serde_cbor::to_vec(entry).map_err(|e| StorageError::Codec {
|
||||
entity: "raw_by_hash",
|
||||
detail: e.to_string(),
|
||||
})?;
|
||||
let blob_value = entry.bytes.clone();
|
||||
self.db
|
||||
.write({
|
||||
let mut batch = WriteBatch::default();
|
||||
batch.put(key.as_bytes(), value);
|
||||
batch.put(blob_key.as_bytes(), blob_value);
|
||||
batch
|
||||
})
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn put_raw_entries_batch(&self, entries: &[RawByHashEntry]) -> StorageResult<()> {
|
||||
if entries.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut batch = WriteBatch::default();
|
||||
for entry in entries {
|
||||
entry.validate_internal()?;
|
||||
let key = raw_by_hash_key(&entry.sha256_hex);
|
||||
let blob_key = raw_blob_key(&entry.sha256_hex);
|
||||
let value = serde_cbor::to_vec(entry).map_err(|e| StorageError::Codec {
|
||||
entity: "raw_by_hash",
|
||||
detail: e.to_string(),
|
||||
})?;
|
||||
batch.put(key.as_bytes(), value);
|
||||
batch.put(blob_key.as_bytes(), entry.bytes.as_slice());
|
||||
}
|
||||
self.db
|
||||
.write(batch)
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn put_blob_bytes_batch(&self, blobs: &[(String, Vec<u8>)]) -> StorageResult<()> {
|
||||
if blobs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut batch = WriteBatch::default();
|
||||
for (sha256_hex, bytes) in blobs {
|
||||
validate_blob_sha256_hex(sha256_hex)?;
|
||||
validate_blob_bytes(bytes)?;
|
||||
let blob_key = raw_blob_key(sha256_hex);
|
||||
batch.put(blob_key.as_bytes(), bytes.as_slice());
|
||||
}
|
||||
self.db
|
||||
.write(batch)
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn delete_raw_entry(&self, sha256_hex: &str) -> StorageResult<()> {
|
||||
let key = raw_by_hash_key(sha256_hex);
|
||||
let blob_key = raw_blob_key(sha256_hex);
|
||||
self.db
|
||||
.write({
|
||||
let mut batch = WriteBatch::default();
|
||||
batch.delete(key.as_bytes());
|
||||
batch.delete(blob_key.as_bytes());
|
||||
batch
|
||||
})
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &PathBuf {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub(crate) fn memory_snapshot(&self, label: impl Into<String>) -> RocksDbMemoryDbSnapshot {
|
||||
memory_db_snapshot_for_column_families(label, self.db.as_ref(), None)
|
||||
}
|
||||
}
|
||||
|
||||
impl ExternalRepoBytesDb {
|
||||
pub fn open(path: impl Into<PathBuf>) -> StorageResult<Self> {
|
||||
let path = path.into();
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
}
|
||||
let mut opts = Options::default();
|
||||
opts.create_if_missing(true);
|
||||
opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
|
||||
opts.set_max_open_files(512);
|
||||
let db = DB::open(&opts, &path).map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(Self {
|
||||
path,
|
||||
db: Arc::new(db),
|
||||
read_only: false,
|
||||
secondary: false,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn open_read_only(path: impl Into<PathBuf>) -> StorageResult<Self> {
|
||||
let path = path.into();
|
||||
let mut opts = Options::default();
|
||||
opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
|
||||
opts.set_max_open_files(512);
|
||||
let db = DB::open_for_read_only(&opts, &path, false)
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(Self {
|
||||
path,
|
||||
db: Arc::new(db),
|
||||
read_only: true,
|
||||
secondary: false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Open the repo-bytes DB as a RocksDB secondary instance.
|
||||
///
|
||||
/// Unlike `open_read_only` (a frozen point-in-time view), a secondary
|
||||
/// instance can follow the live primary via `try_catch_up_with_primary`,
|
||||
/// which is required when the soak keeps writing new object bytes while
|
||||
/// the query service is running.
|
||||
pub fn open_as_secondary(
|
||||
path: impl Into<PathBuf>,
|
||||
secondary_path: impl Into<PathBuf>,
|
||||
) -> StorageResult<Self> {
|
||||
let path = path.into();
|
||||
let secondary_path = secondary_path.into();
|
||||
if let Some(parent) = secondary_path.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
}
|
||||
let mut opts = Options::default();
|
||||
opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
|
||||
opts.set_max_open_files(512);
|
||||
let db = DB::open_as_secondary(&opts, &path, &secondary_path)
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(Self {
|
||||
path,
|
||||
db: Arc::new(db),
|
||||
read_only: true,
|
||||
secondary: true,
|
||||
})
|
||||
}
|
||||
|
||||
/// Pull the secondary view up to the primary's current state. No-op for
|
||||
/// non-secondary handles.
|
||||
pub fn try_catch_up_with_primary(&self) -> StorageResult<()> {
|
||||
if self.secondary {
|
||||
self.db
|
||||
.try_catch_up_with_primary()
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn put_blob_bytes_batch(&self, blobs: &[(String, Vec<u8>)]) -> StorageResult<()> {
|
||||
if blobs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if self.read_only {
|
||||
return Err(StorageError::RocksDb(format!(
|
||||
"repo-bytes DB is read-only: {}",
|
||||
self.path.display()
|
||||
)));
|
||||
}
|
||||
let mut batch = WriteBatch::default();
|
||||
for (sha256_hex, bytes) in blobs {
|
||||
validate_blob_sha256_hex(sha256_hex)?;
|
||||
validate_blob_bytes(bytes)?;
|
||||
let key = repo_bytes_key(sha256_hex);
|
||||
batch.put(key.as_bytes(), bytes.as_slice());
|
||||
}
|
||||
self.db
|
||||
.write(batch)
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn is_read_only(&self) -> bool {
|
||||
self.read_only
|
||||
}
|
||||
|
||||
pub(crate) fn require_existing_blob_bytes_batch(
|
||||
&self,
|
||||
blobs: &[(String, Vec<u8>)],
|
||||
) -> StorageResult<()> {
|
||||
if blobs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let hashes = blobs
|
||||
.iter()
|
||||
.map(|(sha256_hex, _)| sha256_hex.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let existing = self.get_blob_bytes_batch(&hashes)?;
|
||||
for ((sha256_hex, expected_bytes), actual_bytes) in blobs.iter().zip(existing) {
|
||||
match actual_bytes {
|
||||
Some(actual_bytes) if actual_bytes == *expected_bytes => {}
|
||||
Some(_) => {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: "read_only_repo_bytes",
|
||||
detail: format!("existing bytes differ for SHA-256 {sha256_hex}"),
|
||||
});
|
||||
}
|
||||
None => {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: "read_only_repo_bytes",
|
||||
detail: format!("blob is missing for SHA-256 {sha256_hex}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult<Option<Vec<u8>>> {
|
||||
validate_blob_sha256_hex(sha256_hex)?;
|
||||
let key = repo_bytes_key(sha256_hex);
|
||||
self.db
|
||||
.get(key.as_bytes())
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))
|
||||
}
|
||||
|
||||
pub fn get_blob_bytes_batch(
|
||||
&self,
|
||||
sha256_hexes: &[String],
|
||||
) -> StorageResult<Vec<Option<Vec<u8>>>> {
|
||||
if sha256_hexes.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let keys: Vec<String> = sha256_hexes
|
||||
.iter()
|
||||
.map(|hash| {
|
||||
validate_blob_sha256_hex(hash)?;
|
||||
Ok::<String, StorageError>(repo_bytes_key(hash))
|
||||
})
|
||||
.collect::<Result<_, _>>()?;
|
||||
self.db
|
||||
.multi_get(keys.iter().map(|key| key.as_bytes()))
|
||||
.into_iter()
|
||||
.map(|res| res.map_err(|e| StorageError::RocksDb(e.to_string())))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &PathBuf {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub(crate) fn memory_snapshot(&self, label: impl Into<String>) -> RocksDbMemoryDbSnapshot {
|
||||
memory_db_snapshot_for_column_families(label, self.db.as_ref(), None)
|
||||
}
|
||||
}
|
||||
|
||||
impl RawObjectStore for RocksStore {
|
||||
fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>> {
|
||||
self.get_raw_by_hash_entry(sha256_hex)
|
||||
}
|
||||
|
||||
fn get_raw_entries_batch(
|
||||
&self,
|
||||
sha256_hexes: &[String],
|
||||
) -> StorageResult<Vec<Option<RawByHashEntry>>> {
|
||||
self.get_raw_by_hash_entries_batch(sha256_hexes)
|
||||
}
|
||||
|
||||
fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult<Option<Vec<u8>>> {
|
||||
RocksStore::get_blob_bytes(self, sha256_hex)
|
||||
}
|
||||
|
||||
fn get_blob_bytes_batch(&self, sha256_hexes: &[String]) -> StorageResult<Vec<Option<Vec<u8>>>> {
|
||||
RocksStore::get_blob_bytes_batch(self, sha256_hexes)
|
||||
}
|
||||
}
|
||||
|
||||
impl RawObjectStore for ExternalRawStoreDb {
|
||||
fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>> {
|
||||
let key = raw_by_hash_key(sha256_hex);
|
||||
let Some(bytes) = self
|
||||
.db
|
||||
.get(key.as_bytes())
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let entry =
|
||||
serde_cbor::from_slice::<RawByHashEntry>(&bytes).map_err(|e| StorageError::Codec {
|
||||
entity: "raw_by_hash",
|
||||
detail: e.to_string(),
|
||||
})?;
|
||||
entry.validate_internal()?;
|
||||
Ok(Some(entry))
|
||||
}
|
||||
|
||||
fn get_raw_entries_batch(
|
||||
&self,
|
||||
sha256_hexes: &[String],
|
||||
) -> StorageResult<Vec<Option<RawByHashEntry>>> {
|
||||
if sha256_hexes.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let keys: Vec<String> = sha256_hexes
|
||||
.iter()
|
||||
.map(|hash| raw_by_hash_key(hash))
|
||||
.collect();
|
||||
self.db
|
||||
.multi_get(keys.iter().map(|key| key.as_bytes()))
|
||||
.into_iter()
|
||||
.map(|res| {
|
||||
let maybe = res.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
match maybe {
|
||||
Some(bytes) => {
|
||||
let entry =
|
||||
serde_cbor::from_slice::<RawByHashEntry>(&bytes).map_err(|e| {
|
||||
StorageError::Codec {
|
||||
entity: "raw_by_hash",
|
||||
detail: e.to_string(),
|
||||
}
|
||||
})?;
|
||||
entry.validate_internal()?;
|
||||
Ok(Some(entry))
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult<Option<Vec<u8>>> {
|
||||
let key = raw_blob_key(sha256_hex);
|
||||
self.db
|
||||
.get(key.as_bytes())
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))
|
||||
}
|
||||
|
||||
fn get_blob_bytes_batch(&self, sha256_hexes: &[String]) -> StorageResult<Vec<Option<Vec<u8>>>> {
|
||||
if sha256_hexes.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let keys: Vec<String> = sha256_hexes.iter().map(|hash| raw_blob_key(hash)).collect();
|
||||
self.db
|
||||
.multi_get(keys.iter().map(|key| key.as_bytes()))
|
||||
.into_iter()
|
||||
.map(|res| res.map_err(|e| StorageError::RocksDb(e.to_string())))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{ExternalRawStoreDb, ExternalRepoBytesDb, RawObjectStore};
|
||||
use crate::repository::storage::{RawByHashEntry, RocksStore, StorageError, StorageResult};
|
||||
use std::collections::HashMap;
|
||||
|
||||
fn sha256_hex(bytes: &[u8]) -> String {
|
||||
use sha2::{Digest, Sha256};
|
||||
hex::encode(Sha256::digest(bytes))
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct MockRawStore {
|
||||
entries: HashMap<String, RawByHashEntry>,
|
||||
}
|
||||
|
||||
impl RawObjectStore for MockRawStore {
|
||||
fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>> {
|
||||
Ok(self.entries.get(sha256_hex).cloned())
|
||||
}
|
||||
|
||||
fn get_raw_entries_batch(
|
||||
&self,
|
||||
sha256_hexes: &[String],
|
||||
) -> StorageResult<Vec<Option<RawByHashEntry>>> {
|
||||
Ok(sha256_hexes
|
||||
.iter()
|
||||
.map(|hash| self.entries.get(hash).cloned())
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rocks_store_raw_object_store_reads_single_and_batch_entries() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
|
||||
let a = b"object-a".to_vec();
|
||||
let b = b"object-b".to_vec();
|
||||
let a_hash = sha256_hex(&a);
|
||||
let b_hash = sha256_hex(&b);
|
||||
|
||||
store
|
||||
.put_raw_by_hash_entry(&RawByHashEntry::from_bytes(a_hash.clone(), a.clone()))
|
||||
.expect("put a");
|
||||
store
|
||||
.put_raw_by_hash_entry(&RawByHashEntry::from_bytes(b_hash.clone(), b.clone()))
|
||||
.expect("put b");
|
||||
|
||||
let single = store
|
||||
.get_raw_entry(&a_hash)
|
||||
.expect("get single")
|
||||
.expect("present");
|
||||
assert_eq!(single.bytes, a);
|
||||
|
||||
let batch = store
|
||||
.get_raw_entries_batch(&[a_hash.clone(), "00".repeat(32), b_hash.clone()])
|
||||
.expect("get batch");
|
||||
assert_eq!(batch.len(), 3);
|
||||
assert_eq!(
|
||||
batch[0].as_ref().map(|entry| entry.bytes.as_slice()),
|
||||
Some(a.as_slice())
|
||||
);
|
||||
assert!(batch[1].is_none());
|
||||
assert_eq!(
|
||||
batch[2].as_ref().map(|entry| entry.bytes.as_slice()),
|
||||
Some(b.as_slice())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_raw_store_db_roundtrips_entries() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store =
|
||||
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||
|
||||
let mut entry = RawByHashEntry::from_bytes(sha256_hex(b"blob"), b"blob".to_vec());
|
||||
entry
|
||||
.origin_uris
|
||||
.push("rsync://example.test/repo/a.cer".to_string());
|
||||
entry.object_type = Some("cer".to_string());
|
||||
raw_store.put_raw_entry(&entry).expect("put raw entry");
|
||||
|
||||
let got = raw_store
|
||||
.get_raw_entry(&entry.sha256_hex)
|
||||
.expect("read raw entry")
|
||||
.expect("entry exists");
|
||||
assert_eq!(got, entry);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_raw_store_db_batch_writes_and_reads() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store =
|
||||
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||
|
||||
let a = RawByHashEntry::from_bytes(sha256_hex(b"a"), b"a".to_vec());
|
||||
let b = RawByHashEntry::from_bytes(sha256_hex(b"b"), b"b".to_vec());
|
||||
raw_store
|
||||
.put_raw_entries_batch(&[a.clone(), b.clone()])
|
||||
.expect("batch put");
|
||||
|
||||
let batch = raw_store
|
||||
.get_raw_entries_batch(&[a.sha256_hex.clone(), b.sha256_hex.clone()])
|
||||
.expect("batch get");
|
||||
assert_eq!(batch.len(), 2);
|
||||
assert_eq!(batch[0], Some(a));
|
||||
assert_eq!(batch[1], Some(b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_object_store_default_blob_helpers_return_bytes_only() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store = ExternalRawStoreDb::open(td.path().join("nested/raw-store.db"))
|
||||
.expect("open raw store");
|
||||
|
||||
let mut entry = RawByHashEntry::from_bytes(sha256_hex(b"blob"), b"blob".to_vec());
|
||||
entry
|
||||
.origin_uris
|
||||
.push("rsync://example.test/repo/blob.roa".to_string());
|
||||
raw_store.put_raw_entry(&entry).expect("put raw entry");
|
||||
|
||||
let single = raw_store
|
||||
.get_blob_bytes(&entry.sha256_hex)
|
||||
.expect("get blob bytes")
|
||||
.expect("entry exists");
|
||||
assert_eq!(single, b"blob".to_vec());
|
||||
|
||||
let batch = raw_store
|
||||
.get_blob_bytes_batch(&[entry.sha256_hex.clone(), "00".repeat(32)])
|
||||
.expect("get blob bytes batch");
|
||||
assert_eq!(batch, vec![Some(b"blob".to_vec()), None]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_object_store_default_blob_helpers_work_for_custom_store() {
|
||||
let mut store = MockRawStore::default();
|
||||
let a = RawByHashEntry::from_bytes(sha256_hex(b"a"), b"a".to_vec());
|
||||
let b = RawByHashEntry::from_bytes(sha256_hex(b"b"), b"b".to_vec());
|
||||
store.entries.insert(a.sha256_hex.clone(), a.clone());
|
||||
store.entries.insert(b.sha256_hex.clone(), b.clone());
|
||||
|
||||
let single = store
|
||||
.get_blob_bytes(&a.sha256_hex)
|
||||
.expect("single blob bytes")
|
||||
.expect("present");
|
||||
assert_eq!(single, b"a".to_vec());
|
||||
|
||||
let batch = store
|
||||
.get_blob_bytes_batch(&[a.sha256_hex.clone(), "00".repeat(32), b.sha256_hex.clone()])
|
||||
.expect("batch blob bytes");
|
||||
assert_eq!(batch, vec![Some(b"a".to_vec()), None, Some(b"b".to_vec())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rocks_store_blob_helpers_use_external_raw_store_fast_path() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open_with_external_raw_store(
|
||||
&td.path().join("db"),
|
||||
&td.path().join("raw-store.db"),
|
||||
)
|
||||
.expect("open store with external raw store");
|
||||
|
||||
let entry = RawByHashEntry::from_bytes(sha256_hex(b"blob-fast"), b"blob-fast".to_vec());
|
||||
store.put_raw_by_hash_entry(&entry).expect("put");
|
||||
|
||||
let single = store
|
||||
.get_blob_bytes(&entry.sha256_hex)
|
||||
.expect("single blob bytes")
|
||||
.expect("present");
|
||||
assert_eq!(single, b"blob-fast".to_vec());
|
||||
|
||||
let batch = store
|
||||
.get_blob_bytes_batch(&[entry.sha256_hex.clone(), "00".repeat(32)])
|
||||
.expect("batch blob bytes");
|
||||
assert_eq!(batch, vec![Some(b"blob-fast".to_vec()), None]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_raw_store_db_delete_removes_entry() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store =
|
||||
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||
|
||||
let entry = RawByHashEntry::from_bytes(sha256_hex(b"gone"), b"gone".to_vec());
|
||||
raw_store.put_raw_entry(&entry).expect("put");
|
||||
assert!(
|
||||
raw_store
|
||||
.get_raw_entry(&entry.sha256_hex)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
|
||||
raw_store
|
||||
.delete_raw_entry(&entry.sha256_hex)
|
||||
.expect("delete entry");
|
||||
assert!(
|
||||
raw_store
|
||||
.get_raw_entry(&entry.sha256_hex)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_blob_bytes_batch_round_trips_without_raw_entry() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store =
|
||||
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||
|
||||
let a = (sha256_hex(b"blob-a"), b"blob-a".to_vec());
|
||||
let b = (sha256_hex(b"blob-b"), b"blob-b".to_vec());
|
||||
raw_store
|
||||
.put_blob_bytes_batch(&[a.clone(), b.clone()])
|
||||
.expect("put blobs");
|
||||
|
||||
assert_eq!(
|
||||
raw_store.get_blob_bytes(&a.0).expect("get blob a"),
|
||||
Some(a.1.clone())
|
||||
);
|
||||
assert_eq!(
|
||||
raw_store.get_blob_bytes(&b.0).expect("get blob b"),
|
||||
Some(b.1.clone())
|
||||
);
|
||||
assert!(raw_store.get_raw_entry(&a.0).expect("get raw a").is_none());
|
||||
assert!(raw_store.get_raw_entry(&b.0).expect("get raw b").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_blob_bytes_batch_rejects_invalid_inputs() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store =
|
||||
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||
|
||||
let err = raw_store
|
||||
.put_blob_bytes_batch(&[("zz".repeat(32), b"blob".to_vec())])
|
||||
.expect_err("invalid hash should fail");
|
||||
assert!(matches!(err, StorageError::InvalidData { .. }));
|
||||
|
||||
let err = raw_store
|
||||
.put_blob_bytes_batch(&[(sha256_hex(b"blob"), Vec::new())])
|
||||
.expect_err("empty bytes should fail");
|
||||
assert!(matches!(err, StorageError::InvalidData { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_raw_store_db_rejects_invalid_entry_on_put() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store =
|
||||
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||
|
||||
let bad = RawByHashEntry {
|
||||
sha256_hex: "11".repeat(32),
|
||||
bytes: b"blob".to_vec(),
|
||||
origin_uris: Vec::new(),
|
||||
object_type: None,
|
||||
encoding: None,
|
||||
};
|
||||
let err = raw_store
|
||||
.put_raw_entry(&bad)
|
||||
.expect_err("invalid hash should fail");
|
||||
assert!(matches!(err, StorageError::InvalidData { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_raw_store_db_reports_codec_error_for_corrupt_value() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store =
|
||||
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||
raw_store
|
||||
.db
|
||||
.put(b"rawbyhash:deadbeef", b"not-cbor")
|
||||
.expect("inject corrupt bytes");
|
||||
|
||||
let err = raw_store
|
||||
.get_raw_entry("deadbeef")
|
||||
.expect_err("corrupt value should fail");
|
||||
assert!(matches!(
|
||||
err,
|
||||
StorageError::Codec {
|
||||
entity: "raw_by_hash",
|
||||
..
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_raw_store_db_batch_returns_empty_for_empty_request() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store =
|
||||
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||
let entries = raw_store
|
||||
.get_raw_entries_batch(&[])
|
||||
.expect("empty batch succeeds");
|
||||
assert!(entries.is_empty());
|
||||
raw_store
|
||||
.put_raw_entries_batch(&[])
|
||||
.expect("empty put succeeds");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_repo_bytes_db_roundtrips_blob_bytes_without_raw_entry() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let repo_bytes =
|
||||
ExternalRepoBytesDb::open(td.path().join("repo-bytes.db")).expect("open repo bytes");
|
||||
let bytes = b"repo-bytes-object".to_vec();
|
||||
let hash = sha256_hex(&bytes);
|
||||
|
||||
repo_bytes
|
||||
.put_blob_bytes_batch(&[(hash.clone(), bytes.clone())])
|
||||
.expect("put repo bytes");
|
||||
|
||||
assert_eq!(
|
||||
repo_bytes.get_blob_bytes(&hash).expect("get repo bytes"),
|
||||
Some(bytes.clone())
|
||||
);
|
||||
assert_eq!(
|
||||
repo_bytes
|
||||
.get_blob_bytes_batch(&[hash, "00".repeat(32)])
|
||||
.expect("get repo bytes batch"),
|
||||
vec![Some(bytes), None]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_repo_bytes_db_rejects_invalid_inputs() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let repo_bytes =
|
||||
ExternalRepoBytesDb::open(td.path().join("repo-bytes.db")).expect("open repo bytes");
|
||||
|
||||
assert!(
|
||||
repo_bytes
|
||||
.put_blob_bytes_batch(&[("not-a-valid-hash".to_string(), b"blob".to_vec())])
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
repo_bytes
|
||||
.put_blob_bytes_batch(&[(sha256_hex(b"blob"), Vec::new())])
|
||||
.is_err()
|
||||
);
|
||||
assert!(repo_bytes.get_blob_bytes("not-a-valid-hash").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_repo_bytes_db_secondary_catches_up_with_live_primary() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let primary_path = td.path().join("repo-bytes.db");
|
||||
let secondary_path = td.path().join("repo-bytes.secondary");
|
||||
|
||||
let primary = ExternalRepoBytesDb::open(&primary_path).expect("open primary");
|
||||
let bytes_a = b"repo-bytes-a".to_vec();
|
||||
let hash_a = sha256_hex(&bytes_a);
|
||||
primary
|
||||
.put_blob_bytes_batch(&[(hash_a.clone(), bytes_a.clone())])
|
||||
.expect("put a");
|
||||
|
||||
let secondary = ExternalRepoBytesDb::open_as_secondary(&primary_path, &secondary_path)
|
||||
.expect("open secondary");
|
||||
assert!(secondary.secondary);
|
||||
|
||||
// A secondary open does not necessarily see pre-existing data until it
|
||||
// catches up with the primary's manifest.
|
||||
secondary
|
||||
.try_catch_up_with_primary()
|
||||
.expect("initial catch up");
|
||||
assert_eq!(
|
||||
secondary
|
||||
.get_blob_bytes(&hash_a)
|
||||
.expect("get a via secondary"),
|
||||
Some(bytes_a)
|
||||
);
|
||||
|
||||
// Bytes written by the primary *after* the secondary open become
|
||||
// visible after another catch-up (this is the live-soak scenario).
|
||||
let bytes_b = b"repo-bytes-b".to_vec();
|
||||
let hash_b = sha256_hex(&bytes_b);
|
||||
primary
|
||||
.put_blob_bytes_batch(&[(hash_b.clone(), bytes_b.clone())])
|
||||
.expect("put b after secondary open");
|
||||
secondary
|
||||
.try_catch_up_with_primary()
|
||||
.expect("second catch up");
|
||||
assert_eq!(
|
||||
secondary
|
||||
.get_blob_bytes(&hash_b)
|
||||
.expect("get b via secondary"),
|
||||
Some(bytes_b)
|
||||
);
|
||||
}
|
||||
}
|
||||
289
src/repository/current_repo_index.rs
Normal file
289
src/repository/current_repo_index.rs
Normal file
@@ -0,0 +1,289 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
use crate::repository::storage::{RepositoryViewEntry, RepositoryViewState};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct CurrentRepoEntry {
|
||||
pub current_hash: [u8; 32],
|
||||
pub current_hash_hex: String,
|
||||
pub repository_source: String,
|
||||
pub object_type: Option<String>,
|
||||
pub state: RepositoryViewState,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct CurrentRepoObject {
|
||||
pub rsync_uri: String,
|
||||
pub current_hash_hex: String,
|
||||
pub repository_source: String,
|
||||
pub object_type: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
pub struct CurrentRepoIndex {
|
||||
by_uri: HashMap<String, CurrentRepoEntry>,
|
||||
}
|
||||
|
||||
/// Shared handle to the run-wide current repository index. Readers (phase 2
|
||||
/// publication point staging, cache lookups, output snapshots) take
|
||||
/// `.read()` and no longer exclude each other; writers (repo sync transports
|
||||
/// applying repository view entries, run-state reset) take `.write()` and
|
||||
/// stay exclusive.
|
||||
pub type CurrentRepoIndexHandle = Arc<RwLock<CurrentRepoIndex>>;
|
||||
|
||||
impl CurrentRepoIndex {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn shared() -> CurrentRepoIndexHandle {
|
||||
Arc::new(RwLock::new(Self::new()))
|
||||
}
|
||||
|
||||
pub fn get_by_uri(&self, rsync_uri: &str) -> Option<&CurrentRepoEntry> {
|
||||
self.by_uri.get(rsync_uri)
|
||||
}
|
||||
|
||||
pub fn list_scope_uris(&self, repository_source: &str) -> Vec<String> {
|
||||
let mut out = self
|
||||
.by_uri
|
||||
.iter()
|
||||
.filter(|&(_rsync_uri, entry)| entry.repository_source == repository_source)
|
||||
.map(|(rsync_uri, _entry)| rsync_uri.clone())
|
||||
.collect::<Vec<_>>();
|
||||
out.sort();
|
||||
out
|
||||
}
|
||||
|
||||
pub fn active_uri_count(&self) -> usize {
|
||||
self.by_uri.len()
|
||||
}
|
||||
|
||||
pub fn scope_count(&self) -> usize {
|
||||
self.by_uri
|
||||
.values()
|
||||
.map(|entry| entry.repository_source.as_str())
|
||||
.collect::<HashSet<_>>()
|
||||
.len()
|
||||
}
|
||||
|
||||
pub fn snapshot_objects(&self) -> Vec<CurrentRepoObject> {
|
||||
let mut out = self
|
||||
.by_uri
|
||||
.iter()
|
||||
.map(|(rsync_uri, entry)| CurrentRepoObject {
|
||||
rsync_uri: rsync_uri.clone(),
|
||||
current_hash_hex: entry.current_hash_hex.clone(),
|
||||
repository_source: entry.repository_source.clone(),
|
||||
object_type: entry.object_type.clone(),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
out.sort();
|
||||
out
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self.by_uri.clear();
|
||||
}
|
||||
|
||||
pub fn apply_repository_view_entries(
|
||||
&mut self,
|
||||
entries: &[RepositoryViewEntry],
|
||||
) -> Result<(), String> {
|
||||
for entry in entries {
|
||||
self.apply_repository_view_entry(entry)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn apply_repository_view_entry(&mut self, entry: &RepositoryViewEntry) -> Result<(), String> {
|
||||
entry.validate_internal().map_err(|e| e.to_string())?;
|
||||
|
||||
match entry.state {
|
||||
RepositoryViewState::Present | RepositoryViewState::Replaced => {
|
||||
let repository_source = entry.repository_source.clone().ok_or_else(|| {
|
||||
format!(
|
||||
"repository_view entry missing repository_source for current object {}",
|
||||
entry.rsync_uri
|
||||
)
|
||||
})?;
|
||||
let current_hash_hex = entry.current_hash.clone().ok_or_else(|| {
|
||||
format!(
|
||||
"repository_view entry missing current_hash for current object {}",
|
||||
entry.rsync_uri
|
||||
)
|
||||
})?;
|
||||
let current_hash = decode_sha256_hex_32(¤t_hash_hex)?;
|
||||
self.by_uri.insert(
|
||||
entry.rsync_uri.clone(),
|
||||
CurrentRepoEntry {
|
||||
current_hash,
|
||||
current_hash_hex: current_hash_hex.to_ascii_lowercase(),
|
||||
repository_source,
|
||||
object_type: entry.object_type.clone(),
|
||||
state: entry.state,
|
||||
},
|
||||
);
|
||||
}
|
||||
RepositoryViewState::Withdrawn => {
|
||||
self.by_uri.remove(&entry.rsync_uri);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_sha256_hex_32(value: &str) -> Result<[u8; 32], String> {
|
||||
if value.len() != 64 || !value.as_bytes().iter().all(u8::is_ascii_hexdigit) {
|
||||
return Err(format!("invalid sha256 hex: {value}"));
|
||||
}
|
||||
let mut out = [0u8; 32];
|
||||
hex::decode_to_slice(value, &mut out).map_err(|e| format!("hex decode failed: {e}"))?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::CurrentRepoIndex;
|
||||
use crate::repository::storage::{RepositoryViewEntry, RepositoryViewState};
|
||||
|
||||
fn present(source: &str, uri: &str, hash: &str) -> RepositoryViewEntry {
|
||||
RepositoryViewEntry {
|
||||
rsync_uri: uri.to_string(),
|
||||
current_hash: Some(hash.to_string()),
|
||||
repository_source: Some(source.to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_repo_index_tracks_present_and_withdrawn_entries() {
|
||||
let mut index = CurrentRepoIndex::new();
|
||||
let uri = "rsync://example.test/repo/a.roa";
|
||||
let source = "rsync://example.test/repo/";
|
||||
let hash = &"11".repeat(32);
|
||||
index
|
||||
.apply_repository_view_entries(&[present(source, uri, hash)])
|
||||
.expect("apply present");
|
||||
let got = index.get_by_uri(uri).expect("current entry");
|
||||
assert_eq!(got.current_hash_hex, hash.to_string());
|
||||
assert_eq!(index.list_scope_uris(source), vec![uri.to_string()]);
|
||||
|
||||
index
|
||||
.apply_repository_view_entries(&[RepositoryViewEntry {
|
||||
rsync_uri: uri.to_string(),
|
||||
current_hash: Some(hash.to_string()),
|
||||
repository_source: Some(source.to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Withdrawn,
|
||||
}])
|
||||
.expect("apply withdrawn");
|
||||
assert!(index.get_by_uri(uri).is_none());
|
||||
assert!(index.list_scope_uris(source).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_repo_index_moves_uri_between_scopes() {
|
||||
let mut index = CurrentRepoIndex::new();
|
||||
let uri = "rsync://example.test/repo/a.roa";
|
||||
let old_scope = "rsync://example.test/repo/";
|
||||
let new_scope = "https://rrdp.example.test/notification.xml";
|
||||
index
|
||||
.apply_repository_view_entries(&[present(old_scope, uri, &"22".repeat(32))])
|
||||
.expect("apply old scope");
|
||||
index
|
||||
.apply_repository_view_entries(&[present(new_scope, uri, &"33".repeat(32))])
|
||||
.expect("apply new scope");
|
||||
|
||||
assert!(index.list_scope_uris(old_scope).is_empty());
|
||||
assert_eq!(index.list_scope_uris(new_scope), vec![uri.to_string()]);
|
||||
assert_eq!(
|
||||
index.get_by_uri(uri).expect("entry").current_hash_hex,
|
||||
"33".repeat(32)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_repo_index_snapshot_objects_and_counts_are_sorted() {
|
||||
let handle = CurrentRepoIndex::shared();
|
||||
let mut index = handle.write().expect("write-lock index");
|
||||
index
|
||||
.apply_repository_view_entries(&[
|
||||
present(
|
||||
"rsync://example.test/repo-b/",
|
||||
"rsync://example.test/repo-b/b.roa",
|
||||
&"22".repeat(32),
|
||||
),
|
||||
present(
|
||||
"rsync://example.test/repo-a/",
|
||||
"rsync://example.test/repo-a/a.roa",
|
||||
&"11".repeat(32),
|
||||
),
|
||||
])
|
||||
.expect("apply present entries");
|
||||
assert_eq!(index.active_uri_count(), 2);
|
||||
assert_eq!(index.scope_count(), 2);
|
||||
|
||||
let snapshot = index.snapshot_objects();
|
||||
assert_eq!(snapshot.len(), 2);
|
||||
assert_eq!(snapshot[0].rsync_uri, "rsync://example.test/repo-a/a.roa");
|
||||
assert_eq!(snapshot[1].rsync_uri, "rsync://example.test/repo-b/b.roa");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_repo_index_reports_missing_fields_and_invalid_hash() {
|
||||
let mut index = CurrentRepoIndex::new();
|
||||
let err = index
|
||||
.apply_repository_view_entries(&[RepositoryViewEntry {
|
||||
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
|
||||
current_hash: Some("11".repeat(32)),
|
||||
repository_source: None,
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
}])
|
||||
.expect_err("missing source should fail");
|
||||
assert!(err.contains("missing repository_source"), "{err}");
|
||||
|
||||
let err = index
|
||||
.apply_repository_view_entries(&[RepositoryViewEntry {
|
||||
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
|
||||
current_hash: Some("not-a-valid-sha256".to_string()),
|
||||
repository_source: Some("rsync://example.test/repo/".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
}])
|
||||
.expect_err("invalid hash should fail");
|
||||
assert!(err.contains("invalid"), "{err}");
|
||||
|
||||
index
|
||||
.apply_repository_view_entries(&[RepositoryViewEntry {
|
||||
rsync_uri: "rsync://example.test/repo/b.roa".to_string(),
|
||||
current_hash: Some("22".repeat(32)),
|
||||
repository_source: Some("rsync://example.test/repo/".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
}])
|
||||
.expect("valid entry");
|
||||
let got = index.get_by_uri("rsync://example.test/repo/b.roa").unwrap();
|
||||
assert_eq!(got.current_hash_hex, "22".repeat(32));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_repo_index_withdraw_unknown_uri_is_noop() {
|
||||
let mut index = CurrentRepoIndex::new();
|
||||
index
|
||||
.apply_repository_view_entries(&[RepositoryViewEntry {
|
||||
rsync_uri: "rsync://example.test/repo/missing.roa".to_string(),
|
||||
current_hash: None,
|
||||
repository_source: Some("rsync://example.test/repo/".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Withdrawn,
|
||||
}])
|
||||
.expect("withdraw unknown should not fail");
|
||||
assert_eq!(index.active_uri_count(), 0);
|
||||
assert_eq!(index.scope_count(), 0);
|
||||
}
|
||||
}
|
||||
234
src/repository/fetch/current_repository.rs
Normal file
234
src/repository/fetch/current_repository.rs
Normal file
@@ -0,0 +1,234 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::repository::fetch::rsync::{RsyncFetchError, RsyncFetchResult, RsyncFetcher};
|
||||
use crate::repository::fetch::rsync_system::{
|
||||
RsyncScopePolicy, scoped_rsync_failure_dedup_key, scoped_rsync_fetch_uri,
|
||||
};
|
||||
use crate::repository::storage::{RepositoryViewState, RocksStore};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct CurrentRepositoryViewRsyncFetcher {
|
||||
store: Arc<RocksStore>,
|
||||
scope_policy: RsyncScopePolicy,
|
||||
}
|
||||
|
||||
impl CurrentRepositoryViewRsyncFetcher {
|
||||
pub fn new(store: Arc<RocksStore>, scope_policy: RsyncScopePolicy) -> Self {
|
||||
Self {
|
||||
store,
|
||||
scope_policy,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RsyncFetcher for CurrentRepositoryViewRsyncFetcher {
|
||||
fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult<Vec<(String, Vec<u8>)>> {
|
||||
// The source run's live fetcher may have fetched a module root rather
|
||||
// than the publication-point URI that triggered it. Read that same
|
||||
// frozen-view prefix here so the materialized object set stays
|
||||
// equivalent to the publication-point URI that triggered the fetch.
|
||||
let base = scoped_rsync_fetch_uri(self.scope_policy, rsync_base_uri);
|
||||
let entries = self
|
||||
.store
|
||||
.list_repository_view_entries_with_prefix(&base)
|
||||
.map_err(|error| {
|
||||
RsyncFetchError::Fetch(format!(
|
||||
"list frozen repository view failed for {base}: {error}"
|
||||
))
|
||||
})?;
|
||||
let mut objects = Vec::with_capacity(entries.len());
|
||||
for entry in entries {
|
||||
if !matches!(
|
||||
entry.state,
|
||||
RepositoryViewState::Present | RepositoryViewState::Replaced
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
let bytes = self
|
||||
.store
|
||||
.load_current_object_bytes_by_uri(&entry.rsync_uri)
|
||||
.map_err(|error| {
|
||||
RsyncFetchError::Fetch(format!(
|
||||
"load frozen repository object failed for {}: {error}",
|
||||
entry.rsync_uri
|
||||
))
|
||||
})?
|
||||
.ok_or_else(|| {
|
||||
RsyncFetchError::Fetch(format!(
|
||||
"frozen repository object missing for {}",
|
||||
entry.rsync_uri
|
||||
))
|
||||
})?;
|
||||
objects.push((entry.rsync_uri, bytes));
|
||||
}
|
||||
objects.sort_by(|left, right| left.0.cmp(&right.0));
|
||||
if objects.is_empty() {
|
||||
return Err(RsyncFetchError::Fetch(format!(
|
||||
"frozen repository view contains no current objects under {base}"
|
||||
)));
|
||||
}
|
||||
Ok(objects)
|
||||
}
|
||||
|
||||
fn fetch_object(&self, rsync_uri: &str) -> RsyncFetchResult<Vec<u8>> {
|
||||
self.store
|
||||
.load_current_object_bytes_by_uri(rsync_uri)
|
||||
.map_err(|error| {
|
||||
RsyncFetchError::Fetch(format!(
|
||||
"load frozen repository object failed for {rsync_uri}: {error}"
|
||||
))
|
||||
})?
|
||||
.ok_or_else(|| {
|
||||
RsyncFetchError::Fetch(format!("frozen repository object not found: {rsync_uri}"))
|
||||
})
|
||||
}
|
||||
|
||||
fn dedup_key(&self, rsync_base_uri: &str) -> String {
|
||||
scoped_rsync_fetch_uri(self.scope_policy, rsync_base_uri)
|
||||
}
|
||||
|
||||
fn failure_dedup_key(&self, rsync_base_uri: &str) -> Option<String> {
|
||||
scoped_rsync_failure_dedup_key(self.scope_policy, rsync_base_uri)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::repository::storage::{RepositoryViewEntry, RocksStore};
|
||||
|
||||
#[test]
|
||||
fn current_repository_fetcher_returns_sorted_present_objects() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let store = Arc::new(RocksStore::open(dir.path()).expect("open store"));
|
||||
let entries = [
|
||||
("rsync://example.test/repo/b.roa", b"b".as_slice()),
|
||||
("rsync://example.test/repo/a.mft", b"a".as_slice()),
|
||||
];
|
||||
for (uri, bytes) in entries {
|
||||
let hash = crate::repository::sync::store_projection::compute_sha256_hex(bytes);
|
||||
store
|
||||
.put_blob_bytes_batch(&[(hash.clone(), bytes.to_vec())])
|
||||
.expect("put blob");
|
||||
store
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
rsync_uri: uri.to_string(),
|
||||
repository_source: Some("fixture".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
current_hash: Some(hash),
|
||||
})
|
||||
.expect("put view");
|
||||
}
|
||||
let fetcher = CurrentRepositoryViewRsyncFetcher::new(store, RsyncScopePolicy::default());
|
||||
let objects = fetcher
|
||||
.fetch_objects("rsync://example.test/repo/")
|
||||
.expect("fetch objects");
|
||||
assert_eq!(
|
||||
objects
|
||||
.iter()
|
||||
.map(|(uri, _)| uri.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
"rsync://example.test/repo/a.mft",
|
||||
"rsync://example.test/repo/b.roa"
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
fetcher
|
||||
.fetch_object("rsync://example.test/repo/a.mft")
|
||||
.expect("fetch one"),
|
||||
b"a"
|
||||
);
|
||||
assert!(
|
||||
fetcher
|
||||
.fetch_object("rsync://example.test/repo/missing.roa")
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
fetcher.dedup_key("rsync://example.test/repo"),
|
||||
"rsync://example.test/repo/"
|
||||
);
|
||||
assert!(
|
||||
fetcher
|
||||
.fetch_objects("rsync://empty.example/repo/")
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("no current objects")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_repository_fetcher_ignores_withdrawn_and_reports_missing_blob() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let store = Arc::new(RocksStore::open(dir.path()).expect("open store"));
|
||||
store
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
rsync_uri: "rsync://example.test/repo/withdrawn.roa".to_string(),
|
||||
repository_source: Some("fixture".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Withdrawn,
|
||||
current_hash: None,
|
||||
})
|
||||
.expect("put withdrawn view");
|
||||
let missing_hash = "ab".repeat(32);
|
||||
store
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
rsync_uri: "rsync://example.test/repo/missing.roa".to_string(),
|
||||
repository_source: Some("fixture".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
current_hash: Some(missing_hash),
|
||||
})
|
||||
.expect("put missing view");
|
||||
let fetcher = CurrentRepositoryViewRsyncFetcher::new(store, RsyncScopePolicy::default());
|
||||
let error = fetcher
|
||||
.fetch_objects("rsync://example.test/repo/")
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(error.contains("blob bytes missing"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_repository_fetcher_uses_module_scope_and_failure_dedup() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let store = Arc::new(RocksStore::open(dir.path()).expect("open store"));
|
||||
for (uri, bytes) in [
|
||||
("rsync://example.test/repo/ca/a.mft", b"a".as_slice()),
|
||||
("rsync://example.test/repo/other/b.roa", b"b".as_slice()),
|
||||
] {
|
||||
let hash = crate::repository::sync::store_projection::compute_sha256_hex(bytes);
|
||||
store
|
||||
.put_blob_bytes_batch(&[(hash.clone(), bytes.to_vec())])
|
||||
.expect("put blob");
|
||||
store
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
rsync_uri: uri.to_string(),
|
||||
repository_source: Some("fixture".to_string()),
|
||||
object_type: Some("fixture".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
current_hash: Some(hash),
|
||||
})
|
||||
.expect("put view");
|
||||
}
|
||||
|
||||
let module_fetcher = CurrentRepositoryViewRsyncFetcher::new(
|
||||
Arc::clone(&store),
|
||||
RsyncScopePolicy::ModuleRoot,
|
||||
);
|
||||
let objects = module_fetcher
|
||||
.fetch_objects("rsync://example.test/repo/ca/")
|
||||
.expect("fetch module scope");
|
||||
assert_eq!(objects.len(), 2);
|
||||
assert_eq!(
|
||||
module_fetcher.dedup_key("rsync://example.test/repo/ca/"),
|
||||
"rsync://example.test/repo/"
|
||||
);
|
||||
|
||||
let host_fetcher = CurrentRepositoryViewRsyncFetcher::new(store, RsyncScopePolicy::Host);
|
||||
assert_eq!(
|
||||
host_fetcher.failure_dedup_key("rsync://example.test/repo/ca/"),
|
||||
Some("rsync://example.test/".to_string())
|
||||
);
|
||||
}
|
||||
}
|
||||
922
src/repository/fetch/http.rs
Normal file
922
src/repository/fetch/http.rs
Normal file
@@ -0,0 +1,922 @@
|
||||
use std::cell::RefCell;
|
||||
use std::io::Write;
|
||||
use std::time::Duration;
|
||||
|
||||
use reqwest::blocking::Client;
|
||||
use reqwest::header::{HeaderMap, LOCATION};
|
||||
use url::Url;
|
||||
|
||||
use crate::repository::sync::rrdp::{
|
||||
Fetcher, RrdpError, RrdpFetchError, RrdpOrigin, RrdpResourceKind,
|
||||
};
|
||||
|
||||
const MAX_RRDP_REDIRECTS: usize = 10;
|
||||
|
||||
thread_local! {
|
||||
static HTTP_TIMEOUT_OVERRIDE: RefCell<Option<Duration>> = const { RefCell::new(None) };
|
||||
}
|
||||
|
||||
pub fn with_scoped_http_timeout_override<R>(timeout: Duration, f: impl FnOnce() -> R) -> R {
|
||||
HTTP_TIMEOUT_OVERRIDE.with(|cell| {
|
||||
let previous = cell.replace(Some(timeout));
|
||||
let result = f();
|
||||
let _ = cell.replace(previous);
|
||||
result
|
||||
})
|
||||
}
|
||||
|
||||
/// Default User-Agent sent with every outgoing HTTP request (RRDP
|
||||
/// notification/snapshot/delta and TAL/TA downloads).
|
||||
pub const DEFAULT_HTTP_USER_AGENT: &str = "panda-rpki/0.2";
|
||||
|
||||
/// Environment variable that overrides the HTTP User-Agent. Unset, empty, or
|
||||
/// values containing characters that are illegal in a header value fall back
|
||||
/// to [`DEFAULT_HTTP_USER_AGENT`].
|
||||
pub const HTTP_USER_AGENT_ENV: &str = "RPKI_HTTP_USER_AGENT";
|
||||
|
||||
fn resolve_http_user_agent(env_value: Option<String>) -> String {
|
||||
let Some(value) = env_value else {
|
||||
return DEFAULT_HTTP_USER_AGENT.to_string();
|
||||
};
|
||||
let trimmed = value.trim();
|
||||
let valid = !trimmed.is_empty() && trimmed.bytes().all(|b| (0x20..=0x7e).contains(&b));
|
||||
if valid {
|
||||
trimmed.to_string()
|
||||
} else {
|
||||
DEFAULT_HTTP_USER_AGENT.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct HttpFetcherConfig {
|
||||
/// Connection-establishment timeout for HTTP requests.
|
||||
pub connect_timeout: Duration,
|
||||
/// Short timeout used for connection establishment and small metadata objects.
|
||||
pub timeout: Duration,
|
||||
/// Larger timeout used for RRDP snapshot / delta bodies.
|
||||
pub large_body_timeout: Duration,
|
||||
pub user_agent: String,
|
||||
/// Extra PEM trust anchors for HTTPS transport tests or controlled endpoints.
|
||||
pub extra_root_certificates_pem: Vec<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl Default for HttpFetcherConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
connect_timeout: Duration::from_secs(15),
|
||||
timeout: Duration::from_secs(30),
|
||||
large_body_timeout: Duration::from_secs(180),
|
||||
user_agent: resolve_http_user_agent(std::env::var(HTTP_USER_AGENT_ENV).ok()),
|
||||
extra_root_certificates_pem: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal blocking HTTP(S) fetcher for validation runs.
|
||||
///
|
||||
/// This is used for:
|
||||
/// - downloading TAL / TA certificates (RFC 8630 §2)
|
||||
/// - fetching RRDP notification/snapshot files (RFC 8182 §3.4)
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct BlockingHttpFetcher {
|
||||
short_client: Client,
|
||||
large_body_client: Client,
|
||||
retry_short_client: Client,
|
||||
rrdp_short_client: Client,
|
||||
rrdp_large_body_client: Client,
|
||||
rrdp_retry_short_client: Client,
|
||||
short_timeout: Duration,
|
||||
large_body_timeout: Duration,
|
||||
}
|
||||
|
||||
impl BlockingHttpFetcher {
|
||||
pub fn new(config: HttpFetcherConfig) -> Result<Self, String> {
|
||||
let short_timeout = config.timeout;
|
||||
let large_body_timeout = std::cmp::max(config.large_body_timeout, config.timeout);
|
||||
let connect_timeout = std::cmp::min(config.connect_timeout, config.timeout);
|
||||
let short_client = Self::client_builder(
|
||||
&config,
|
||||
connect_timeout,
|
||||
config.timeout,
|
||||
config.user_agent.clone(),
|
||||
)?
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let large_body_client = Self::client_builder(
|
||||
&config,
|
||||
connect_timeout,
|
||||
large_body_timeout,
|
||||
config.user_agent.clone(),
|
||||
)?
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let retry_short_client = Self::client_builder(
|
||||
&config,
|
||||
Duration::from_secs(1),
|
||||
Duration::from_secs(1),
|
||||
config.user_agent.clone(),
|
||||
)?
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let rrdp_short_client = Self::rrdp_client_builder(
|
||||
&config,
|
||||
connect_timeout,
|
||||
config.timeout,
|
||||
config.user_agent.clone(),
|
||||
)?
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let rrdp_large_body_client = Self::rrdp_client_builder(
|
||||
&config,
|
||||
connect_timeout,
|
||||
large_body_timeout,
|
||||
config.user_agent.clone(),
|
||||
)?
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let rrdp_retry_short_client = Self::rrdp_client_builder(
|
||||
&config,
|
||||
Duration::from_secs(1),
|
||||
Duration::from_secs(1),
|
||||
config.user_agent.clone(),
|
||||
)?
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(Self {
|
||||
short_client,
|
||||
large_body_client,
|
||||
retry_short_client,
|
||||
rrdp_short_client,
|
||||
rrdp_large_body_client,
|
||||
rrdp_retry_short_client,
|
||||
short_timeout,
|
||||
large_body_timeout,
|
||||
})
|
||||
}
|
||||
|
||||
fn client_builder(
|
||||
config: &HttpFetcherConfig,
|
||||
connect_timeout: Duration,
|
||||
timeout: Duration,
|
||||
user_agent: String,
|
||||
) -> Result<reqwest::blocking::ClientBuilder, String> {
|
||||
let mut builder = Client::builder()
|
||||
.connect_timeout(connect_timeout)
|
||||
.timeout(timeout)
|
||||
.user_agent(user_agent);
|
||||
for (idx, pem) in config.extra_root_certificates_pem.iter().enumerate() {
|
||||
let certificate = reqwest::Certificate::from_pem(pem)
|
||||
.map_err(|e| format!("parse HTTP root certificate #{idx} failed: {e}"))?;
|
||||
builder = builder.add_root_certificate(certificate);
|
||||
}
|
||||
Ok(builder)
|
||||
}
|
||||
|
||||
fn rrdp_client_builder(
|
||||
config: &HttpFetcherConfig,
|
||||
connect_timeout: Duration,
|
||||
timeout: Duration,
|
||||
user_agent: String,
|
||||
) -> Result<reqwest::blocking::ClientBuilder, String> {
|
||||
Ok(
|
||||
Self::client_builder(config, connect_timeout, timeout, user_agent)?
|
||||
.redirect(reqwest::redirect::Policy::none()),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn fetch_bytes(&self, uri: &str) -> Result<Vec<u8>, String> {
|
||||
let started = std::time::Instant::now();
|
||||
let (client, timeout_profile, timeout_value) = self.client_for_uri(uri);
|
||||
let resp = client.get(uri).send().map_err(|e| {
|
||||
let msg = format!("http request failed: {e:?}");
|
||||
crate::logging::progress::emit!(
|
||||
"http_fetch_failed",
|
||||
serde_json::json!({
|
||||
"uri": uri,
|
||||
"stage": "request",
|
||||
"timeout_profile": timeout_profile,
|
||||
"request_timeout_ms": timeout_value.as_millis() as u64,
|
||||
"duration_ms": started.elapsed().as_millis() as u64,
|
||||
"error": msg,
|
||||
}),
|
||||
);
|
||||
msg
|
||||
})?;
|
||||
|
||||
let status = resp.status();
|
||||
let headers = resp.headers().clone();
|
||||
if !status.is_success() {
|
||||
let msg = format!(
|
||||
"http status {status}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}",
|
||||
header_value(&headers, "content-type"),
|
||||
header_value(&headers, "content-encoding"),
|
||||
header_value(&headers, "content-length"),
|
||||
header_value(&headers, "transfer-encoding"),
|
||||
);
|
||||
crate::logging::progress::emit!(
|
||||
"http_fetch_failed",
|
||||
serde_json::json!({
|
||||
"uri": uri,
|
||||
"stage": "status",
|
||||
"timeout_profile": timeout_profile,
|
||||
"request_timeout_ms": timeout_value.as_millis() as u64,
|
||||
"duration_ms": started.elapsed().as_millis() as u64,
|
||||
"status": status.as_u16(),
|
||||
"content_type": header_value_opt(&headers, "content-type"),
|
||||
"content_encoding": header_value_opt(&headers, "content-encoding"),
|
||||
"content_length": header_value_opt(&headers, "content-length"),
|
||||
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
|
||||
"error": msg,
|
||||
}),
|
||||
);
|
||||
return Err(msg);
|
||||
}
|
||||
|
||||
match resp.bytes() {
|
||||
Ok(bytes) => {
|
||||
let duration_ms = started.elapsed().as_millis() as u64;
|
||||
if (duration_ms as f64) / 1000.0 >= crate::logging::progress::slow_threshold_secs()
|
||||
{
|
||||
crate::logging::progress::emit!(
|
||||
"http_fetch_slow",
|
||||
serde_json::json!({
|
||||
"uri": uri,
|
||||
"status": status.as_u16(),
|
||||
"timeout_profile": timeout_profile,
|
||||
"request_timeout_ms": timeout_value.as_millis() as u64,
|
||||
"duration_ms": duration_ms,
|
||||
"bytes": bytes.len(),
|
||||
"content_type": header_value_opt(&headers, "content-type"),
|
||||
"content_encoding": header_value_opt(&headers, "content-encoding"),
|
||||
"content_length": header_value_opt(&headers, "content-length"),
|
||||
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
|
||||
}),
|
||||
);
|
||||
}
|
||||
Ok(bytes.to_vec())
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = format!(
|
||||
"http read body failed: {e}; status={}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}",
|
||||
status,
|
||||
header_value(&headers, "content-type"),
|
||||
header_value(&headers, "content-encoding"),
|
||||
header_value(&headers, "content-length"),
|
||||
header_value(&headers, "transfer-encoding"),
|
||||
);
|
||||
crate::logging::progress::emit!(
|
||||
"http_fetch_failed",
|
||||
serde_json::json!({
|
||||
"uri": uri,
|
||||
"stage": "read_body",
|
||||
"timeout_profile": timeout_profile,
|
||||
"request_timeout_ms": timeout_value.as_millis() as u64,
|
||||
"duration_ms": started.elapsed().as_millis() as u64,
|
||||
"status": status.as_u16(),
|
||||
"content_type": header_value_opt(&headers, "content-type"),
|
||||
"content_encoding": header_value_opt(&headers, "content-encoding"),
|
||||
"content_length": header_value_opt(&headers, "content-length"),
|
||||
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
|
||||
"error": msg,
|
||||
}),
|
||||
);
|
||||
Err(msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn client_for_uri(&self, uri: &str) -> (&Client, &'static str, Duration) {
|
||||
if let Some(timeout) = HTTP_TIMEOUT_OVERRIDE.with(|cell| *cell.borrow()) {
|
||||
return (&self.retry_short_client, "retry_short", timeout);
|
||||
}
|
||||
if uses_large_body_timeout(uri) {
|
||||
(
|
||||
&self.large_body_client,
|
||||
"large_body",
|
||||
self.large_body_timeout,
|
||||
)
|
||||
} else {
|
||||
(&self.short_client, "short", self.short_timeout)
|
||||
}
|
||||
}
|
||||
|
||||
fn rrdp_client_for_uri(&self, uri: &str) -> (&Client, &'static str, Duration) {
|
||||
if let Some(timeout) = HTTP_TIMEOUT_OVERRIDE.with(|cell| *cell.borrow()) {
|
||||
return (&self.rrdp_retry_short_client, "retry_short", timeout);
|
||||
}
|
||||
if uses_large_body_timeout(uri) {
|
||||
(
|
||||
&self.rrdp_large_body_client,
|
||||
"large_body",
|
||||
self.large_body_timeout,
|
||||
)
|
||||
} else {
|
||||
(&self.rrdp_short_client, "short", self.short_timeout)
|
||||
}
|
||||
}
|
||||
|
||||
fn rrdp_response(
|
||||
&self,
|
||||
resource: RrdpResourceKind,
|
||||
uri: &str,
|
||||
notification_origin: &RrdpOrigin,
|
||||
) -> Result<(reqwest::blocking::Response, &'static str, Duration), RrdpFetchError> {
|
||||
let mut current = Url::parse(uri).map_err(|e| RrdpError::InvalidRrdpUri {
|
||||
resource,
|
||||
detail: e.to_string(),
|
||||
})?;
|
||||
let current_origin = RrdpOrigin::from_url(¤t, resource)?;
|
||||
if ¤t_origin != notification_origin {
|
||||
return Err(RrdpError::CrossOriginReference {
|
||||
resource,
|
||||
notification_origin: notification_origin.to_string(),
|
||||
resource_origin: current_origin.to_string(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
|
||||
for redirects in 0..=MAX_RRDP_REDIRECTS {
|
||||
let (client, profile, timeout) = self.rrdp_client_for_uri(current.as_str());
|
||||
let response = client
|
||||
.get(current.clone())
|
||||
.send()
|
||||
.map_err(|e| RrdpFetchError::Fetch(format!("http request failed: {e:?}")))?;
|
||||
if !response.status().is_redirection() {
|
||||
return Ok((response, profile, timeout));
|
||||
}
|
||||
if redirects == MAX_RRDP_REDIRECTS {
|
||||
return Err(RrdpError::RedirectLimitExceeded {
|
||||
resource,
|
||||
max: MAX_RRDP_REDIRECTS,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let location = response
|
||||
.headers()
|
||||
.get(LOCATION)
|
||||
.ok_or(RrdpError::RedirectLocationMissing { resource })?
|
||||
.to_str()
|
||||
.map_err(|_| RrdpError::InvalidRedirectLocation {
|
||||
resource,
|
||||
location: "<non-UTF-8 header>".to_string(),
|
||||
})?;
|
||||
let next = current
|
||||
.join(location)
|
||||
.map_err(|_| RrdpError::InvalidRedirectLocation {
|
||||
resource,
|
||||
location: location.to_string(),
|
||||
})?;
|
||||
let next_origin = RrdpOrigin::from_url(&next, resource)?;
|
||||
crate::logging::progress::emit!(
|
||||
"rrdp_redirect_checked",
|
||||
serde_json::json!({
|
||||
"resource": resource.to_string(),
|
||||
"from": current.as_str(),
|
||||
"to": next.as_str(),
|
||||
"notification_origin": notification_origin.to_string(),
|
||||
"same_origin": &next_origin == notification_origin,
|
||||
"rfc": "RFC 9674 §3.2",
|
||||
}),
|
||||
);
|
||||
if &next_origin != notification_origin {
|
||||
crate::logging::progress::emit!(
|
||||
"rrdp_cross_origin_rejected",
|
||||
serde_json::json!({
|
||||
"resource": resource.to_string(),
|
||||
"notification_origin": notification_origin.to_string(),
|
||||
"resource_origin": next_origin.to_string(),
|
||||
"reason": "redirect",
|
||||
"rfc": "RFC 9674 §3.2",
|
||||
}),
|
||||
);
|
||||
return Err(RrdpError::CrossOriginRedirect {
|
||||
resource,
|
||||
notification_origin: notification_origin.to_string(),
|
||||
redirect_origin: next_origin.to_string(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
current = next;
|
||||
}
|
||||
unreachable!("redirect loop either returns or reaches its limit")
|
||||
}
|
||||
|
||||
fn rrdp_fetch_bytes(
|
||||
&self,
|
||||
resource: RrdpResourceKind,
|
||||
uri: &str,
|
||||
notification_origin: &RrdpOrigin,
|
||||
) -> Result<Vec<u8>, RrdpFetchError> {
|
||||
let (response, _profile, _timeout) =
|
||||
self.rrdp_response(resource, uri, notification_origin)?;
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
return Err(RrdpFetchError::Fetch(format!("http status {status}")));
|
||||
}
|
||||
response
|
||||
.bytes()
|
||||
.map(|b| b.to_vec())
|
||||
.map_err(|e| RrdpFetchError::Fetch(format!("http read body failed: {e}")))
|
||||
}
|
||||
|
||||
fn rrdp_fetch_to_writer(
|
||||
&self,
|
||||
resource: RrdpResourceKind,
|
||||
uri: &str,
|
||||
notification_origin: &RrdpOrigin,
|
||||
out: &mut dyn Write,
|
||||
) -> Result<u64, RrdpFetchError> {
|
||||
let (mut response, _profile, _timeout) =
|
||||
self.rrdp_response(resource, uri, notification_origin)?;
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
return Err(RrdpFetchError::Fetch(format!("http status {status}")));
|
||||
}
|
||||
response
|
||||
.copy_to(out)
|
||||
.map_err(|e| RrdpFetchError::Fetch(format!("http stream body failed: {e}")))
|
||||
}
|
||||
}
|
||||
|
||||
impl Fetcher for BlockingHttpFetcher {
|
||||
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
|
||||
self.fetch_bytes(uri)
|
||||
}
|
||||
|
||||
fn fetch_to_writer(&self, uri: &str, out: &mut dyn Write) -> Result<u64, String> {
|
||||
let started = std::time::Instant::now();
|
||||
let (client, timeout_profile, timeout_value) = self.client_for_uri(uri);
|
||||
let resp = client.get(uri).send().map_err(|e| {
|
||||
let msg = format!("http request failed: {e:?}");
|
||||
crate::logging::progress::emit!(
|
||||
"http_fetch_failed",
|
||||
serde_json::json!({
|
||||
"uri": uri,
|
||||
"stage": "request",
|
||||
"timeout_profile": timeout_profile,
|
||||
"request_timeout_ms": timeout_value.as_millis() as u64,
|
||||
"duration_ms": started.elapsed().as_millis() as u64,
|
||||
"error": msg,
|
||||
}),
|
||||
);
|
||||
msg
|
||||
})?;
|
||||
|
||||
let status = resp.status();
|
||||
let headers = resp.headers().clone();
|
||||
if !status.is_success() {
|
||||
let msg = format!(
|
||||
"http status {status}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}",
|
||||
header_value(&headers, "content-type"),
|
||||
header_value(&headers, "content-encoding"),
|
||||
header_value(&headers, "content-length"),
|
||||
header_value(&headers, "transfer-encoding"),
|
||||
);
|
||||
crate::logging::progress::emit!(
|
||||
"http_fetch_failed",
|
||||
serde_json::json!({
|
||||
"uri": uri,
|
||||
"stage": "status",
|
||||
"timeout_profile": timeout_profile,
|
||||
"request_timeout_ms": timeout_value.as_millis() as u64,
|
||||
"duration_ms": started.elapsed().as_millis() as u64,
|
||||
"status": status.as_u16(),
|
||||
"content_type": header_value_opt(&headers, "content-type"),
|
||||
"content_encoding": header_value_opt(&headers, "content-encoding"),
|
||||
"content_length": header_value_opt(&headers, "content-length"),
|
||||
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
|
||||
"error": msg,
|
||||
}),
|
||||
);
|
||||
return Err(msg);
|
||||
}
|
||||
|
||||
let mut resp = resp;
|
||||
match resp.copy_to(out) {
|
||||
Ok(bytes) => {
|
||||
let duration_ms = started.elapsed().as_millis() as u64;
|
||||
if (duration_ms as f64) / 1000.0 >= crate::logging::progress::slow_threshold_secs()
|
||||
{
|
||||
crate::logging::progress::emit!(
|
||||
"http_fetch_slow",
|
||||
serde_json::json!({
|
||||
"uri": uri,
|
||||
"status": status.as_u16(),
|
||||
"timeout_profile": timeout_profile,
|
||||
"request_timeout_ms": timeout_value.as_millis() as u64,
|
||||
"duration_ms": duration_ms,
|
||||
"bytes": bytes,
|
||||
"content_type": header_value_opt(&headers, "content-type"),
|
||||
"content_encoding": header_value_opt(&headers, "content-encoding"),
|
||||
"content_length": header_value_opt(&headers, "content-length"),
|
||||
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
|
||||
}),
|
||||
);
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = format!(
|
||||
"http stream body failed: {e}; status={}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}",
|
||||
status,
|
||||
header_value(&headers, "content-type"),
|
||||
header_value(&headers, "content-encoding"),
|
||||
header_value(&headers, "content-length"),
|
||||
header_value(&headers, "transfer-encoding"),
|
||||
);
|
||||
crate::logging::progress::emit!(
|
||||
"http_fetch_failed",
|
||||
serde_json::json!({
|
||||
"uri": uri,
|
||||
"stage": "stream_body",
|
||||
"timeout_profile": timeout_profile,
|
||||
"request_timeout_ms": timeout_value.as_millis() as u64,
|
||||
"duration_ms": started.elapsed().as_millis() as u64,
|
||||
"status": status.as_u16(),
|
||||
"content_type": header_value_opt(&headers, "content-type"),
|
||||
"content_encoding": header_value_opt(&headers, "content-encoding"),
|
||||
"content_length": header_value_opt(&headers, "content-length"),
|
||||
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
|
||||
"error": msg,
|
||||
}),
|
||||
);
|
||||
Err(msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn fetch_rrdp(
|
||||
&self,
|
||||
resource: RrdpResourceKind,
|
||||
uri: &str,
|
||||
notification_origin: &RrdpOrigin,
|
||||
) -> Result<Vec<u8>, RrdpFetchError> {
|
||||
self.rrdp_fetch_bytes(resource, uri, notification_origin)
|
||||
}
|
||||
|
||||
fn fetch_rrdp_to_writer(
|
||||
&self,
|
||||
resource: RrdpResourceKind,
|
||||
uri: &str,
|
||||
notification_origin: &RrdpOrigin,
|
||||
out: &mut dyn Write,
|
||||
) -> Result<u64, RrdpFetchError> {
|
||||
self.rrdp_fetch_to_writer(resource, uri, notification_origin, out)
|
||||
}
|
||||
}
|
||||
|
||||
fn header_value(headers: &HeaderMap, name: &str) -> String {
|
||||
header_value_opt(headers, name).unwrap_or_else(|| "<none>".to_string())
|
||||
}
|
||||
|
||||
fn header_value_opt(headers: &HeaderMap, name: &str) -> Option<String> {
|
||||
headers
|
||||
.get(name)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|v| v.to_string())
|
||||
}
|
||||
|
||||
fn uses_large_body_timeout(uri: &str) -> bool {
|
||||
uri.starts_with("https://") && uri.ends_with(".xml") && !uri.ends_with("notification.xml")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpListener;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::thread;
|
||||
use std::time::Duration as StdDuration;
|
||||
|
||||
fn spawn_one_shot_http_server(status_line: &'static str, body: &'static [u8]) -> String {
|
||||
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
|
||||
let addr = listener.local_addr().expect("addr");
|
||||
thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().expect("accept");
|
||||
let mut buf = [0u8; 1024];
|
||||
let _ = stream.read(&mut buf);
|
||||
let hdr = format!(
|
||||
"{status_line}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
body.len()
|
||||
);
|
||||
stream.write_all(hdr.as_bytes()).expect("write hdr");
|
||||
stream.write_all(body).expect("write body");
|
||||
});
|
||||
format!("http://{}/", addr)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_bytes_returns_body_on_success() {
|
||||
let url = spawn_one_shot_http_server("HTTP/1.1 200 OK", b"hello");
|
||||
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
||||
timeout: Duration::from_secs(2),
|
||||
..HttpFetcherConfig::default()
|
||||
})
|
||||
.expect("http");
|
||||
let got = http.fetch_bytes(&url).expect("fetch");
|
||||
assert_eq!(got, b"hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_bytes_rejects_non_success_status() {
|
||||
let url = spawn_one_shot_http_server("HTTP/1.1 404 Not Found", b"");
|
||||
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
||||
timeout: Duration::from_secs(2),
|
||||
..HttpFetcherConfig::default()
|
||||
})
|
||||
.expect("http");
|
||||
let err = http.fetch_bytes(&url).unwrap_err();
|
||||
assert!(err.contains("http status"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_bytes_times_out_on_idle_body_read() {
|
||||
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
|
||||
let addr = listener.local_addr().expect("addr");
|
||||
thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().expect("accept");
|
||||
let mut buf = [0u8; 1024];
|
||||
let _ = stream.read(&mut buf);
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nh")
|
||||
.expect("write partial body");
|
||||
std::thread::sleep(StdDuration::from_secs(2));
|
||||
let _ = stream.write_all(b"ello");
|
||||
});
|
||||
let url = format!("http://{}/", addr);
|
||||
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
||||
timeout: Duration::from_secs(1),
|
||||
..HttpFetcherConfig::default()
|
||||
})
|
||||
.expect("http");
|
||||
let err = http.fetch_bytes(&url).unwrap_err();
|
||||
assert!(err.contains("http read body failed"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_to_writer_streams_body_on_success() {
|
||||
let url = spawn_one_shot_http_server("HTTP/1.1 200 OK", b"writer-body");
|
||||
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
||||
timeout: Duration::from_secs(2),
|
||||
..HttpFetcherConfig::default()
|
||||
})
|
||||
.expect("http");
|
||||
let mut out = Vec::new();
|
||||
let bytes = http.fetch_to_writer(&url, &mut out).expect("stream");
|
||||
assert_eq!(bytes, 11);
|
||||
assert_eq!(out, b"writer-body");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_to_writer_rejects_non_success_status() {
|
||||
let url = spawn_one_shot_http_server("HTTP/1.1 500 Internal Server Error", b"boom");
|
||||
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
||||
timeout: Duration::from_secs(2),
|
||||
..HttpFetcherConfig::default()
|
||||
})
|
||||
.expect("http");
|
||||
let mut out = Vec::new();
|
||||
let err = http.fetch_to_writer(&url, &mut out).unwrap_err();
|
||||
assert!(err.contains("http status"), "{err}");
|
||||
assert!(out.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_to_writer_times_out_on_idle_stream_read() {
|
||||
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
|
||||
let addr = listener.local_addr().expect("addr");
|
||||
thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().expect("accept");
|
||||
let mut buf = [0u8; 1024];
|
||||
let _ = stream.read(&mut buf);
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nh")
|
||||
.expect("write partial body");
|
||||
std::thread::sleep(StdDuration::from_secs(2));
|
||||
let _ = stream.write_all(b"ello");
|
||||
});
|
||||
let url = format!("http://{}/", addr);
|
||||
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
||||
timeout: Duration::from_secs(1),
|
||||
..HttpFetcherConfig::default()
|
||||
})
|
||||
.expect("http");
|
||||
let mut out = Vec::new();
|
||||
let err = http.fetch_to_writer(&url, &mut out).unwrap_err();
|
||||
assert!(err.contains("http stream body failed"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uses_large_body_timeout_selects_rrdp_snapshot_and_delta_not_notification() {
|
||||
assert!(!uses_large_body_timeout(
|
||||
"https://rrdp.example.test/notification.xml"
|
||||
));
|
||||
assert!(uses_large_body_timeout(
|
||||
"https://rrdp.example.test/session/123/snapshot.xml"
|
||||
));
|
||||
assert!(uses_large_body_timeout(
|
||||
"https://rrdp.example.test/session/123/delta-42.xml"
|
||||
));
|
||||
assert!(!uses_large_body_timeout(
|
||||
"https://tal.example.test/example.tal"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_for_uri_selects_expected_timeout_profile() {
|
||||
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
||||
timeout: Duration::from_secs(3),
|
||||
large_body_timeout: Duration::from_secs(9),
|
||||
..HttpFetcherConfig::default()
|
||||
})
|
||||
.expect("http");
|
||||
|
||||
let (_, profile_short, timeout_short) =
|
||||
http.client_for_uri("https://example.test/root.tal");
|
||||
assert_eq!(profile_short, "short");
|
||||
assert_eq!(timeout_short, Duration::from_secs(3));
|
||||
|
||||
let (_, profile_large, timeout_large) =
|
||||
http.client_for_uri("https://rrdp.example.test/session/1/snapshot.xml");
|
||||
assert_eq!(profile_large, "large_body");
|
||||
assert_eq!(timeout_large, Duration::from_secs(9));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_http_user_agent_falls_back_to_default() {
|
||||
assert_eq!(resolve_http_user_agent(None), DEFAULT_HTTP_USER_AGENT);
|
||||
assert_eq!(
|
||||
resolve_http_user_agent(Some(String::new())),
|
||||
DEFAULT_HTTP_USER_AGENT
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_http_user_agent(Some(" ".to_string())),
|
||||
DEFAULT_HTTP_USER_AGENT
|
||||
);
|
||||
// Control characters are illegal in header values; fall back instead
|
||||
// of letting reqwest fail to build the client.
|
||||
assert_eq!(
|
||||
resolve_http_user_agent(Some("bad\u{1}ua".to_string())),
|
||||
DEFAULT_HTTP_USER_AGENT
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_http_user_agent(Some("non-ascii-ua-中".to_string())),
|
||||
DEFAULT_HTTP_USER_AGENT
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_http_user_agent_accepts_custom_value() {
|
||||
assert_eq!(
|
||||
resolve_http_user_agent(Some("panda-rpki/9.9 (test)".to_string())),
|
||||
"panda-rpki/9.9 (test)"
|
||||
);
|
||||
// Surrounding whitespace is trimmed.
|
||||
assert_eq!(
|
||||
resolve_http_user_agent(Some(" custom/1.0 ".to_string())),
|
||||
"custom/1.0"
|
||||
);
|
||||
}
|
||||
|
||||
fn spawn_user_agent_capture_server() -> (String, std::sync::mpsc::Receiver<String>) {
|
||||
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
|
||||
let addr = listener.local_addr().expect("addr");
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().expect("accept");
|
||||
let mut buf = [0u8; 4096];
|
||||
let read = stream.read(&mut buf).expect("read request");
|
||||
let request = String::from_utf8_lossy(&buf[..read]).to_string();
|
||||
let body = b"ok";
|
||||
let hdr = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
body.len()
|
||||
);
|
||||
stream.write_all(hdr.as_bytes()).expect("write hdr");
|
||||
stream.write_all(body).expect("write body");
|
||||
let _ = tx.send(request);
|
||||
});
|
||||
(format!("http://{}/", addr), rx)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_bytes_sends_configured_user_agent_header() {
|
||||
let (url, rx) = spawn_user_agent_capture_server();
|
||||
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
||||
timeout: Duration::from_secs(2),
|
||||
user_agent: "panda-rpki/0.2".to_string(),
|
||||
..HttpFetcherConfig::default()
|
||||
})
|
||||
.expect("http");
|
||||
http.fetch_bytes(&url).expect("fetch");
|
||||
let request = rx.recv().expect("request captured");
|
||||
assert!(
|
||||
request
|
||||
.to_ascii_lowercase()
|
||||
.contains("user-agent: panda-rpki/0.2"),
|
||||
"{request}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_bytes_sends_custom_user_agent_header() {
|
||||
let (url, rx) = spawn_user_agent_capture_server();
|
||||
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
||||
timeout: Duration::from_secs(2),
|
||||
user_agent: "custom-marker/7.7".to_string(),
|
||||
..HttpFetcherConfig::default()
|
||||
})
|
||||
.expect("http");
|
||||
http.fetch_bytes(&url).expect("fetch");
|
||||
let request = rx.recv().expect("request captured");
|
||||
assert!(
|
||||
request
|
||||
.to_ascii_lowercase()
|
||||
.contains("user-agent: custom-marker/7.7"),
|
||||
"{request}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rrdp_same_origin_redirect_is_followed_manually() {
|
||||
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
|
||||
let addr = listener.local_addr().expect("addr");
|
||||
thread::spawn(move || {
|
||||
for response in [
|
||||
b"HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
.as_slice(),
|
||||
b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok".as_slice(),
|
||||
] {
|
||||
let (mut stream, _) = listener.accept().expect("accept");
|
||||
let mut buf = [0u8; 1024];
|
||||
let _ = stream.read(&mut buf);
|
||||
stream.write_all(response).expect("response");
|
||||
}
|
||||
});
|
||||
let uri = format!("http://{addr}/start");
|
||||
let origin = RrdpOrigin::parse(&uri, RrdpResourceKind::Notification).expect("origin");
|
||||
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
||||
timeout: Duration::from_secs(2),
|
||||
..HttpFetcherConfig::default()
|
||||
})
|
||||
.expect("http");
|
||||
let body = http
|
||||
.fetch_rrdp(RrdpResourceKind::Notification, &uri, &origin)
|
||||
.expect("same-origin redirect accepted");
|
||||
assert_eq!(body, b"ok");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rrdp_cross_origin_redirect_is_rejected_before_target_request() {
|
||||
let target = TcpListener::bind(("127.0.0.1", 0)).expect("bind target");
|
||||
target.set_nonblocking(true).expect("nonblocking target");
|
||||
let target_addr = target.local_addr().expect("target addr");
|
||||
let target_hits = Arc::new(AtomicUsize::new(0));
|
||||
let target_hits_worker = Arc::clone(&target_hits);
|
||||
let target_thread = thread::spawn(move || {
|
||||
let deadline = std::time::Instant::now() + StdDuration::from_millis(400);
|
||||
while std::time::Instant::now() < deadline {
|
||||
match target.accept() {
|
||||
Ok((mut stream, _)) => {
|
||||
target_hits_worker.fetch_add(1, Ordering::SeqCst);
|
||||
let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n");
|
||||
}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
|
||||
std::thread::sleep(StdDuration::from_millis(5));
|
||||
}
|
||||
Err(e) => panic!("target accept: {e}"),
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let source = TcpListener::bind(("127.0.0.1", 0)).expect("bind source");
|
||||
let source_addr = source.local_addr().expect("source addr");
|
||||
thread::spawn(move || {
|
||||
let (mut stream, _) = source.accept().expect("accept source");
|
||||
let mut buf = [0u8; 1024];
|
||||
let _ = stream.read(&mut buf);
|
||||
let response = format!(
|
||||
"HTTP/1.1 302 Found\r\nLocation: http://{target_addr}/foreign\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
);
|
||||
stream.write_all(response.as_bytes()).expect("redirect");
|
||||
});
|
||||
let uri = format!("http://{source_addr}/start");
|
||||
let origin = RrdpOrigin::parse(&uri, RrdpResourceKind::Notification).expect("origin");
|
||||
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
||||
timeout: Duration::from_secs(2),
|
||||
..HttpFetcherConfig::default()
|
||||
})
|
||||
.expect("http");
|
||||
let err = http
|
||||
.fetch_rrdp(RrdpResourceKind::Notification, &uri, &origin)
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpFetchError::Rrdp(RrdpError::CrossOriginRedirect { .. })
|
||||
));
|
||||
target_thread.join().expect("target thread");
|
||||
assert_eq!(target_hits.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
}
|
||||
4
src/repository/fetch/mod.rs
Normal file
4
src/repository/fetch/mod.rs
Normal file
@@ -0,0 +1,4 @@
|
||||
pub mod current_repository;
|
||||
pub mod http;
|
||||
pub mod rsync;
|
||||
pub mod rsync_system;
|
||||
214
src/repository/fetch/rsync.rs
Normal file
214
src/repository/fetch/rsync.rs
Normal file
@@ -0,0 +1,214 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum RsyncFetchError {
|
||||
#[error("rsync fetch error: {0}")]
|
||||
Fetch(String),
|
||||
}
|
||||
|
||||
pub type RsyncFetchResult<T> = Result<T, RsyncFetchError>;
|
||||
|
||||
pub fn normalize_rsync_base_uri(s: &str) -> String {
|
||||
if s.ends_with('/') {
|
||||
s.to_string()
|
||||
} else {
|
||||
format!("{s}/")
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch repository objects from a publication point.
|
||||
///
|
||||
/// v1: this is intentionally abstract so unit tests can use a mock, and later we can
|
||||
/// back it by calling the system `rsync` binary (RFC 6481 §5; RFC 8182 §3.4.5).
|
||||
pub trait RsyncFetcher: Send + Sync {
|
||||
/// Return a list of objects as `(rsync_uri, bytes)` pairs.
|
||||
fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult<Vec<(String, Vec<u8>)>>;
|
||||
|
||||
/// Fetch one object by exact rsync URI.
|
||||
///
|
||||
/// The default implementation fetches the parent directory and filters the exact
|
||||
/// object. Live fetchers should override this to avoid widening one-object TAL
|
||||
/// bootstrap fetches into whole publication point or module synchronizations.
|
||||
fn fetch_object(&self, rsync_uri: &str) -> RsyncFetchResult<Vec<u8>> {
|
||||
let base = parent_rsync_uri(rsync_uri).map_err(RsyncFetchError::Fetch)?;
|
||||
self.fetch_objects(&base)?
|
||||
.into_iter()
|
||||
.find(|(uri, _)| uri == rsync_uri)
|
||||
.map(|(_, bytes)| bytes)
|
||||
.ok_or_else(|| RsyncFetchError::Fetch(format!("rsync object not found: {rsync_uri}")))
|
||||
}
|
||||
|
||||
/// Stream fetched objects to a visitor without requiring callers to materialize the
|
||||
/// full result vector in memory.
|
||||
fn visit_objects(
|
||||
&self,
|
||||
rsync_base_uri: &str,
|
||||
visitor: &mut dyn FnMut(String, Vec<u8>) -> Result<(), String>,
|
||||
) -> RsyncFetchResult<(usize, u64)> {
|
||||
let objects = self.fetch_objects(rsync_base_uri)?;
|
||||
let mut count = 0usize;
|
||||
let mut bytes_total = 0u64;
|
||||
for (uri, bytes) in objects {
|
||||
bytes_total += bytes.len() as u64;
|
||||
count += 1;
|
||||
visitor(uri, bytes).map_err(RsyncFetchError::Fetch)?;
|
||||
}
|
||||
Ok((count, bytes_total))
|
||||
}
|
||||
|
||||
/// Return the deduplication key used by orchestration layers.
|
||||
///
|
||||
/// By default this is the normalized publication point base URI. Fetchers that
|
||||
/// intentionally widen their fetch scope (for example to a full rsync module)
|
||||
/// should override this so callers can safely deduplicate at the same scope.
|
||||
fn dedup_key(&self, rsync_base_uri: &str) -> String {
|
||||
normalize_rsync_base_uri(rsync_base_uri)
|
||||
}
|
||||
|
||||
/// Return an optional failure-only deduplication key.
|
||||
///
|
||||
/// This key is only used to short-circuit repeated failed rsync fallbacks. It must
|
||||
/// not be used to reuse successful fetch results unless it is identical to
|
||||
/// `dedup_key`, because a successful fetch for one publication point does not imply
|
||||
/// that another publication point under the same host has been fetched.
|
||||
fn failure_dedup_key(&self, _rsync_base_uri: &str) -> Option<String> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn parent_rsync_uri(rsync_uri: &str) -> Result<String, String> {
|
||||
let parsed = url::Url::parse(rsync_uri).map_err(|e| e.to_string())?;
|
||||
if parsed.scheme() != "rsync" {
|
||||
return Err(format!("not an rsync URI: {rsync_uri}"));
|
||||
}
|
||||
let host = parsed
|
||||
.host_str()
|
||||
.ok_or_else(|| format!("missing host in rsync URI: {rsync_uri}"))?;
|
||||
let segments = parsed
|
||||
.path_segments()
|
||||
.ok_or_else(|| format!("missing path in rsync URI: {rsync_uri}"))?
|
||||
.collect::<Vec<_>>();
|
||||
if segments.is_empty() || segments.last().copied().unwrap_or_default().is_empty() {
|
||||
return Err(format!(
|
||||
"rsync URI must reference a file object: {rsync_uri}"
|
||||
));
|
||||
}
|
||||
let parent_segments = &segments[..segments.len() - 1];
|
||||
// Preserve an explicitly supplied port. Dropping it changes the URI
|
||||
// identity and prevents the default one-object implementation from
|
||||
// matching objects returned by a fetcher (notably for test/private
|
||||
// publication points that specify :873 explicitly).
|
||||
let authority = match parsed.port() {
|
||||
Some(port) => format!("{host}:{port}"),
|
||||
None => host.to_string(),
|
||||
};
|
||||
let mut parent = format!("rsync://{authority}/");
|
||||
if !parent_segments.is_empty() {
|
||||
parent.push_str(&parent_segments.join("/"));
|
||||
parent.push('/');
|
||||
}
|
||||
Ok(parent)
|
||||
}
|
||||
|
||||
/// A simple "rsync" implementation backed by a local directory.
|
||||
///
|
||||
/// This is primarily meant for offline tests and fixtures. The key generation mimics rsync URIs:
|
||||
/// `rsync_base_uri` + relative path (with `/` separators).
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct LocalDirRsyncFetcher {
|
||||
pub root_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl LocalDirRsyncFetcher {
|
||||
pub fn new(root_dir: impl Into<PathBuf>) -> Self {
|
||||
Self {
|
||||
root_dir: root_dir.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RsyncFetcher for LocalDirRsyncFetcher {
|
||||
fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult<Vec<(String, Vec<u8>)>> {
|
||||
let base = normalize_rsync_base_uri(rsync_base_uri);
|
||||
let mut out = Vec::new();
|
||||
walk_dir_collect(&self.root_dir, &self.root_dir, &base, &mut out)
|
||||
.map_err(RsyncFetchError::Fetch)?;
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
fn walk_dir_collect(
|
||||
root: &Path,
|
||||
current: &Path,
|
||||
rsync_base_uri: &str,
|
||||
out: &mut Vec<(String, Vec<u8>)>,
|
||||
) -> Result<(), String> {
|
||||
let rd = std::fs::read_dir(current).map_err(|e| e.to_string())?;
|
||||
for entry in rd {
|
||||
let entry = entry.map_err(|e| e.to_string())?;
|
||||
let path = entry.path();
|
||||
let meta = entry.metadata().map_err(|e| e.to_string())?;
|
||||
if meta.is_dir() {
|
||||
walk_dir_collect(root, &path, rsync_base_uri, out)?;
|
||||
continue;
|
||||
}
|
||||
if !meta.is_file() {
|
||||
continue;
|
||||
}
|
||||
let rel = path
|
||||
.strip_prefix(root)
|
||||
.map_err(|e| e.to_string())?
|
||||
.to_string_lossy()
|
||||
.replace('\\', "/");
|
||||
let uri = format!("{rsync_base_uri}{rel}");
|
||||
let bytes = std::fs::read(&path).map_err(|e| e.to_string())?;
|
||||
out.push((uri, bytes));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn local_dir_rsync_fetcher_collects_files_and_normalizes_base_uri() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
std::fs::create_dir_all(tmp.path().join("nested")).expect("mkdir");
|
||||
std::fs::write(tmp.path().join("a.mft"), b"a").expect("write");
|
||||
std::fs::write(tmp.path().join("nested").join("b.roa"), b"b").expect("write");
|
||||
|
||||
let f = LocalDirRsyncFetcher::new(tmp.path());
|
||||
let mut objects = f
|
||||
.fetch_objects("rsync://example.net/repo")
|
||||
.expect("fetch_objects");
|
||||
objects.sort_by(|(a, _), (b, _)| a.cmp(b));
|
||||
|
||||
assert_eq!(objects.len(), 2);
|
||||
assert_eq!(objects[0].0, "rsync://example.net/repo/a.mft");
|
||||
assert_eq!(objects[0].1, b"a");
|
||||
assert_eq!(objects[1].0, "rsync://example.net/repo/nested/b.roa");
|
||||
assert_eq!(objects[1].1, b"b");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_dir_rsync_fetcher_reports_read_dir_errors() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let missing = tmp.path().join("missing");
|
||||
let f = LocalDirRsyncFetcher::new(missing);
|
||||
let err = f.fetch_objects("rsync://example.net/repo").unwrap_err();
|
||||
match err {
|
||||
RsyncFetchError::Fetch(msg) => assert!(!msg.is_empty()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_dedup_key_is_normalized_base_uri() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let fetcher = LocalDirRsyncFetcher::new(tmp.path());
|
||||
assert_eq!(
|
||||
fetcher.dedup_key("rsync://example.net/repo"),
|
||||
"rsync://example.net/repo/"
|
||||
);
|
||||
}
|
||||
}
|
||||
927
src/repository/fetch/rsync_system.rs
Normal file
927
src/repository/fetch/rsync_system.rs
Normal file
@@ -0,0 +1,927 @@
|
||||
use std::cell::RefCell;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::process::Stdio;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
|
||||
use sha2::Digest;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::repository::fetch::rsync::{
|
||||
RsyncFetchError, RsyncFetchResult, RsyncFetcher, normalize_rsync_base_uri,
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
#[derive(Default)]
|
||||
pub enum RsyncScopePolicy {
|
||||
Host,
|
||||
PublicationPoint,
|
||||
#[default]
|
||||
ModuleRoot,
|
||||
}
|
||||
|
||||
impl RsyncScopePolicy {
|
||||
pub fn parse_cli_value(value: &str) -> Result<Self, String> {
|
||||
match value {
|
||||
"host" => Ok(Self::Host),
|
||||
"publication-point" => Ok(Self::PublicationPoint),
|
||||
"module-root" => Ok(Self::ModuleRoot),
|
||||
_ => Err(format!(
|
||||
"invalid --rsync-scope: {value}; expected host, publication-point, or module-root"
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SystemRsyncConfig {
|
||||
pub rsync_bin: PathBuf,
|
||||
pub connect_timeout: Duration,
|
||||
pub timeout: Duration,
|
||||
pub extra_args: Vec<String>,
|
||||
/// Optional root directory for persistent rsync mirrors.
|
||||
///
|
||||
/// When set, callers may choose to sync into stable subdirectories under this
|
||||
/// root (instead of a temporary directory) to benefit from rsync's incremental
|
||||
/// behavior across runs.
|
||||
///
|
||||
/// Note: actual mirror behavior is implemented separately from config wiring.
|
||||
pub mirror_root: Option<PathBuf>,
|
||||
pub scope_policy: RsyncScopePolicy,
|
||||
}
|
||||
|
||||
impl Default for SystemRsyncConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
rsync_bin: PathBuf::from("rsync"),
|
||||
connect_timeout: Duration::from_secs(15),
|
||||
timeout: Duration::from_secs(30),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: None,
|
||||
scope_policy: RsyncScopePolicy::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A `RsyncFetcher` implementation backed by the system `rsync` binary.
|
||||
///
|
||||
/// This is intended for live synchronization runs. For unit tests and local fixtures,
|
||||
/// prefer `LocalDirRsyncFetcher`.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SystemRsyncFetcher {
|
||||
config: SystemRsyncConfig,
|
||||
}
|
||||
|
||||
thread_local! {
|
||||
static RSYNC_TIMEOUT_OVERRIDE: RefCell<Option<Duration>> = const { RefCell::new(None) };
|
||||
static RSYNC_FAIL_FAST_PROFILE: RefCell<Option<RsyncFailFastProfile>> = const { RefCell::new(None) };
|
||||
}
|
||||
|
||||
pub fn with_scoped_rsync_timeout_override<R>(timeout: Duration, f: impl FnOnce() -> R) -> R {
|
||||
RSYNC_TIMEOUT_OVERRIDE.with(|cell| {
|
||||
let previous = cell.replace(Some(timeout));
|
||||
let result = f();
|
||||
let _ = cell.replace(previous);
|
||||
result
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct RsyncFailFastProfile {
|
||||
pub initial_wall_clock_timeout: Duration,
|
||||
pub max_wall_clock_timeout: Duration,
|
||||
pub max_attempts: usize,
|
||||
}
|
||||
|
||||
pub fn with_scoped_rsync_fail_fast_profile<R>(
|
||||
profile: RsyncFailFastProfile,
|
||||
f: impl FnOnce() -> R,
|
||||
) -> R {
|
||||
RSYNC_FAIL_FAST_PROFILE.with(|cell| {
|
||||
let previous = cell.replace(Some(profile));
|
||||
let result = f();
|
||||
let _ = cell.replace(previous);
|
||||
result
|
||||
})
|
||||
}
|
||||
|
||||
impl SystemRsyncFetcher {
|
||||
pub fn new(config: SystemRsyncConfig) -> Self {
|
||||
Self { config }
|
||||
}
|
||||
|
||||
fn mirror_dst_dir(&self, normalized_rsync_base_uri: &str) -> Result<Option<PathBuf>, String> {
|
||||
let Some(root) = self.config.mirror_root.as_ref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
std::fs::create_dir_all(root)
|
||||
.map_err(|e| format!("create rsync mirror root failed: {}: {e}", root.display()))?;
|
||||
|
||||
let hash = hex::encode(sha2::Sha256::digest(normalized_rsync_base_uri.as_bytes()));
|
||||
let dir = root.join(hash);
|
||||
std::fs::create_dir_all(&dir).map_err(|e| {
|
||||
format!(
|
||||
"create rsync mirror directory failed: {}: {e}",
|
||||
dir.display()
|
||||
)
|
||||
})?;
|
||||
Ok(Some(dir))
|
||||
}
|
||||
|
||||
fn run_rsync(&self, src: &str, dst: &Path) -> Result<(), String> {
|
||||
let fail_fast = RSYNC_FAIL_FAST_PROFILE.with(|cell| *cell.borrow());
|
||||
if let Some(profile) = fail_fast {
|
||||
return self.run_rsync_fail_fast(src, dst, profile);
|
||||
}
|
||||
self.run_rsync_once(src, dst, None, false)
|
||||
}
|
||||
|
||||
fn run_rsync_once(
|
||||
&self,
|
||||
src: &str,
|
||||
dst: &Path,
|
||||
wall_clock_timeout: Option<Duration>,
|
||||
keep_partial: bool,
|
||||
) -> Result<(), String> {
|
||||
// `--timeout` is I/O timeout in seconds (applies to network reads/writes).
|
||||
let timeout =
|
||||
RSYNC_TIMEOUT_OVERRIDE.with(|cell| cell.borrow().unwrap_or(self.config.timeout));
|
||||
let connect_timeout_secs = self.config.connect_timeout.as_secs().max(1).to_string();
|
||||
let timeout_secs = timeout.as_secs().max(1).to_string();
|
||||
let is_remote_rsync = src.starts_with("rsync://");
|
||||
|
||||
let mut cmd = Command::new(&self.config.rsync_bin);
|
||||
cmd.arg("-rt")
|
||||
.arg("--delete")
|
||||
.arg("--timeout")
|
||||
.arg(timeout_secs)
|
||||
.args(&self.config.extra_args);
|
||||
if is_remote_rsync {
|
||||
cmd.arg("--contimeout").arg(connect_timeout_secs);
|
||||
}
|
||||
if keep_partial {
|
||||
cmd.arg("--partial");
|
||||
}
|
||||
cmd.arg(src)
|
||||
.arg(dst)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
|
||||
let mut child = cmd
|
||||
.spawn()
|
||||
.map_err(|e| format!("rsync spawn failed: {e}"))?;
|
||||
if let Some(limit) = wall_clock_timeout {
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
match child
|
||||
.try_wait()
|
||||
.map_err(|e| format!("rsync wait failed: {e}"))?
|
||||
{
|
||||
Some(_status) => {
|
||||
let out = child
|
||||
.wait_with_output()
|
||||
.map_err(|e| format!("rsync wait_with_output failed: {e}"))?;
|
||||
if out.status.success() {
|
||||
return Ok(());
|
||||
}
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
return Err(format!(
|
||||
"rsync failed: status={} stdout={} stderr={}",
|
||||
out.status,
|
||||
stdout.trim(),
|
||||
stderr.trim()
|
||||
));
|
||||
}
|
||||
None => {
|
||||
if started.elapsed() >= limit {
|
||||
let _ = child.kill();
|
||||
let out = child
|
||||
.wait_with_output()
|
||||
.map_err(|e| format!("rsync wait_with_output failed: {e}"))?;
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
return Err(format!(
|
||||
"rsync wall-clock timeout after {}s: stdout={} stderr={}",
|
||||
limit.as_secs(),
|
||||
stdout.trim(),
|
||||
stderr.trim()
|
||||
));
|
||||
}
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let out = child
|
||||
.wait_with_output()
|
||||
.map_err(|e| format!("rsync wait_with_output failed: {e}"))?;
|
||||
if out.status.success() {
|
||||
return Ok(());
|
||||
}
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
Err(format!(
|
||||
"rsync failed: status={} stdout={} stderr={}",
|
||||
out.status,
|
||||
stdout.trim(),
|
||||
stderr.trim()
|
||||
))
|
||||
}
|
||||
|
||||
fn run_rsync_fail_fast(
|
||||
&self,
|
||||
src: &str,
|
||||
dst: &Path,
|
||||
profile: RsyncFailFastProfile,
|
||||
) -> Result<(), String> {
|
||||
let mut attempt = 0usize;
|
||||
let mut timeout = profile.initial_wall_clock_timeout;
|
||||
let mut previous_progress = (0usize, 0u64);
|
||||
let mut zero_progress_attempts = 0usize;
|
||||
let max_timeout = std::cmp::max(
|
||||
profile.max_wall_clock_timeout,
|
||||
profile.initial_wall_clock_timeout,
|
||||
);
|
||||
|
||||
loop {
|
||||
attempt += 1;
|
||||
match self.run_rsync_once(src, dst, Some(timeout), true) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(err) => {
|
||||
if is_hard_fail_rsync_error(&err) {
|
||||
return Err(format!(
|
||||
"rsync fail-fast hard-fail on attempt {}: {}",
|
||||
attempt, err
|
||||
));
|
||||
}
|
||||
if !err.contains("wall-clock timeout") {
|
||||
return Err(err);
|
||||
}
|
||||
let progress = dir_progress(dst)
|
||||
.map_err(|e| format!("rsync fail-fast progress stat failed: {e}"))?;
|
||||
if progress == (0, 0) {
|
||||
zero_progress_attempts += 1;
|
||||
if zero_progress_attempts >= 2 || attempt >= profile.max_attempts {
|
||||
return Err(format!(
|
||||
"rsync fail-fast gave up after {} attempts with no progress: {}",
|
||||
attempt, err
|
||||
));
|
||||
}
|
||||
} else if progress == previous_progress {
|
||||
return Err(format!(
|
||||
"rsync fail-fast gave up after {} attempts with no additional progress: {}",
|
||||
attempt, err
|
||||
));
|
||||
} else {
|
||||
previous_progress = progress;
|
||||
}
|
||||
|
||||
if attempt >= profile.max_attempts {
|
||||
return Err(format!(
|
||||
"rsync fail-fast exhausted {} attempts: {}",
|
||||
profile.max_attempts, err
|
||||
));
|
||||
}
|
||||
timeout = std::cmp::min(timeout.saturating_mul(2), max_timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn scope_fetch_uri(&self, rsync_base_uri: &str) -> String {
|
||||
scoped_rsync_fetch_uri(self.config.scope_policy, rsync_base_uri)
|
||||
}
|
||||
}
|
||||
|
||||
impl RsyncFetcher for SystemRsyncFetcher {
|
||||
fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult<Vec<(String, Vec<u8>)>> {
|
||||
let mut out = Vec::new();
|
||||
self.visit_objects(rsync_base_uri, &mut |uri, bytes| {
|
||||
out.push((uri, bytes));
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn fetch_object(&self, rsync_uri: &str) -> RsyncFetchResult<Vec<u8>> {
|
||||
let parsed =
|
||||
url::Url::parse(rsync_uri).map_err(|e| RsyncFetchError::Fetch(e.to_string()))?;
|
||||
if parsed.scheme() != "rsync" {
|
||||
return Err(RsyncFetchError::Fetch(format!(
|
||||
"not an rsync URI: {rsync_uri}"
|
||||
)));
|
||||
}
|
||||
let file_name = parsed
|
||||
.path_segments()
|
||||
.and_then(|mut segments| segments.rfind(|segment| !segment.is_empty()))
|
||||
.ok_or_else(|| {
|
||||
RsyncFetchError::Fetch(format!(
|
||||
"rsync URI must reference a file object: {rsync_uri}"
|
||||
))
|
||||
})?;
|
||||
let tmp = TempDir::new().map_err(RsyncFetchError::Fetch)?;
|
||||
self.run_rsync(rsync_uri, tmp.path())
|
||||
.map_err(RsyncFetchError::Fetch)?;
|
||||
let object_path = tmp.path().join(file_name);
|
||||
std::fs::read(&object_path).map_err(|e| {
|
||||
RsyncFetchError::Fetch(format!(
|
||||
"read fetched rsync object failed: {}: {e}",
|
||||
object_path.display()
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn visit_objects(
|
||||
&self,
|
||||
rsync_base_uri: &str,
|
||||
visitor: &mut dyn FnMut(String, Vec<u8>) -> Result<(), String>,
|
||||
) -> RsyncFetchResult<(usize, u64)> {
|
||||
let base = self.scope_fetch_uri(rsync_base_uri);
|
||||
let mut count = 0usize;
|
||||
let mut bytes_total = 0u64;
|
||||
let mut wrapped = |uri: String, bytes: Vec<u8>| -> Result<(), String> {
|
||||
bytes_total += bytes.len() as u64;
|
||||
count += 1;
|
||||
visitor(uri, bytes)
|
||||
};
|
||||
|
||||
if let Some(dst) = self
|
||||
.mirror_dst_dir(&base)
|
||||
.map_err(|e| RsyncFetchError::Fetch(e.to_string()))?
|
||||
{
|
||||
self.run_rsync(&base, &dst)
|
||||
.map_err(RsyncFetchError::Fetch)?;
|
||||
walk_dir_visit(&dst, &dst, &base, &mut wrapped).map_err(RsyncFetchError::Fetch)?;
|
||||
return Ok((count, bytes_total));
|
||||
}
|
||||
|
||||
let tmp = TempDir::new().map_err(|e| RsyncFetchError::Fetch(e.to_string()))?;
|
||||
self.run_rsync(&base, tmp.path())
|
||||
.map_err(RsyncFetchError::Fetch)?;
|
||||
walk_dir_visit(tmp.path(), tmp.path(), &base, &mut wrapped)
|
||||
.map_err(RsyncFetchError::Fetch)?;
|
||||
Ok((count, bytes_total))
|
||||
}
|
||||
|
||||
fn dedup_key(&self, rsync_base_uri: &str) -> String {
|
||||
self.scope_fetch_uri(rsync_base_uri)
|
||||
}
|
||||
|
||||
fn failure_dedup_key(&self, rsync_base_uri: &str) -> Option<String> {
|
||||
scoped_rsync_failure_dedup_key(self.config.scope_policy, rsync_base_uri)
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the exact successful-fetch scope used by the live rsync fetcher.
|
||||
///
|
||||
/// The request scope is shared by live and materialized repository paths so their
|
||||
/// request keys and object projection scope remain equivalent.
|
||||
pub fn scoped_rsync_fetch_uri(scope_policy: RsyncScopePolicy, rsync_base_uri: &str) -> String {
|
||||
match scope_policy {
|
||||
RsyncScopePolicy::Host | RsyncScopePolicy::PublicationPoint => {
|
||||
normalize_rsync_base_uri(rsync_base_uri)
|
||||
}
|
||||
RsyncScopePolicy::ModuleRoot => rsync_module_root_uri(rsync_base_uri)
|
||||
.unwrap_or_else(|| normalize_rsync_base_uri(rsync_base_uri)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the live fetcher's failure-deduplication key for the configured
|
||||
/// scope. This is part of the transport request identity, even though host
|
||||
/// scope deliberately does not widen the successful fetch URI.
|
||||
pub fn scoped_rsync_failure_dedup_key(
|
||||
scope_policy: RsyncScopePolicy,
|
||||
rsync_base_uri: &str,
|
||||
) -> Option<String> {
|
||||
match scope_policy {
|
||||
RsyncScopePolicy::Host => rsync_host_scope_uri(rsync_base_uri),
|
||||
RsyncScopePolicy::PublicationPoint | RsyncScopePolicy::ModuleRoot => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn rsync_host_scope_uri(rsync_base_uri: &str) -> Option<String> {
|
||||
let parsed = url::Url::parse(rsync_base_uri).ok()?;
|
||||
if parsed.scheme() != "rsync" {
|
||||
return None;
|
||||
}
|
||||
Some(format!("rsync://{}/", parsed.host_str()?))
|
||||
}
|
||||
|
||||
struct TempDir {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl TempDir {
|
||||
fn new() -> Result<Self, String> {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!("rpki-system-rsync-{}", Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&p).map_err(|e| e.to_string())?;
|
||||
Ok(Self { path: p })
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TempDir {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_dir_all(&self.path);
|
||||
}
|
||||
}
|
||||
|
||||
fn rsync_module_root_uri(s: &str) -> Option<String> {
|
||||
let normalized = normalize_rsync_base_uri(s);
|
||||
let rest = normalized.strip_prefix("rsync://")?;
|
||||
let (authority, path) = rest.split_once('/')?;
|
||||
|
||||
let mut segments: Vec<&str> = path
|
||||
.split('/')
|
||||
.filter(|segment| !segment.is_empty())
|
||||
.collect();
|
||||
if segments.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let module = segments.remove(0);
|
||||
Some(format!("rsync://{authority}/{module}/"))
|
||||
}
|
||||
|
||||
fn dir_progress(root: &Path) -> Result<(usize, u64), String> {
|
||||
if !root.exists() {
|
||||
return Ok((0, 0));
|
||||
}
|
||||
let mut files = 0usize;
|
||||
let mut bytes = 0u64;
|
||||
let mut stack = vec![root.to_path_buf()];
|
||||
while let Some(path) = stack.pop() {
|
||||
let rd = std::fs::read_dir(&path).map_err(|e| e.to_string())?;
|
||||
for entry in rd {
|
||||
let entry = entry.map_err(|e| e.to_string())?;
|
||||
let path = entry.path();
|
||||
let meta = entry.metadata().map_err(|e| e.to_string())?;
|
||||
if meta.is_dir() {
|
||||
stack.push(path);
|
||||
} else if meta.is_file() {
|
||||
files += 1;
|
||||
bytes += meta.len();
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok((files, bytes))
|
||||
}
|
||||
|
||||
fn is_hard_fail_rsync_error(msg: &str) -> bool {
|
||||
let lower = msg.to_ascii_lowercase();
|
||||
lower.contains("no route to host")
|
||||
|| lower.contains("network is unreachable")
|
||||
|| lower.contains("connection refused")
|
||||
|| lower.contains("name or service not known")
|
||||
}
|
||||
|
||||
fn walk_dir_visit(
|
||||
root: &Path,
|
||||
current: &Path,
|
||||
rsync_base_uri: &str,
|
||||
visitor: &mut dyn FnMut(String, Vec<u8>) -> Result<(), String>,
|
||||
) -> Result<(), String> {
|
||||
let rd = std::fs::read_dir(current).map_err(|e| e.to_string())?;
|
||||
for entry in rd {
|
||||
let entry = entry.map_err(|e| e.to_string())?;
|
||||
let path = entry.path();
|
||||
let meta = entry.metadata().map_err(|e| e.to_string())?;
|
||||
if meta.is_dir() {
|
||||
walk_dir_visit(root, &path, rsync_base_uri, visitor)?;
|
||||
continue;
|
||||
}
|
||||
if !meta.is_file() {
|
||||
continue;
|
||||
}
|
||||
let rel = path
|
||||
.strip_prefix(root)
|
||||
.map_err(|e| e.to_string())?
|
||||
.to_string_lossy()
|
||||
.replace('\\', "/");
|
||||
let uri = format!("{rsync_base_uri}{rel}");
|
||||
let bytes = std::fs::read(&path).map_err(|e| e.to_string())?;
|
||||
visitor(uri, bytes)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn normalize_rsync_base_uri_appends_slash_when_missing() {
|
||||
assert_eq!(
|
||||
normalize_rsync_base_uri("rsync://example.net/repo"),
|
||||
"rsync://example.net/repo/".to_string()
|
||||
);
|
||||
assert_eq!(
|
||||
normalize_rsync_base_uri("rsync://example.net/repo/"),
|
||||
"rsync://example.net/repo/".to_string()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn walk_dir_collect_collects_files_and_normalizes_backslashes_in_uri() {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let root = temp.path();
|
||||
std::fs::create_dir_all(root.join("sub")).expect("mkdir");
|
||||
std::fs::write(root.join("sub").join("a.cer"), b"x").expect("write");
|
||||
std::fs::write(root.join("b\\c.mft"), b"y").expect("write backslash file");
|
||||
|
||||
let mut out: Vec<(String, Vec<u8>)> = Vec::new();
|
||||
walk_dir_visit(
|
||||
root,
|
||||
root,
|
||||
"rsync://example.net/repo/",
|
||||
&mut |uri, bytes| {
|
||||
out.push((uri, bytes));
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.expect("walk");
|
||||
out.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
|
||||
assert_eq!(out.len(), 2);
|
||||
assert_eq!(out[0].0, "rsync://example.net/repo/b/c.mft");
|
||||
assert_eq!(out[0].1, b"y");
|
||||
assert_eq!(out[1].0, "rsync://example.net/repo/sub/a.cer");
|
||||
assert_eq!(out[1].1, b"x");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rsync_module_root_uri_returns_host_and_module_only() {
|
||||
assert_eq!(
|
||||
rsync_module_root_uri("rsync://example.net/repo/ta/ca/publication-point/"),
|
||||
Some("rsync://example.net/repo/".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
rsync_module_root_uri("rsync://example.net/repo/ta/"),
|
||||
Some("rsync://example.net/repo/".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
rsync_module_root_uri("rsync://example.net/repo/"),
|
||||
Some("rsync://example.net/repo/".to_string())
|
||||
);
|
||||
assert_eq!(rsync_module_root_uri("https://example.net/repo"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rsync_host_scope_uri_returns_host_only() {
|
||||
assert_eq!(
|
||||
rsync_host_scope_uri("rsync://example.net/repo/ta/ca/publication-point/"),
|
||||
Some("rsync://example.net/".to_string())
|
||||
);
|
||||
assert_eq!(rsync_host_scope_uri("https://example.net/repo"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn system_rsync_dedup_key_uses_module_root_by_default() {
|
||||
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig::default());
|
||||
assert_eq!(
|
||||
fetcher.dedup_key("rsync://example.net/repo/ta/ca/publication-point/"),
|
||||
"rsync://example.net/repo/"
|
||||
);
|
||||
assert_eq!(
|
||||
fetcher.failure_dedup_key("rsync://example.net/repo/ta/ca/publication-point/"),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn system_rsync_host_scope_does_not_widen_success_fetch_scope() {
|
||||
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
scope_policy: RsyncScopePolicy::Host,
|
||||
..SystemRsyncConfig::default()
|
||||
});
|
||||
assert_eq!(
|
||||
fetcher.dedup_key("rsync://example.net/repo/ta/ca/publication-point/"),
|
||||
"rsync://example.net/repo/ta/ca/publication-point/"
|
||||
);
|
||||
assert_eq!(
|
||||
fetcher.scope_fetch_uri("rsync://example.net/repo/ta/ca/publication-point/"),
|
||||
"rsync://example.net/repo/ta/ca/publication-point/"
|
||||
);
|
||||
assert_eq!(
|
||||
fetcher.failure_dedup_key("rsync://example.net/repo/ta/ca/publication-point/"),
|
||||
Some("rsync://example.net/".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn system_rsync_dedup_key_uses_publication_point_scope_when_configured() {
|
||||
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
scope_policy: RsyncScopePolicy::PublicationPoint,
|
||||
..SystemRsyncConfig::default()
|
||||
});
|
||||
assert_eq!(
|
||||
fetcher.dedup_key("rsync://example.net/repo/ta/ca/publication-point/"),
|
||||
"rsync://example.net/repo/ta/ca/publication-point/"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn system_rsync_dedup_key_uses_module_root_when_configured() {
|
||||
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
scope_policy: RsyncScopePolicy::ModuleRoot,
|
||||
..SystemRsyncConfig::default()
|
||||
});
|
||||
assert_eq!(
|
||||
fetcher.dedup_key("rsync://example.net/repo/ta/ca/publication-point/"),
|
||||
"rsync://example.net/repo/"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn system_rsync_fetcher_reports_spawn_and_exit_errors() {
|
||||
let dst = tempfile::tempdir().expect("tempdir");
|
||||
|
||||
// 1) Spawn error.
|
||||
let f = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
rsync_bin: PathBuf::from("/this/does/not/exist/rsync"),
|
||||
connect_timeout: Duration::from_secs(1),
|
||||
timeout: Duration::from_secs(1),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: None,
|
||||
scope_policy: RsyncScopePolicy::default(),
|
||||
});
|
||||
let e = f
|
||||
.run_rsync("rsync://example.net/repo/", dst.path())
|
||||
.expect_err("spawn must fail");
|
||||
assert!(e.contains("rsync spawn failed:"), "{e}");
|
||||
|
||||
// 2) Non-zero exit status.
|
||||
let f = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
rsync_bin: PathBuf::from("false"),
|
||||
connect_timeout: Duration::from_secs(1),
|
||||
timeout: Duration::from_secs(1),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: None,
|
||||
scope_policy: RsyncScopePolicy::default(),
|
||||
});
|
||||
let e = f
|
||||
.run_rsync("rsync://example.net/repo/", dst.path())
|
||||
.expect_err("false must fail");
|
||||
assert!(e.contains("rsync failed:"), "{e}");
|
||||
assert!(e.contains("status="), "{e}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mirror_dst_dir_reports_root_creation_error() {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let root_file = temp.path().join("mirror-root-file");
|
||||
std::fs::write(&root_file, b"not a directory").expect("write root file");
|
||||
|
||||
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
rsync_bin: PathBuf::from("rsync"),
|
||||
connect_timeout: Duration::from_secs(1),
|
||||
timeout: Duration::from_secs(1),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: Some(root_file.clone()),
|
||||
scope_policy: RsyncScopePolicy::default(),
|
||||
});
|
||||
|
||||
let err = fetcher
|
||||
.mirror_dst_dir("rsync://example.net/repo/")
|
||||
.expect_err("file mirror root must fail");
|
||||
assert!(err.contains("create rsync mirror root failed"), "{err}");
|
||||
assert!(err.contains(&root_file.display().to_string()), "{err}");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn mirror_dst_dir_reports_directory_creation_error_inside_root() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let root = temp.path().join("mirror");
|
||||
std::fs::create_dir_all(&root).expect("mkdir root");
|
||||
let mut perms = std::fs::metadata(&root).expect("metadata").permissions();
|
||||
perms.set_mode(0o555);
|
||||
std::fs::set_permissions(&root, perms).expect("chmod root readonly");
|
||||
|
||||
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
rsync_bin: PathBuf::from("rsync"),
|
||||
connect_timeout: Duration::from_secs(1),
|
||||
timeout: Duration::from_secs(1),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: Some(root.clone()),
|
||||
scope_policy: RsyncScopePolicy::default(),
|
||||
});
|
||||
|
||||
let err = fetcher
|
||||
.mirror_dst_dir("rsync://example.net/repo/")
|
||||
.expect_err("readonly mirror root must fail");
|
||||
assert!(
|
||||
err.contains("create rsync mirror directory failed"),
|
||||
"{err}"
|
||||
);
|
||||
|
||||
let mut perms = std::fs::metadata(&root).expect("metadata").permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&root, perms).expect("restore perms");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn walk_dir_collect_ignores_non_file_entries() {
|
||||
use std::os::unix::net::UnixListener;
|
||||
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let root = temp.path();
|
||||
std::fs::write(root.join("a.cer"), b"x").expect("write file");
|
||||
let socket_path = root.join("skip.sock");
|
||||
let _listener = UnixListener::bind(&socket_path).expect("bind socket");
|
||||
|
||||
let mut out: Vec<(String, Vec<u8>)> = Vec::new();
|
||||
walk_dir_visit(
|
||||
root,
|
||||
root,
|
||||
"rsync://example.net/repo/",
|
||||
&mut |uri, bytes| {
|
||||
out.push((uri, bytes));
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.expect("walk");
|
||||
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].0, "rsync://example.net/repo/a.cer");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn rsync_fail_fast_retries_when_progress_is_made() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let script = temp.path().join("fake-rsync.sh");
|
||||
let state = temp.path().join("state.txt");
|
||||
std::fs::write(
|
||||
&script,
|
||||
format!(
|
||||
"#!/usr/bin/env bash\nset -euo pipefail\nSTATE=\"{}\"\nDST=\"${{@: -1}}\"\nCOUNT=0\nif [[ -f \"$STATE\" ]]; then COUNT=$(cat \"$STATE\"); fi\nCOUNT=$((COUNT+1))\necho \"$COUNT\" > \"$STATE\"\nmkdir -p \"$DST\"\nif [[ \"$COUNT\" -eq 1 ]]; then\n echo first > \"$DST/part1\"\n sleep 2\nelse\n echo second > \"$DST/part2\"\nfi\n",
|
||||
state.display()
|
||||
),
|
||||
)
|
||||
.expect("write script");
|
||||
let mut perms = std::fs::metadata(&script).unwrap().permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&script, perms).unwrap();
|
||||
|
||||
let dst = temp.path().join("dst");
|
||||
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
rsync_bin: script,
|
||||
connect_timeout: Duration::from_secs(15),
|
||||
timeout: Duration::from_secs(60),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: None,
|
||||
scope_policy: RsyncScopePolicy::default(),
|
||||
});
|
||||
|
||||
fetcher
|
||||
.run_rsync_fail_fast(
|
||||
"rsync://example.net/repo/",
|
||||
&dst,
|
||||
RsyncFailFastProfile {
|
||||
initial_wall_clock_timeout: Duration::from_secs(1),
|
||||
max_wall_clock_timeout: Duration::from_secs(4),
|
||||
max_attempts: 3,
|
||||
},
|
||||
)
|
||||
.expect("eventual success");
|
||||
|
||||
assert!(dst.join("part1").exists());
|
||||
assert!(dst.join("part2").exists());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn rsync_fail_fast_gives_up_after_two_zero_progress_timeouts() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let script = temp.path().join("fake-rsync.sh");
|
||||
std::fs::write(&script, "#!/usr/bin/env bash\nset -euo pipefail\nsleep 5\n")
|
||||
.expect("write script");
|
||||
let mut perms = std::fs::metadata(&script).unwrap().permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&script, perms).unwrap();
|
||||
|
||||
let dst = temp.path().join("dst");
|
||||
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
rsync_bin: script,
|
||||
connect_timeout: Duration::from_secs(15),
|
||||
timeout: Duration::from_secs(60),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: None,
|
||||
scope_policy: RsyncScopePolicy::default(),
|
||||
});
|
||||
|
||||
let err = fetcher
|
||||
.run_rsync_fail_fast(
|
||||
"rsync://example.net/repo/",
|
||||
&dst,
|
||||
RsyncFailFastProfile {
|
||||
initial_wall_clock_timeout: Duration::from_secs(1),
|
||||
max_wall_clock_timeout: Duration::from_secs(2),
|
||||
max_attempts: 4,
|
||||
},
|
||||
)
|
||||
.expect_err("must fail");
|
||||
assert!(err.contains("no progress"), "{err}");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn rsync_fail_fast_hard_fail_stops_after_first_attempt() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let script = temp.path().join("fake-rsync.sh");
|
||||
let state = temp.path().join("state.txt");
|
||||
std::fs::write(
|
||||
&script,
|
||||
format!(
|
||||
"#!/usr/bin/env bash\nset -euo pipefail\nSTATE=\"{}\"\nCOUNT=0\nif [[ -f \"$STATE\" ]]; then COUNT=$(cat \"$STATE\"); fi\nCOUNT=$((COUNT+1))\necho \"$COUNT\" > \"$STATE\"\necho 'rsync: [Receiver] failed to connect to host (1.2.3.4): Connection refused (111)' >&2\nexit 10\n",
|
||||
state.display()
|
||||
),
|
||||
)
|
||||
.expect("write script");
|
||||
let mut perms = std::fs::metadata(&script).unwrap().permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&script, perms).unwrap();
|
||||
|
||||
let dst = temp.path().join("dst");
|
||||
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
rsync_bin: script,
|
||||
connect_timeout: Duration::from_secs(15),
|
||||
timeout: Duration::from_secs(60),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: None,
|
||||
scope_policy: RsyncScopePolicy::default(),
|
||||
});
|
||||
|
||||
let err = fetcher
|
||||
.run_rsync_fail_fast(
|
||||
"rsync://example.net/repo/",
|
||||
&dst,
|
||||
RsyncFailFastProfile {
|
||||
initial_wall_clock_timeout: Duration::from_secs(10),
|
||||
max_wall_clock_timeout: Duration::from_secs(80),
|
||||
max_attempts: 4,
|
||||
},
|
||||
)
|
||||
.expect_err("must hard fail");
|
||||
assert!(err.contains("hard-fail"), "{err}");
|
||||
let count = std::fs::read_to_string(&state).unwrap();
|
||||
assert_eq!(count.trim(), "1");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn run_rsync_once_passes_contimeout_and_timeout_args() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let script = temp.path().join("capture-rsync.sh");
|
||||
let args_file = temp.path().join("args.txt");
|
||||
std::fs::write(
|
||||
&script,
|
||||
format!(
|
||||
"#!/usr/bin/env bash\nset -euo pipefail\nprintf '%s\\n' \"$@\" > \"{}\"\nDST=\"${{@: -1}}\"\nmkdir -p \"$DST\"\n",
|
||||
args_file.display()
|
||||
),
|
||||
)
|
||||
.expect("write script");
|
||||
let mut perms = std::fs::metadata(&script).unwrap().permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&script, perms).unwrap();
|
||||
|
||||
let dst = temp.path().join("dst");
|
||||
let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig {
|
||||
rsync_bin: script,
|
||||
connect_timeout: Duration::from_secs(15),
|
||||
timeout: Duration::from_secs(30),
|
||||
extra_args: Vec::new(),
|
||||
mirror_root: None,
|
||||
scope_policy: RsyncScopePolicy::default(),
|
||||
});
|
||||
|
||||
fetcher
|
||||
.run_rsync_once("rsync://example.net/repo/", &dst, None, false)
|
||||
.expect("rsync");
|
||||
|
||||
let args = std::fs::read_to_string(&args_file).expect("read args");
|
||||
assert!(args.contains("--contimeout\n15\n"), "{args}");
|
||||
assert!(args.contains("--timeout\n30\n"), "{args}");
|
||||
}
|
||||
}
|
||||
6
src/repository/mod.rs
Normal file
6
src/repository/mod.rs
Normal file
@@ -0,0 +1,6 @@
|
||||
//! Repository transport, protocol state, and object storage.
|
||||
pub mod blob_store;
|
||||
pub mod current_repo_index;
|
||||
pub mod fetch;
|
||||
pub mod storage;
|
||||
pub mod sync;
|
||||
43
src/repository/storage.rs
Normal file
43
src/repository/storage.rs
Normal file
@@ -0,0 +1,43 @@
|
||||
mod config;
|
||||
mod keys;
|
||||
mod memory;
|
||||
mod pack;
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
|
||||
use rocksdb::checkpoint::Checkpoint;
|
||||
use rocksdb::{ColumnFamily, DB, Direction, IteratorMode, Options, WriteBatch};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::repository::blob_store::{ExternalRawStoreDb, ExternalRepoBytesDb, RawObjectStore};
|
||||
|
||||
use config::*;
|
||||
pub use config::{
|
||||
ALL_COLUMN_FAMILY_NAMES, CF_MANIFEST_ANTI_ROLLBACK, CF_RAW_BY_HASH, CF_REPOSITORY_VIEW,
|
||||
CF_RRDP_SOURCE, CF_RRDP_SOURCE_MEMBER, CF_RRDP_URI_OWNER, column_family_descriptors,
|
||||
};
|
||||
use keys::*;
|
||||
pub(crate) use memory::memory_db_snapshot_for_column_families;
|
||||
pub use memory::{
|
||||
RocksDbColumnFamilyMemoryProperties, RocksDbMemoryDbSnapshot, RocksDbMemoryProperties,
|
||||
RocksDbMemorySnapshot, RocksDbMemoryTotals,
|
||||
};
|
||||
use pack::compute_sha256_32;
|
||||
pub use pack::{PackBytes, PackFile, PackTime};
|
||||
|
||||
pub struct RocksStore {
|
||||
db: DB,
|
||||
external_raw_store: Option<ExternalRawStoreDb>,
|
||||
external_repo_bytes: Option<ExternalRepoBytesDb>,
|
||||
}
|
||||
include!("storage/models_core.rs");
|
||||
include!("storage/models_summary.rs");
|
||||
include!("storage/store_lifecycle.rs");
|
||||
include!("storage/store_repository.rs");
|
||||
include!("storage/store_manifest.rs");
|
||||
include!("storage/store_transport_rrdp.rs");
|
||||
include!("storage/verification.rs");
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "storage/tests.rs"]
|
||||
mod tests;
|
||||
43
src/repository/storage/config.rs
Normal file
43
src/repository/storage/config.rs
Normal file
@@ -0,0 +1,43 @@
|
||||
use rocksdb::{ColumnFamilyDescriptor, DBCompressionType, Options};
|
||||
|
||||
pub const CF_REPOSITORY_VIEW: &str = "repository_view";
|
||||
pub const CF_RAW_BY_HASH: &str = "raw_by_hash";
|
||||
pub const CF_RAW_BLOB: &str = "raw_blob";
|
||||
/// Persistent RFC 9286 manifest freshness metadata. This is protocol state,
|
||||
/// not a validation-result cache.
|
||||
pub const CF_MANIFEST_ANTI_ROLLBACK: &str = "manifest_anti_rollback";
|
||||
pub const CF_RRDP_SOURCE: &str = "rrdp_source";
|
||||
pub const CF_RRDP_SOURCE_MEMBER: &str = "rrdp_source_member";
|
||||
pub const CF_RRDP_URI_OWNER: &str = "rrdp_uri_owner";
|
||||
|
||||
pub const ALL_COLUMN_FAMILY_NAMES: &[&str] = &[
|
||||
CF_REPOSITORY_VIEW,
|
||||
CF_RAW_BY_HASH,
|
||||
CF_RAW_BLOB,
|
||||
CF_MANIFEST_ANTI_ROLLBACK,
|
||||
CF_RRDP_SOURCE,
|
||||
CF_RRDP_SOURCE_MEMBER,
|
||||
CF_RRDP_URI_OWNER,
|
||||
];
|
||||
|
||||
pub(super) const REPOSITORY_VIEW_KEY_PREFIX: &str = "repo_view:";
|
||||
pub(super) const RAW_BY_HASH_KEY_PREFIX: &str = "rawbyhash:";
|
||||
pub(super) const RAW_BLOB_KEY_PREFIX: &str = "rawblob:";
|
||||
pub(super) const MANIFEST_ANTI_ROLLBACK_KEY_PREFIX: &str = "manifest_anti_rollback:";
|
||||
pub(super) const RRDP_SOURCE_KEY_PREFIX: &str = "rrdp_source:";
|
||||
pub(super) const RRDP_SOURCE_MEMBER_KEY_PREFIX: &str = "rrdp_source_member:";
|
||||
pub(super) const RRDP_URI_OWNER_KEY_PREFIX: &str = "rrdp_uri_owner:";
|
||||
|
||||
pub(super) fn configure_work_db_options(opts: &mut Options) {
|
||||
opts.set_compression_type(DBCompressionType::Lz4);
|
||||
}
|
||||
pub fn column_family_descriptors() -> Vec<ColumnFamilyDescriptor> {
|
||||
ALL_COLUMN_FAMILY_NAMES
|
||||
.iter()
|
||||
.map(|name| {
|
||||
let mut opts = Options::default();
|
||||
configure_work_db_options(&mut opts);
|
||||
ColumnFamilyDescriptor::new(*name, opts)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
130
src/repository/storage/keys.rs
Normal file
130
src/repository/storage/keys.rs
Normal file
@@ -0,0 +1,130 @@
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
|
||||
use super::config::*;
|
||||
use super::pack::PackTime;
|
||||
use super::{StorageError, StorageResult};
|
||||
|
||||
pub(super) fn repository_view_key(rsync_uri: &str) -> String {
|
||||
format!("{REPOSITORY_VIEW_KEY_PREFIX}{rsync_uri}")
|
||||
}
|
||||
|
||||
pub(super) fn repository_view_prefix(rsync_uri_prefix: &str) -> String {
|
||||
format!("{REPOSITORY_VIEW_KEY_PREFIX}{rsync_uri_prefix}")
|
||||
}
|
||||
|
||||
pub(super) fn raw_by_hash_key(sha256_hex: &str) -> String {
|
||||
format!("{RAW_BY_HASH_KEY_PREFIX}{sha256_hex}")
|
||||
}
|
||||
|
||||
pub(super) fn raw_blob_key(sha256_hex: &str) -> String {
|
||||
format!("{RAW_BLOB_KEY_PREFIX}{sha256_hex}")
|
||||
}
|
||||
|
||||
pub(super) fn manifest_anti_rollback_key(manifest_rsync_uri: &str) -> String {
|
||||
format!("{MANIFEST_ANTI_ROLLBACK_KEY_PREFIX}{manifest_rsync_uri}")
|
||||
}
|
||||
|
||||
pub(super) fn rrdp_source_key(notify_uri: &str) -> String {
|
||||
format!("{RRDP_SOURCE_KEY_PREFIX}{notify_uri}")
|
||||
}
|
||||
|
||||
pub(super) fn rrdp_source_member_key(notify_uri: &str, rsync_uri: &str) -> String {
|
||||
format!("{RRDP_SOURCE_MEMBER_KEY_PREFIX}{notify_uri}:{rsync_uri}")
|
||||
}
|
||||
|
||||
pub(super) fn rrdp_source_member_prefix(notify_uri: &str) -> String {
|
||||
format!("{RRDP_SOURCE_MEMBER_KEY_PREFIX}{notify_uri}:")
|
||||
}
|
||||
|
||||
pub(super) fn rrdp_uri_owner_key(rsync_uri: &str) -> String {
|
||||
format!("{RRDP_URI_OWNER_KEY_PREFIX}{rsync_uri}")
|
||||
}
|
||||
|
||||
pub(super) fn encode_cbor<T: Serialize>(value: &T, entity: &'static str) -> StorageResult<Vec<u8>> {
|
||||
serde_cbor::to_vec(value).map_err(|e| StorageError::Codec {
|
||||
entity,
|
||||
detail: e.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn decode_cbor<T: DeserializeOwned>(
|
||||
bytes: &[u8],
|
||||
entity: &'static str,
|
||||
) -> StorageResult<T> {
|
||||
serde_cbor::from_slice(bytes).map_err(|e| StorageError::Codec {
|
||||
entity,
|
||||
detail: e.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn validate_non_empty(field: &'static str, value: &str) -> StorageResult<()> {
|
||||
if value.is_empty() {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: field,
|
||||
detail: "must not be empty".to_string(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn validate_sha256_hex(field: &'static str, value: &str) -> StorageResult<()> {
|
||||
if value.len() != 64 || !value.as_bytes().iter().all(u8::is_ascii_hexdigit) {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: field,
|
||||
detail: "must be a 64-character lowercase or uppercase SHA-256 hex string".to_string(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn decode_sha256_hex_32(field: &'static str, value: &str) -> StorageResult<[u8; 32]> {
|
||||
validate_sha256_hex(field, value)?;
|
||||
let mut out = [0u8; 32];
|
||||
hex::decode_to_slice(value, &mut out).map_err(|e| StorageError::InvalidData {
|
||||
entity: field,
|
||||
detail: format!("hex decode failed: {e}"),
|
||||
})?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub(super) fn validate_manifest_number_be(field: &'static str, value: &[u8]) -> StorageResult<()> {
|
||||
if value.is_empty() {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: field,
|
||||
detail: "must not be empty".to_string(),
|
||||
});
|
||||
}
|
||||
if value.len() > 20 {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: field,
|
||||
detail: "must be at most 20 octets".to_string(),
|
||||
});
|
||||
}
|
||||
if value.len() > 1 && value[0] == 0 {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: field,
|
||||
detail: "must be minimal big-endian without leading zeros".to_string(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn validate_sha256_digest_bytes(field: &'static str, value: &[u8]) -> StorageResult<()> {
|
||||
if value.len() != 32 {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: field,
|
||||
detail: format!("must be 32 bytes, got {}", value.len()),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn parse_time(
|
||||
field: &'static str,
|
||||
value: &PackTime,
|
||||
) -> StorageResult<time::OffsetDateTime> {
|
||||
value.parse().map_err(|detail| StorageError::InvalidData {
|
||||
entity: field,
|
||||
detail,
|
||||
})
|
||||
}
|
||||
191
src/repository/storage/memory.rs
Normal file
191
src/repository/storage/memory.rs
Normal file
@@ -0,0 +1,191 @@
|
||||
//! RocksDB and process-memory observation.
|
||||
//!
|
||||
//! This module owns the serialization-friendly memory snapshot model and the
|
||||
//! mapping from RocksDB property names to that model. Storage mutation stays
|
||||
//! in the parent module.
|
||||
|
||||
use rocksdb::DB;
|
||||
use serde::Serialize;
|
||||
|
||||
const ROCKSDB_MEMORY_PROPERTY_NAMES: &[(&str, &str)] = &[
|
||||
("cur_size_all_mem_tables", "rocksdb.cur-size-all-mem-tables"),
|
||||
("size_all_mem_tables", "rocksdb.size-all-mem-tables"),
|
||||
(
|
||||
"estimate_table_readers_mem",
|
||||
"rocksdb.estimate-table-readers-mem",
|
||||
),
|
||||
("block_cache_capacity", "rocksdb.block-cache-capacity"),
|
||||
("block_cache_usage", "rocksdb.block-cache-usage"),
|
||||
(
|
||||
"block_cache_pinned_usage",
|
||||
"rocksdb.block-cache-pinned-usage",
|
||||
),
|
||||
("num_snapshots", "rocksdb.num-snapshots"),
|
||||
("background_errors", "rocksdb.background-errors"),
|
||||
];
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
|
||||
pub struct RocksDbMemoryProperties {
|
||||
pub cur_size_all_mem_tables: Option<u64>,
|
||||
pub size_all_mem_tables: Option<u64>,
|
||||
pub estimate_table_readers_mem: Option<u64>,
|
||||
pub block_cache_capacity: Option<u64>,
|
||||
pub block_cache_usage: Option<u64>,
|
||||
pub block_cache_pinned_usage: Option<u64>,
|
||||
pub num_snapshots: Option<u64>,
|
||||
pub background_errors: Option<u64>,
|
||||
pub errors: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub struct RocksDbColumnFamilyMemoryProperties {
|
||||
pub name: String,
|
||||
pub properties: RocksDbMemoryProperties,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub struct RocksDbMemoryDbSnapshot {
|
||||
pub label: String,
|
||||
pub properties: RocksDbMemoryProperties,
|
||||
pub column_families: Vec<RocksDbColumnFamilyMemoryProperties>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
|
||||
pub struct RocksDbMemoryTotals {
|
||||
pub cur_size_all_mem_tables: u64,
|
||||
pub size_all_mem_tables: u64,
|
||||
pub estimate_table_readers_mem: u64,
|
||||
pub block_cache_capacity: u64,
|
||||
pub block_cache_usage: u64,
|
||||
pub block_cache_pinned_usage: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub struct RocksDbMemorySnapshot {
|
||||
pub databases: Vec<RocksDbMemoryDbSnapshot>,
|
||||
pub totals: RocksDbMemoryTotals,
|
||||
}
|
||||
|
||||
impl RocksDbMemoryTotals {
|
||||
pub(super) fn add_properties(&mut self, properties: &RocksDbMemoryProperties) {
|
||||
self.cur_size_all_mem_tables += properties.cur_size_all_mem_tables.unwrap_or(0);
|
||||
self.size_all_mem_tables += properties.size_all_mem_tables.unwrap_or(0);
|
||||
self.estimate_table_readers_mem += properties.estimate_table_readers_mem.unwrap_or(0);
|
||||
self.block_cache_capacity += properties.block_cache_capacity.unwrap_or(0);
|
||||
self.block_cache_usage += properties.block_cache_usage.unwrap_or(0);
|
||||
self.block_cache_pinned_usage += properties.block_cache_pinned_usage.unwrap_or(0);
|
||||
}
|
||||
}
|
||||
|
||||
fn set_memory_property(properties: &mut RocksDbMemoryProperties, name: &str, value: u64) {
|
||||
match name {
|
||||
"cur_size_all_mem_tables" => properties.cur_size_all_mem_tables = Some(value),
|
||||
"size_all_mem_tables" => properties.size_all_mem_tables = Some(value),
|
||||
"estimate_table_readers_mem" => properties.estimate_table_readers_mem = Some(value),
|
||||
"block_cache_capacity" => properties.block_cache_capacity = Some(value),
|
||||
"block_cache_usage" => properties.block_cache_usage = Some(value),
|
||||
"block_cache_pinned_usage" => properties.block_cache_pinned_usage = Some(value),
|
||||
"num_snapshots" => properties.num_snapshots = Some(value),
|
||||
"background_errors" => properties.background_errors = Some(value),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_rocksdb_property_int(raw: Option<String>) -> Option<u64> {
|
||||
raw.and_then(|value| value.trim().parse::<u64>().ok())
|
||||
}
|
||||
|
||||
fn memory_properties_for_db(db: &DB) -> RocksDbMemoryProperties {
|
||||
let mut properties = RocksDbMemoryProperties::default();
|
||||
for (field_name, property_name) in ROCKSDB_MEMORY_PROPERTY_NAMES {
|
||||
match db.property_value(*property_name) {
|
||||
Ok(value) => {
|
||||
if let Some(parsed) = parse_rocksdb_property_int(value) {
|
||||
set_memory_property(&mut properties, field_name, parsed);
|
||||
}
|
||||
}
|
||||
Err(err) => properties.errors.push(format!("{property_name}: {}", err)),
|
||||
}
|
||||
}
|
||||
properties
|
||||
}
|
||||
|
||||
fn memory_properties_for_cf(db: &DB, cf_name: &'static str) -> RocksDbColumnFamilyMemoryProperties {
|
||||
let mut properties = RocksDbMemoryProperties::default();
|
||||
let Some(cf) = db.cf_handle(cf_name) else {
|
||||
properties
|
||||
.errors
|
||||
.push(format!("missing column family: {cf_name}"));
|
||||
return RocksDbColumnFamilyMemoryProperties {
|
||||
name: cf_name.to_string(),
|
||||
properties,
|
||||
};
|
||||
};
|
||||
for (field_name, property_name) in ROCKSDB_MEMORY_PROPERTY_NAMES {
|
||||
match db.property_value_cf(cf, *property_name) {
|
||||
Ok(value) => {
|
||||
if let Some(parsed) = parse_rocksdb_property_int(value) {
|
||||
set_memory_property(&mut properties, field_name, parsed);
|
||||
}
|
||||
}
|
||||
Err(err) => properties.errors.push(format!("{property_name}: {}", err)),
|
||||
}
|
||||
}
|
||||
RocksDbColumnFamilyMemoryProperties {
|
||||
name: cf_name.to_string(),
|
||||
properties,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn memory_db_snapshot_for_column_families(
|
||||
label: impl Into<String>,
|
||||
db: &DB,
|
||||
column_families: Option<&[&'static str]>,
|
||||
) -> RocksDbMemoryDbSnapshot {
|
||||
RocksDbMemoryDbSnapshot {
|
||||
label: label.into(),
|
||||
properties: memory_properties_for_db(db),
|
||||
column_families: column_families
|
||||
.map(|names| {
|
||||
names
|
||||
.iter()
|
||||
.map(|name| memory_properties_for_cf(db, name))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
RocksDbMemoryProperties, RocksDbMemoryTotals, parse_rocksdb_property_int,
|
||||
set_memory_property,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn parses_rocksdb_integer_properties_without_panicking() {
|
||||
assert_eq!(
|
||||
parse_rocksdb_property_int(Some(" 42 ".to_string())),
|
||||
Some(42)
|
||||
);
|
||||
assert_eq!(
|
||||
parse_rocksdb_property_int(Some("not-a-number".to_string())),
|
||||
None
|
||||
);
|
||||
assert_eq!(parse_rocksdb_property_int(None), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aggregates_only_known_memory_properties() {
|
||||
let mut properties = RocksDbMemoryProperties::default();
|
||||
set_memory_property(&mut properties, "block_cache_usage", 11);
|
||||
set_memory_property(&mut properties, "unknown", 99);
|
||||
|
||||
let mut totals = RocksDbMemoryTotals::default();
|
||||
totals.add_properties(&properties);
|
||||
|
||||
assert_eq!(totals.block_cache_usage, 11);
|
||||
assert_eq!(totals.block_cache_capacity, 0);
|
||||
}
|
||||
}
|
||||
153
src/repository/storage/models_core.rs
Normal file
153
src/repository/storage/models_core.rs
Normal file
@@ -0,0 +1,153 @@
|
||||
// Repository view, raw-object records, and protocol rollback metadata.
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum StorageError {
|
||||
#[error("rocksdb error: {0}")]
|
||||
RocksDb(String),
|
||||
#[error("missing column family: {0}")]
|
||||
MissingColumnFamily(&'static str),
|
||||
#[error("cbor codec error for {entity}: {detail}")]
|
||||
Codec { entity: &'static str, detail: String },
|
||||
#[error("invalid {entity}: {detail}")]
|
||||
InvalidData { entity: &'static str, detail: String },
|
||||
}
|
||||
|
||||
pub type StorageResult<T> = Result<T, StorageError>;
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
|
||||
pub struct RepositoryBlobVerificationSummary {
|
||||
pub current_objects: u64,
|
||||
pub bytes_verified: u64,
|
||||
pub batches: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum RrdpDeltaOp {
|
||||
Upsert { rsync_uri: String, bytes: Vec<u8> },
|
||||
Delete { rsync_uri: String },
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum RepositoryViewState {
|
||||
Present,
|
||||
Withdrawn,
|
||||
Replaced,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RepositoryViewEntry {
|
||||
pub rsync_uri: String,
|
||||
pub current_hash: Option<String>,
|
||||
pub repository_source: Option<String>,
|
||||
pub object_type: Option<String>,
|
||||
pub state: RepositoryViewState,
|
||||
}
|
||||
|
||||
impl RepositoryViewEntry {
|
||||
pub fn validate_internal(&self) -> StorageResult<()> {
|
||||
validate_non_empty("repository_view.rsync_uri", &self.rsync_uri)?;
|
||||
if let Some(source) = &self.repository_source {
|
||||
validate_non_empty("repository_view.repository_source", source)?;
|
||||
}
|
||||
match self.state {
|
||||
RepositoryViewState::Present | RepositoryViewState::Replaced => {
|
||||
let hash = self.current_hash.as_deref().ok_or(StorageError::InvalidData {
|
||||
entity: "repository_view",
|
||||
detail: "current_hash is required when state is present or replaced".to_string(),
|
||||
})?;
|
||||
validate_sha256_hex("repository_view.current_hash", hash)?;
|
||||
}
|
||||
RepositoryViewState::Withdrawn => {
|
||||
if let Some(hash) = &self.current_hash {
|
||||
validate_sha256_hex("repository_view.current_hash", hash)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RawByHashEntry {
|
||||
pub sha256_hex: String,
|
||||
pub bytes: Vec<u8>,
|
||||
pub origin_uris: Vec<String>,
|
||||
pub object_type: Option<String>,
|
||||
pub encoding: Option<String>,
|
||||
}
|
||||
|
||||
impl RawByHashEntry {
|
||||
pub fn from_bytes(sha256_hex: impl Into<String>, bytes: Vec<u8>) -> Self {
|
||||
Self { sha256_hex: sha256_hex.into(), bytes, origin_uris: Vec::new(), object_type: None, encoding: None }
|
||||
}
|
||||
|
||||
pub fn validate_internal(&self) -> StorageResult<()> {
|
||||
validate_sha256_hex("raw_by_hash.sha256_hex", &self.sha256_hex)?;
|
||||
if self.bytes.is_empty() {
|
||||
return Err(StorageError::InvalidData { entity: "raw_by_hash", detail: "bytes must not be empty".to_string() });
|
||||
}
|
||||
let computed = hex::encode(compute_sha256_32(&self.bytes));
|
||||
if computed != self.sha256_hex.to_ascii_lowercase() {
|
||||
return Err(StorageError::InvalidData { entity: "raw_by_hash", detail: "sha256_hex does not match bytes".to_string() });
|
||||
}
|
||||
let mut seen = HashSet::with_capacity(self.origin_uris.len());
|
||||
for uri in &self.origin_uris {
|
||||
validate_non_empty("raw_by_hash.origin_uris[]", uri)?;
|
||||
if !seen.insert(uri.as_str()) {
|
||||
return Err(StorageError::InvalidData { entity: "raw_by_hash", detail: format!("duplicate origin URI: {uri}") });
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct CurrentObjectWithHash {
|
||||
pub current_hash_hex: String,
|
||||
pub current_hash: [u8; 32],
|
||||
pub bytes: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ValidatedManifestMeta {
|
||||
pub validated_manifest_number: Vec<u8>,
|
||||
pub validated_manifest_this_update: PackTime,
|
||||
pub validated_manifest_next_update: PackTime,
|
||||
}
|
||||
|
||||
impl ValidatedManifestMeta {
|
||||
pub fn validate_internal(&self) -> StorageResult<()> {
|
||||
validate_manifest_number_be("validated_manifest_meta.validated_manifest_number", &self.validated_manifest_number)?;
|
||||
let this_update = parse_time("validated_manifest_meta.validated_manifest_this_update", &self.validated_manifest_this_update)?;
|
||||
let next_update = parse_time("validated_manifest_meta.validated_manifest_next_update", &self.validated_manifest_next_update)?;
|
||||
if next_update < this_update {
|
||||
return Err(StorageError::InvalidData { entity: "validated_manifest_meta", detail: "validated_manifest_next_update must be >= validated_manifest_this_update".to_string() });
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Persistent manifest freshness metadata used by RFC 9286 anti-rollback
|
||||
/// checks. It stores no validated object output.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ManifestAntiRollbackMeta {
|
||||
pub manifest_rsync_uri: String,
|
||||
pub manifest_number_be: Vec<u8>,
|
||||
pub manifest_this_update: PackTime,
|
||||
pub manifest_sha256: Vec<u8>,
|
||||
pub updated_at_validation_time: PackTime,
|
||||
}
|
||||
|
||||
impl ManifestAntiRollbackMeta {
|
||||
pub fn validate_internal(&self) -> StorageResult<()> {
|
||||
validate_non_empty("manifest_anti_rollback.manifest_rsync_uri", &self.manifest_rsync_uri)?;
|
||||
validate_manifest_number_be("manifest_anti_rollback.manifest_number_be", &self.manifest_number_be)?;
|
||||
parse_time("manifest_anti_rollback.manifest_this_update", &self.manifest_this_update)?;
|
||||
validate_sha256_digest_bytes("manifest_anti_rollback.manifest_sha256", &self.manifest_sha256)?;
|
||||
parse_time("manifest_anti_rollback.updated_at_validation_time", &self.updated_at_validation_time)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub use crate::ccr::projection::CcrManifestProjection;
|
||||
117
src/repository/storage/models_summary.rs
Normal file
117
src/repository/storage/models_summary.rs
Normal file
@@ -0,0 +1,117 @@
|
||||
// RRDP protocol state records.
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum RrdpSourceSyncState {
|
||||
Empty,
|
||||
SnapshotOnly,
|
||||
DeltaReady,
|
||||
Error,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RrdpSourceRecord {
|
||||
pub notify_uri: String,
|
||||
pub last_session_id: Option<String>,
|
||||
pub last_serial: Option<u64>,
|
||||
pub first_seen_at: PackTime,
|
||||
pub last_seen_at: PackTime,
|
||||
pub last_sync_at: Option<PackTime>,
|
||||
pub sync_state: RrdpSourceSyncState,
|
||||
pub last_snapshot_uri: Option<String>,
|
||||
pub last_snapshot_hash: Option<String>,
|
||||
pub last_error: Option<String>,
|
||||
}
|
||||
|
||||
impl RrdpSourceRecord {
|
||||
pub fn validate_internal(&self) -> StorageResult<()> {
|
||||
validate_non_empty("rrdp_source.notify_uri", &self.notify_uri)?;
|
||||
if let Some(session_id) = &self.last_session_id {
|
||||
validate_non_empty("rrdp_source.last_session_id", session_id)?;
|
||||
}
|
||||
parse_time("rrdp_source.first_seen_at", &self.first_seen_at)?;
|
||||
parse_time("rrdp_source.last_seen_at", &self.last_seen_at)?;
|
||||
if let Some(last_sync_at) = &self.last_sync_at {
|
||||
parse_time("rrdp_source.last_sync_at", last_sync_at)?;
|
||||
}
|
||||
if let Some(last_snapshot_uri) = &self.last_snapshot_uri {
|
||||
validate_non_empty("rrdp_source.last_snapshot_uri", last_snapshot_uri)?;
|
||||
}
|
||||
if let Some(last_snapshot_hash) = &self.last_snapshot_hash {
|
||||
validate_sha256_hex("rrdp_source.last_snapshot_hash", last_snapshot_hash)?;
|
||||
}
|
||||
if let Some(last_error) = &self.last_error {
|
||||
validate_non_empty("rrdp_source.last_error", last_error)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RrdpSourceMemberRecord {
|
||||
pub notify_uri: String,
|
||||
pub rsync_uri: String,
|
||||
pub current_hash: Option<String>,
|
||||
pub object_type: Option<String>,
|
||||
pub present: bool,
|
||||
pub last_confirmed_session_id: String,
|
||||
pub last_confirmed_serial: u64,
|
||||
pub last_changed_at: PackTime,
|
||||
}
|
||||
|
||||
impl RrdpSourceMemberRecord {
|
||||
pub fn validate_internal(&self) -> StorageResult<()> {
|
||||
validate_non_empty("rrdp_source_member.notify_uri", &self.notify_uri)?;
|
||||
validate_non_empty("rrdp_source_member.rsync_uri", &self.rsync_uri)?;
|
||||
validate_non_empty(
|
||||
"rrdp_source_member.last_confirmed_session_id",
|
||||
&self.last_confirmed_session_id,
|
||||
)?;
|
||||
if self.present {
|
||||
let hash = self.current_hash.as_deref().ok_or(StorageError::InvalidData {
|
||||
entity: "rrdp_source_member",
|
||||
detail: "current_hash is required when present=true".to_string(),
|
||||
})?;
|
||||
validate_sha256_hex("rrdp_source_member.current_hash", hash)?;
|
||||
} else if let Some(hash) = &self.current_hash {
|
||||
validate_sha256_hex("rrdp_source_member.current_hash", hash)?;
|
||||
}
|
||||
parse_time("rrdp_source_member.last_changed_at", &self.last_changed_at)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum RrdpUriOwnerState {
|
||||
Active,
|
||||
Conflict,
|
||||
Withdrawn,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RrdpUriOwnerRecord {
|
||||
pub rsync_uri: String,
|
||||
pub notify_uri: String,
|
||||
pub current_hash: Option<String>,
|
||||
pub last_confirmed_session_id: String,
|
||||
pub last_confirmed_serial: u64,
|
||||
pub last_changed_at: PackTime,
|
||||
pub owner_state: RrdpUriOwnerState,
|
||||
}
|
||||
|
||||
impl RrdpUriOwnerRecord {
|
||||
pub fn validate_internal(&self) -> StorageResult<()> {
|
||||
validate_non_empty("rrdp_uri_owner.rsync_uri", &self.rsync_uri)?;
|
||||
validate_non_empty("rrdp_uri_owner.notify_uri", &self.notify_uri)?;
|
||||
validate_non_empty(
|
||||
"rrdp_uri_owner.last_confirmed_session_id",
|
||||
&self.last_confirmed_session_id,
|
||||
)?;
|
||||
if let Some(hash) = &self.current_hash {
|
||||
validate_sha256_hex("rrdp_uri_owner.current_hash", hash)?;
|
||||
}
|
||||
parse_time("rrdp_uri_owner.last_changed_at", &self.last_changed_at)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
200
src/repository/storage/pack.rs
Normal file
200
src/repository/storage/pack.rs
Normal file
@@ -0,0 +1,200 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::Digest;
|
||||
|
||||
use crate::repository::blob_store::{ExternalRawStoreDb, ExternalRepoBytesDb, RawObjectStore};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum PackBytes {
|
||||
Eager(std::sync::Arc<[u8]>),
|
||||
LazyExternal {
|
||||
sha256_hex: String,
|
||||
store: std::sync::Arc<ExternalRawStoreDb>,
|
||||
cache: std::sync::Arc<std::sync::OnceLock<std::sync::Arc<[u8]>>>,
|
||||
},
|
||||
LazyRepoBytes {
|
||||
sha256_hex: String,
|
||||
store: std::sync::Arc<ExternalRepoBytesDb>,
|
||||
cache: std::sync::Arc<std::sync::OnceLock<std::sync::Arc<[u8]>>>,
|
||||
},
|
||||
}
|
||||
|
||||
impl PackBytes {
|
||||
pub fn eager(bytes: Vec<u8>) -> Self {
|
||||
Self::Eager(std::sync::Arc::from(bytes))
|
||||
}
|
||||
|
||||
pub fn lazy_external(sha256_hex: String, store: std::sync::Arc<ExternalRawStoreDb>) -> Self {
|
||||
Self::LazyExternal {
|
||||
sha256_hex,
|
||||
store,
|
||||
cache: std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn lazy_repo_bytes(sha256_hex: String, store: std::sync::Arc<ExternalRepoBytesDb>) -> Self {
|
||||
Self::LazyRepoBytes {
|
||||
sha256_hex,
|
||||
store,
|
||||
cache: std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_slice(&self) -> Result<&[u8], String> {
|
||||
match self {
|
||||
Self::Eager(bytes) => Ok(bytes.as_ref()),
|
||||
Self::LazyExternal {
|
||||
sha256_hex,
|
||||
store,
|
||||
cache,
|
||||
} => {
|
||||
if cache.get().is_none() {
|
||||
let bytes = store
|
||||
.get_blob_bytes(sha256_hex)
|
||||
.map_err(|e| e.to_string())?
|
||||
.ok_or_else(|| format!("missing raw blob for sha256={sha256_hex}"))?;
|
||||
let _ = cache.set(std::sync::Arc::from(bytes));
|
||||
}
|
||||
let bytes = cache
|
||||
.get()
|
||||
.ok_or_else(|| format!("missing raw blob cache for sha256={sha256_hex}"))?;
|
||||
Ok(bytes.as_ref())
|
||||
}
|
||||
Self::LazyRepoBytes {
|
||||
sha256_hex,
|
||||
store,
|
||||
cache,
|
||||
} => {
|
||||
if cache.get().is_none() {
|
||||
let bytes = store
|
||||
.get_blob_bytes(sha256_hex)
|
||||
.map_err(|e| e.to_string())?
|
||||
.ok_or_else(|| format!("missing repo bytes for sha256={sha256_hex}"))?;
|
||||
let _ = cache.set(std::sync::Arc::from(bytes));
|
||||
}
|
||||
let bytes = cache
|
||||
.get()
|
||||
.ok_or_else(|| format!("missing repo bytes cache for sha256={sha256_hex}"))?;
|
||||
Ok(bytes.as_ref())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_vec(&self) -> Result<Vec<u8>, String> {
|
||||
Ok(self.as_slice()?.to_vec())
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for PackBytes {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
match (self.as_slice(), other.as_slice()) {
|
||||
(Ok(a), Ok(b)) => a == b,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for PackBytes {}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PackFile {
|
||||
pub rsync_uri: String,
|
||||
pub bytes: PackBytes,
|
||||
pub sha256: [u8; 32],
|
||||
}
|
||||
|
||||
impl PackFile {
|
||||
pub fn new(rsync_uri: impl Into<String>, bytes: PackBytes, sha256: [u8; 32]) -> Self {
|
||||
Self {
|
||||
rsync_uri: rsync_uri.into(),
|
||||
bytes,
|
||||
sha256,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_bytes_with_sha256(
|
||||
rsync_uri: impl Into<String>,
|
||||
bytes: Vec<u8>,
|
||||
sha256: [u8; 32],
|
||||
) -> Self {
|
||||
Self::new(rsync_uri, PackBytes::eager(bytes), sha256)
|
||||
}
|
||||
|
||||
pub fn from_lazy_external_raw_store(
|
||||
rsync_uri: impl Into<String>,
|
||||
sha256_hex: String,
|
||||
sha256: [u8; 32],
|
||||
store: std::sync::Arc<ExternalRawStoreDb>,
|
||||
) -> Self {
|
||||
Self::new(
|
||||
rsync_uri,
|
||||
PackBytes::lazy_external(sha256_hex, store),
|
||||
sha256,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn from_lazy_repo_bytes(
|
||||
rsync_uri: impl Into<String>,
|
||||
sha256_hex: String,
|
||||
sha256: [u8; 32],
|
||||
store: std::sync::Arc<ExternalRepoBytesDb>,
|
||||
) -> Self {
|
||||
Self::new(
|
||||
rsync_uri,
|
||||
PackBytes::lazy_repo_bytes(sha256_hex, store),
|
||||
sha256,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn from_bytes_compute_sha256(rsync_uri: impl Into<String>, bytes: Vec<u8>) -> Self {
|
||||
let sha256 = compute_sha256_32(&bytes);
|
||||
Self::new(rsync_uri, PackBytes::eager(bytes), sha256)
|
||||
}
|
||||
|
||||
pub fn bytes(&self) -> Result<&[u8], String> {
|
||||
self.bytes.as_slice()
|
||||
}
|
||||
|
||||
pub fn bytes_cloned(&self) -> Result<Vec<u8>, String> {
|
||||
self.bytes.to_vec()
|
||||
}
|
||||
|
||||
pub fn compute_sha256(&self) -> Result<[u8; 32], String> {
|
||||
Ok(compute_sha256_32(self.bytes()?))
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for PackFile {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.rsync_uri == other.rsync_uri
|
||||
&& self.sha256 == other.sha256
|
||||
&& self.bytes == other.bytes
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for PackFile {}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PackTime {
|
||||
pub rfc3339_utc: String,
|
||||
}
|
||||
|
||||
impl PackTime {
|
||||
pub fn from_utc_offset_datetime(t: time::OffsetDateTime) -> Self {
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
let utc = t.to_offset(time::UtcOffset::UTC);
|
||||
let s = utc.format(&Rfc3339).expect("format RFC 3339 UTC time");
|
||||
Self { rfc3339_utc: s }
|
||||
}
|
||||
|
||||
pub fn parse(&self) -> Result<time::OffsetDateTime, String> {
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
time::OffsetDateTime::parse(&self.rfc3339_utc, &Rfc3339).map_err(|e| e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn compute_sha256_32(bytes: &[u8]) -> [u8; 32] {
|
||||
let digest = sha2::Sha256::digest(bytes);
|
||||
let mut out = [0u8; 32];
|
||||
out.copy_from_slice(&digest);
|
||||
out
|
||||
}
|
||||
158
src/repository/storage/store_lifecycle.rs
Normal file
158
src/repository/storage/store_lifecycle.rs
Normal file
@@ -0,0 +1,158 @@
|
||||
// RocksDB lifecycle and external repository stores.
|
||||
|
||||
impl RocksStore {
|
||||
pub fn create_read_only_checkpoint(source: &Path, destination: &Path) -> StorageResult<()> {
|
||||
reject_unsupported_column_families(source)?;
|
||||
if destination.exists() {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: "work_db_checkpoint",
|
||||
detail: format!(
|
||||
"checkpoint destination already exists: {}",
|
||||
destination.display()
|
||||
),
|
||||
});
|
||||
}
|
||||
if let Some(parent) = destination.parent() {
|
||||
std::fs::create_dir_all(parent)
|
||||
.map_err(|error| StorageError::RocksDb(error.to_string()))?;
|
||||
}
|
||||
let mut options = Options::default();
|
||||
configure_work_db_options(&mut options);
|
||||
let descriptors = column_family_descriptors();
|
||||
let db = DB::open_cf_descriptors_read_only(
|
||||
&options,
|
||||
source,
|
||||
descriptors,
|
||||
false,
|
||||
)
|
||||
.map_err(|error| StorageError::RocksDb(error.to_string()))?;
|
||||
let checkpoint =
|
||||
Checkpoint::new(&db).map_err(|error| StorageError::RocksDb(error.to_string()))?;
|
||||
checkpoint
|
||||
.create_checkpoint(destination)
|
||||
.map_err(|error| StorageError::RocksDb(error.to_string()))
|
||||
}
|
||||
|
||||
pub fn open(path: &Path) -> StorageResult<Self> {
|
||||
reject_unsupported_column_families(path)?;
|
||||
let mut base_opts = Options::default();
|
||||
base_opts.create_if_missing(true);
|
||||
base_opts.create_missing_column_families(true);
|
||||
configure_work_db_options(&mut base_opts);
|
||||
let descriptors = column_family_descriptors();
|
||||
|
||||
let db = DB::open_cf_descriptors(
|
||||
&base_opts,
|
||||
path,
|
||||
descriptors,
|
||||
)
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
|
||||
Ok(Self {
|
||||
db,
|
||||
external_raw_store: None,
|
||||
external_repo_bytes: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn open_with_external_raw_store(path: &Path, raw_store_path: &Path) -> StorageResult<Self> {
|
||||
Self::open_with_external_stores(path, Some(raw_store_path), None)
|
||||
}
|
||||
|
||||
pub fn open_with_external_repo_bytes(
|
||||
path: &Path,
|
||||
repo_bytes_path: &Path,
|
||||
) -> StorageResult<Self> {
|
||||
Self::open_with_external_stores(path, None, Some(repo_bytes_path))
|
||||
}
|
||||
|
||||
pub fn open_with_external_repo_bytes_read_only(
|
||||
path: &Path,
|
||||
repo_bytes_path: &Path,
|
||||
) -> StorageResult<Self> {
|
||||
let mut store = Self::open(path)?;
|
||||
store.external_repo_bytes = Some(ExternalRepoBytesDb::open_read_only(repo_bytes_path)?);
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
pub fn open_with_external_stores(
|
||||
path: &Path,
|
||||
raw_store_path: Option<&Path>,
|
||||
repo_bytes_path: Option<&Path>,
|
||||
) -> StorageResult<Self> {
|
||||
let mut store = Self::open(path)?;
|
||||
if let Some(raw_store_path) = raw_store_path {
|
||||
store.external_raw_store = Some(ExternalRawStoreDb::open(raw_store_path)?);
|
||||
}
|
||||
if let Some(repo_bytes_path) = repo_bytes_path {
|
||||
store.external_repo_bytes = Some(ExternalRepoBytesDb::open(repo_bytes_path)?);
|
||||
}
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
pub(crate) fn external_raw_store_ref(&self) -> Option<&ExternalRawStoreDb> {
|
||||
self.external_raw_store.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn external_repo_bytes_ref(&self) -> Option<&ExternalRepoBytesDb> {
|
||||
self.external_repo_bytes.as_ref()
|
||||
}
|
||||
|
||||
pub fn memory_snapshot(&self) -> RocksDbMemorySnapshot {
|
||||
let mut databases = Vec::new();
|
||||
databases.push(memory_db_snapshot_for_column_families(
|
||||
"work-db",
|
||||
&self.db,
|
||||
Some(ALL_COLUMN_FAMILY_NAMES),
|
||||
));
|
||||
if let Some(raw_store) = self.external_raw_store.as_ref() {
|
||||
databases.push(raw_store.memory_snapshot("raw-store.db"));
|
||||
}
|
||||
if let Some(repo_bytes) = self.external_repo_bytes.as_ref() {
|
||||
databases.push(repo_bytes.memory_snapshot("repo-bytes.db"));
|
||||
}
|
||||
|
||||
let mut totals = RocksDbMemoryTotals::default();
|
||||
for db in &databases {
|
||||
totals.add_properties(&db.properties);
|
||||
}
|
||||
RocksDbMemorySnapshot { databases, totals }
|
||||
}
|
||||
|
||||
fn cf(&self, name: &'static str) -> StorageResult<&ColumnFamily> {
|
||||
self.db
|
||||
.cf_handle(name)
|
||||
.ok_or(StorageError::MissingColumnFamily(name))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
fn reject_unsupported_column_families(path: &Path) -> StorageResult<()> {
|
||||
// An empty directory is a valid first-run state root. A RocksDB directory
|
||||
// has CURRENT; inspect its column families before opening so an older
|
||||
// private schema cannot be silently ignored or partially loaded.
|
||||
if !path.join("CURRENT").exists() {
|
||||
return Ok(());
|
||||
}
|
||||
let options = Options::default();
|
||||
let existing = DB::list_cf(&options, path)
|
||||
.map_err(|error| StorageError::RocksDb(format!("inspect state schema: {error}")))?;
|
||||
let supported: std::collections::HashSet<&str> = std::iter::once("default")
|
||||
.chain(ALL_COLUMN_FAMILY_NAMES.iter().copied())
|
||||
.collect();
|
||||
let unsupported: Vec<_> = existing
|
||||
.iter()
|
||||
.filter(|name| !supported.contains(name.as_str()))
|
||||
.cloned()
|
||||
.collect();
|
||||
if unsupported.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
Err(StorageError::InvalidData {
|
||||
entity: "state_schema",
|
||||
detail: format!(
|
||||
"unsupported column families {:?}; create a new state directory instead of reusing this database",
|
||||
unsupported
|
||||
),
|
||||
})
|
||||
}
|
||||
33
src/repository/storage/store_manifest.rs
Normal file
33
src/repository/storage/store_manifest.rs
Normal file
@@ -0,0 +1,33 @@
|
||||
// Manifest rollback metadata is protocol state retained independently from
|
||||
// repository objects and validation outputs.
|
||||
|
||||
impl RocksStore {
|
||||
pub fn get_manifest_anti_rollback_meta(
|
||||
&self,
|
||||
manifest_rsync_uri: &str,
|
||||
) -> StorageResult<Option<ManifestAntiRollbackMeta>> {
|
||||
let cf = self.cf(CF_MANIFEST_ANTI_ROLLBACK)?;
|
||||
let key = manifest_anti_rollback_key(manifest_rsync_uri);
|
||||
let Some(bytes) = self
|
||||
.db
|
||||
.get_cf(cf, key.as_bytes())
|
||||
.map_err(|error| StorageError::RocksDb(error.to_string()))?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let metadata = decode_cbor::<ManifestAntiRollbackMeta>(&bytes, "manifest_anti_rollback")?;
|
||||
metadata.validate_internal()?;
|
||||
Ok(Some(metadata))
|
||||
}
|
||||
|
||||
pub fn put_manifest_anti_rollback_meta(&self, metadata: &ManifestAntiRollbackMeta) -> StorageResult<()> {
|
||||
metadata.validate_internal()?;
|
||||
let cf = self.cf(CF_MANIFEST_ANTI_ROLLBACK)?;
|
||||
let key = manifest_anti_rollback_key(&metadata.manifest_rsync_uri);
|
||||
let value = encode_cbor(metadata, "manifest_anti_rollback")?;
|
||||
self.db
|
||||
.put_cf(cf, key.as_bytes(), value)
|
||||
.map_err(|error| StorageError::RocksDb(error.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
363
src/repository/storage/store_repository.rs
Normal file
363
src/repository/storage/store_repository.rs
Normal file
@@ -0,0 +1,363 @@
|
||||
// Repository-view and raw-object/blob storage operations.
|
||||
|
||||
impl RocksStore {
|
||||
|
||||
pub fn put_repository_view_entry(&self, entry: &RepositoryViewEntry) -> StorageResult<()> {
|
||||
entry.validate_internal()?;
|
||||
let cf = self.cf(CF_REPOSITORY_VIEW)?;
|
||||
let key = repository_view_key(&entry.rsync_uri);
|
||||
let value = encode_cbor(entry, "repository_view")?;
|
||||
self.db
|
||||
.put_cf(cf, key.as_bytes(), value)
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_repository_view_entry(
|
||||
&self,
|
||||
rsync_uri: &str,
|
||||
) -> StorageResult<Option<RepositoryViewEntry>> {
|
||||
let cf = self.cf(CF_REPOSITORY_VIEW)?;
|
||||
let key = repository_view_key(rsync_uri);
|
||||
let Some(bytes) = self
|
||||
.db
|
||||
.get_cf(cf, key.as_bytes())
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let entry = decode_cbor::<RepositoryViewEntry>(&bytes, "repository_view")?;
|
||||
entry.validate_internal()?;
|
||||
Ok(Some(entry))
|
||||
}
|
||||
|
||||
pub fn delete_repository_view_entry(&self, rsync_uri: &str) -> StorageResult<()> {
|
||||
let cf = self.cf(CF_REPOSITORY_VIEW)?;
|
||||
let key = repository_view_key(rsync_uri);
|
||||
self.db
|
||||
.delete_cf(cf, key.as_bytes())
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn put_projection_batch(
|
||||
&self,
|
||||
repository_view_entries: &[RepositoryViewEntry],
|
||||
member_records: &[RrdpSourceMemberRecord],
|
||||
owner_records: &[RrdpUriOwnerRecord],
|
||||
) -> StorageResult<()> {
|
||||
if repository_view_entries.is_empty()
|
||||
&& member_records.is_empty()
|
||||
&& owner_records.is_empty()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let repo_cf = self.cf(CF_REPOSITORY_VIEW)?;
|
||||
let member_cf = self.cf(CF_RRDP_SOURCE_MEMBER)?;
|
||||
let owner_cf = self.cf(CF_RRDP_URI_OWNER)?;
|
||||
let mut batch = WriteBatch::default();
|
||||
|
||||
for entry in repository_view_entries {
|
||||
entry.validate_internal()?;
|
||||
let key = repository_view_key(&entry.rsync_uri);
|
||||
let value = encode_cbor(entry, "repository_view")?;
|
||||
batch.put_cf(repo_cf, key.as_bytes(), value);
|
||||
}
|
||||
for record in member_records {
|
||||
record.validate_internal()?;
|
||||
let key = rrdp_source_member_key(&record.notify_uri, &record.rsync_uri);
|
||||
let value = encode_cbor(record, "rrdp_source_member")?;
|
||||
batch.put_cf(member_cf, key.as_bytes(), value);
|
||||
}
|
||||
for record in owner_records {
|
||||
record.validate_internal()?;
|
||||
let key = rrdp_uri_owner_key(&record.rsync_uri);
|
||||
let value = encode_cbor(record, "rrdp_uri_owner")?;
|
||||
batch.put_cf(owner_cf, key.as_bytes(), value);
|
||||
}
|
||||
|
||||
self.write_batch(batch)
|
||||
}
|
||||
|
||||
pub fn list_repository_view_entries_with_prefix(
|
||||
&self,
|
||||
rsync_uri_prefix: &str,
|
||||
) -> StorageResult<Vec<RepositoryViewEntry>> {
|
||||
let cf = self.cf(CF_REPOSITORY_VIEW)?;
|
||||
let prefix = repository_view_prefix(rsync_uri_prefix);
|
||||
let mode = IteratorMode::From(prefix.as_bytes(), Direction::Forward);
|
||||
self.db
|
||||
.iterator_cf(cf, mode)
|
||||
.take_while(|res| match res {
|
||||
Ok((key, _)) => key.starts_with(prefix.as_bytes()),
|
||||
Err(_) => false,
|
||||
})
|
||||
.map(|res| {
|
||||
let (_key, value) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
let entry = decode_cbor::<RepositoryViewEntry>(&value, "repository_view")?;
|
||||
entry.validate_internal()?;
|
||||
Ok(entry)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn verify_current_repository_blobs(
|
||||
&self,
|
||||
batch_size: usize,
|
||||
) -> StorageResult<RepositoryBlobVerificationSummary> {
|
||||
if batch_size == 0 {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: "repository_blob_verification",
|
||||
detail: "batch_size must be greater than zero".to_string(),
|
||||
});
|
||||
}
|
||||
let cf = self.cf(CF_REPOSITORY_VIEW)?;
|
||||
let prefix = repository_view_prefix("");
|
||||
let mode = IteratorMode::From(prefix.as_bytes(), Direction::Forward);
|
||||
let mut summary = RepositoryBlobVerificationSummary::default();
|
||||
let mut batch = Vec::with_capacity(batch_size);
|
||||
for item in self.db.iterator_cf(cf, mode) {
|
||||
let (key, value) = item.map_err(|error| StorageError::RocksDb(error.to_string()))?;
|
||||
if !key.starts_with(prefix.as_bytes()) {
|
||||
break;
|
||||
}
|
||||
let entry = decode_cbor::<RepositoryViewEntry>(&value, "repository_view")?;
|
||||
entry.validate_internal()?;
|
||||
if !matches!(
|
||||
entry.state,
|
||||
RepositoryViewState::Present | RepositoryViewState::Replaced
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
let hash = entry
|
||||
.current_hash
|
||||
.clone()
|
||||
.ok_or(StorageError::InvalidData {
|
||||
entity: "repository_blob_verification",
|
||||
detail: format!("current hash missing for {}", entry.rsync_uri),
|
||||
})?;
|
||||
batch.push((entry.rsync_uri, hash));
|
||||
if batch.len() >= batch_size {
|
||||
verify_repository_blob_batch(self, &batch, &mut summary)?;
|
||||
batch.clear();
|
||||
}
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
verify_repository_blob_batch(self, &batch, &mut summary)?;
|
||||
}
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
pub fn put_raw_by_hash_entry(&self, entry: &RawByHashEntry) -> StorageResult<()> {
|
||||
entry.validate_internal()?;
|
||||
if let Some(raw_store) = self.external_raw_store.as_ref() {
|
||||
return raw_store.put_raw_entry(entry);
|
||||
}
|
||||
let cf = self.cf(CF_RAW_BY_HASH)?;
|
||||
let key = raw_by_hash_key(&entry.sha256_hex);
|
||||
let value = encode_cbor(entry, "raw_by_hash")?;
|
||||
self.db
|
||||
.put_cf(cf, key.as_bytes(), value)
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn put_raw_by_hash_entries_batch(&self, entries: &[RawByHashEntry]) -> StorageResult<()> {
|
||||
if entries.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(raw_store) = self.external_raw_store.as_ref() {
|
||||
return raw_store.put_raw_entries_batch(entries);
|
||||
}
|
||||
|
||||
let cf = self.cf(CF_RAW_BY_HASH)?;
|
||||
let mut batch = WriteBatch::default();
|
||||
for entry in entries {
|
||||
entry.validate_internal()?;
|
||||
let key = raw_by_hash_key(&entry.sha256_hex);
|
||||
let value = encode_cbor(entry, "raw_by_hash")?;
|
||||
batch.put_cf(cf, key.as_bytes(), value);
|
||||
}
|
||||
self.write_batch(batch)
|
||||
}
|
||||
|
||||
pub fn put_raw_by_hash_entries_batch_unchecked(
|
||||
&self,
|
||||
entries: &[RawByHashEntry],
|
||||
) -> StorageResult<()> {
|
||||
if entries.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(raw_store) = self.external_raw_store.as_ref() {
|
||||
return raw_store.put_raw_entries_batch(entries);
|
||||
}
|
||||
|
||||
let cf = self.cf(CF_RAW_BY_HASH)?;
|
||||
let mut batch = WriteBatch::default();
|
||||
for entry in entries {
|
||||
let key = raw_by_hash_key(&entry.sha256_hex);
|
||||
let value = encode_cbor(entry, "raw_by_hash")?;
|
||||
batch.put_cf(cf, key.as_bytes(), value);
|
||||
}
|
||||
self.write_batch(batch)
|
||||
}
|
||||
|
||||
pub fn put_blob_bytes_batch(&self, blobs: &[(String, Vec<u8>)]) -> StorageResult<()> {
|
||||
if blobs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(repo_bytes) = self.external_repo_bytes.as_ref() {
|
||||
if repo_bytes.is_read_only() {
|
||||
return repo_bytes.require_existing_blob_bytes_batch(blobs);
|
||||
}
|
||||
return repo_bytes.put_blob_bytes_batch(blobs);
|
||||
}
|
||||
if let Some(raw_store) = self.external_raw_store.as_ref() {
|
||||
return raw_store.put_blob_bytes_batch(blobs);
|
||||
}
|
||||
let cf = self.cf(CF_RAW_BLOB)?;
|
||||
let mut batch = WriteBatch::default();
|
||||
for (sha256_hex, bytes) in blobs {
|
||||
validate_sha256_hex("raw_blob.sha256_hex", sha256_hex)?;
|
||||
if bytes.is_empty() {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: "raw_blob",
|
||||
detail: "bytes must not be empty".to_string(),
|
||||
});
|
||||
}
|
||||
let key = raw_blob_key(sha256_hex);
|
||||
batch.put_cf(cf, key.as_bytes(), bytes.as_slice());
|
||||
}
|
||||
self.write_batch(batch)
|
||||
}
|
||||
|
||||
pub fn delete_raw_by_hash_entry(&self, sha256_hex: &str) -> StorageResult<()> {
|
||||
validate_sha256_hex("raw_by_hash.sha256_hex", sha256_hex)?;
|
||||
if let Some(raw_store) = self.external_raw_store.as_ref() {
|
||||
return raw_store.delete_raw_entry(sha256_hex);
|
||||
}
|
||||
let cf = self.cf(CF_RAW_BY_HASH)?;
|
||||
let key = raw_by_hash_key(sha256_hex);
|
||||
self.db
|
||||
.delete_cf(cf, key.as_bytes())
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_raw_by_hash_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>> {
|
||||
if let Some(raw_store) = self.external_raw_store.as_ref() {
|
||||
return raw_store.get_raw_entry(sha256_hex);
|
||||
}
|
||||
let cf = self.cf(CF_RAW_BY_HASH)?;
|
||||
let key = raw_by_hash_key(sha256_hex);
|
||||
let Some(bytes) = self
|
||||
.db
|
||||
.get_cf(cf, key.as_bytes())
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let entry = decode_cbor::<RawByHashEntry>(&bytes, "raw_by_hash")?;
|
||||
entry.validate_internal()?;
|
||||
Ok(Some(entry))
|
||||
}
|
||||
|
||||
pub fn get_raw_by_hash_entries_batch(
|
||||
&self,
|
||||
sha256_hexes: &[String],
|
||||
) -> StorageResult<Vec<Option<RawByHashEntry>>> {
|
||||
if sha256_hexes.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
if let Some(raw_store) = self.external_raw_store.as_ref() {
|
||||
return raw_store.get_raw_entries_batch(sha256_hexes);
|
||||
}
|
||||
|
||||
let cf = self.cf(CF_RAW_BY_HASH)?;
|
||||
let keys: Vec<String> = sha256_hexes
|
||||
.iter()
|
||||
.map(|hash| raw_by_hash_key(hash))
|
||||
.collect();
|
||||
self.db
|
||||
.multi_get_cf(keys.iter().map(|key| (cf, key.as_bytes())))
|
||||
.into_iter()
|
||||
.map(|res| {
|
||||
let maybe = res.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
match maybe {
|
||||
Some(bytes) => {
|
||||
let entry = decode_cbor::<RawByHashEntry>(&bytes, "raw_by_hash")?;
|
||||
entry.validate_internal()?;
|
||||
Ok(Some(entry))
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult<Option<Vec<u8>>> {
|
||||
if let Some(repo_bytes) = self.external_repo_bytes.as_ref() {
|
||||
return repo_bytes.get_blob_bytes(sha256_hex);
|
||||
}
|
||||
if let Some(raw_store) = self.external_raw_store.as_ref() {
|
||||
return raw_store.get_blob_bytes(sha256_hex);
|
||||
}
|
||||
validate_sha256_hex("raw_blob.sha256_hex", sha256_hex)?;
|
||||
let cf = self.cf(CF_RAW_BLOB)?;
|
||||
let key = raw_blob_key(sha256_hex);
|
||||
if let Some(bytes) = self
|
||||
.db
|
||||
.get_cf(cf, key.as_bytes())
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?
|
||||
{
|
||||
return Ok(Some(bytes));
|
||||
}
|
||||
self.get_raw_by_hash_entry(sha256_hex)
|
||||
.map(|entry| entry.map(|entry| entry.bytes))
|
||||
}
|
||||
|
||||
pub fn get_blob_bytes_batch(
|
||||
&self,
|
||||
sha256_hexes: &[String],
|
||||
) -> StorageResult<Vec<Option<Vec<u8>>>> {
|
||||
if sha256_hexes.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
if let Some(repo_bytes) = self.external_repo_bytes.as_ref() {
|
||||
return repo_bytes.get_blob_bytes_batch(sha256_hexes);
|
||||
}
|
||||
if let Some(raw_store) = self.external_raw_store.as_ref() {
|
||||
return raw_store.get_blob_bytes_batch(sha256_hexes);
|
||||
}
|
||||
|
||||
let cf = self.cf(CF_RAW_BLOB)?;
|
||||
let keys: Vec<String> = sha256_hexes
|
||||
.iter()
|
||||
.map(|hash| {
|
||||
validate_sha256_hex("raw_blob.sha256_hex", hash)?;
|
||||
Ok::<String, StorageError>(raw_blob_key(hash))
|
||||
})
|
||||
.collect::<Result<_, _>>()?;
|
||||
let blob_results: Vec<Option<Vec<u8>>> = self
|
||||
.db
|
||||
.multi_get_cf(keys.iter().map(|key| (cf, key.as_bytes())))
|
||||
.into_iter()
|
||||
.map(|res| res.map_err(|e| StorageError::RocksDb(e.to_string())))
|
||||
.collect::<Result<_, _>>()?;
|
||||
|
||||
let mut out = Vec::with_capacity(sha256_hexes.len());
|
||||
for (sha256_hex, maybe_blob) in sha256_hexes.iter().zip(blob_results.into_iter()) {
|
||||
if maybe_blob.is_some() {
|
||||
out.push(maybe_blob);
|
||||
} else {
|
||||
out.push(
|
||||
self.get_raw_by_hash_entry(sha256_hex)?
|
||||
.map(|entry| entry.bytes),
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
}
|
||||
108
src/repository/storage/store_transport_rrdp.rs
Normal file
108
src/repository/storage/store_transport_rrdp.rs
Normal file
@@ -0,0 +1,108 @@
|
||||
// RRDP source/session state and repository-view reads.
|
||||
|
||||
impl RocksStore {
|
||||
pub fn put_rrdp_source_record(&self, record: &RrdpSourceRecord) -> StorageResult<()> {
|
||||
record.validate_internal()?;
|
||||
let cf = self.cf(CF_RRDP_SOURCE)?;
|
||||
let key = rrdp_source_key(&record.notify_uri);
|
||||
let value = encode_cbor(record, "rrdp_source")?;
|
||||
self.db.put_cf(cf, key.as_bytes(), value).map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_rrdp_source_record(&self, notify_uri: &str) -> StorageResult<Option<RrdpSourceRecord>> {
|
||||
let cf = self.cf(CF_RRDP_SOURCE)?;
|
||||
let key = rrdp_source_key(notify_uri);
|
||||
let Some(bytes) = self.db.get_cf(cf, key.as_bytes()).map_err(|e| StorageError::RocksDb(e.to_string()))? else { return Ok(None) };
|
||||
let record = decode_cbor::<RrdpSourceRecord>(&bytes, "rrdp_source")?;
|
||||
record.validate_internal()?;
|
||||
Ok(Some(record))
|
||||
}
|
||||
|
||||
pub fn put_rrdp_source_member_record(&self, record: &RrdpSourceMemberRecord) -> StorageResult<()> {
|
||||
record.validate_internal()?;
|
||||
let cf = self.cf(CF_RRDP_SOURCE_MEMBER)?;
|
||||
let key = rrdp_source_member_key(&record.notify_uri, &record.rsync_uri);
|
||||
let value = encode_cbor(record, "rrdp_source_member")?;
|
||||
self.db.put_cf(cf, key.as_bytes(), value).map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_rrdp_source_member_record(&self, notify_uri: &str, rsync_uri: &str) -> StorageResult<Option<RrdpSourceMemberRecord>> {
|
||||
let cf = self.cf(CF_RRDP_SOURCE_MEMBER)?;
|
||||
let key = rrdp_source_member_key(notify_uri, rsync_uri);
|
||||
let Some(bytes) = self.db.get_cf(cf, key.as_bytes()).map_err(|e| StorageError::RocksDb(e.to_string()))? else { return Ok(None) };
|
||||
let record = decode_cbor::<RrdpSourceMemberRecord>(&bytes, "rrdp_source_member")?;
|
||||
record.validate_internal()?;
|
||||
Ok(Some(record))
|
||||
}
|
||||
|
||||
pub fn list_rrdp_source_member_records(&self, notify_uri: &str) -> StorageResult<Vec<RrdpSourceMemberRecord>> {
|
||||
let cf = self.cf(CF_RRDP_SOURCE_MEMBER)?;
|
||||
let prefix = rrdp_source_member_prefix(notify_uri);
|
||||
let mode = IteratorMode::From(prefix.as_bytes(), Direction::Forward);
|
||||
self.db.iterator_cf(cf, mode).take_while(|res| match res { Ok((key, _)) => key.starts_with(prefix.as_bytes()), Err(_) => false }).map(|res| {
|
||||
let (_key, value) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
let record = decode_cbor::<RrdpSourceMemberRecord>(&value, "rrdp_source_member")?;
|
||||
record.validate_internal()?;
|
||||
Ok(record)
|
||||
}).collect()
|
||||
}
|
||||
|
||||
pub fn list_current_rrdp_source_members(&self, notify_uri: &str) -> StorageResult<Vec<RrdpSourceMemberRecord>> {
|
||||
let mut records = self.list_rrdp_source_member_records(notify_uri)?;
|
||||
records.retain(|record| record.present);
|
||||
records.sort_by(|a, b| a.rsync_uri.cmp(&b.rsync_uri));
|
||||
Ok(records)
|
||||
}
|
||||
|
||||
pub fn is_current_rrdp_source_member(&self, notify_uri: &str, rsync_uri: &str) -> StorageResult<bool> {
|
||||
Ok(matches!(self.get_rrdp_source_member_record(notify_uri, rsync_uri)?, Some(record) if record.present))
|
||||
}
|
||||
|
||||
pub fn load_current_object_bytes_by_uri(&self, rsync_uri: &str) -> StorageResult<Option<Vec<u8>>> {
|
||||
Ok(self.load_current_object_with_hash_by_uri(rsync_uri)?.map(|object| object.bytes))
|
||||
}
|
||||
|
||||
pub fn load_current_object_with_hash_by_uri(&self, rsync_uri: &str) -> StorageResult<Option<CurrentObjectWithHash>> {
|
||||
let Some(view) = self.get_repository_view_entry(rsync_uri)? else { return Ok(None) };
|
||||
match view.state {
|
||||
RepositoryViewState::Withdrawn => Ok(None),
|
||||
RepositoryViewState::Present | RepositoryViewState::Replaced => {
|
||||
let hash = view.current_hash.as_deref().ok_or(StorageError::InvalidData { entity: "repository_view", detail: format!("current_hash missing for current object URI: {rsync_uri}") })?;
|
||||
let bytes = self.get_blob_bytes(hash)?.ok_or(StorageError::InvalidData { entity: "repository_view", detail: format!("blob bytes missing for current object URI: {rsync_uri} (hash={hash})") })?;
|
||||
Ok(Some(CurrentObjectWithHash { current_hash_hex: hash.to_ascii_lowercase(), current_hash: decode_sha256_hex_32("repository_view.current_hash", hash)?, bytes }))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn put_rrdp_uri_owner_record(&self, record: &RrdpUriOwnerRecord) -> StorageResult<()> {
|
||||
record.validate_internal()?;
|
||||
let cf = self.cf(CF_RRDP_URI_OWNER)?;
|
||||
let key = rrdp_uri_owner_key(&record.rsync_uri);
|
||||
let value = encode_cbor(record, "rrdp_uri_owner")?;
|
||||
self.db.put_cf(cf, key.as_bytes(), value).map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_rrdp_uri_owner_record(&self, rsync_uri: &str) -> StorageResult<Option<RrdpUriOwnerRecord>> {
|
||||
let cf = self.cf(CF_RRDP_URI_OWNER)?;
|
||||
let key = rrdp_uri_owner_key(rsync_uri);
|
||||
let Some(bytes) = self.db.get_cf(cf, key.as_bytes()).map_err(|e| StorageError::RocksDb(e.to_string()))? else { return Ok(None) };
|
||||
let record = decode_cbor::<RrdpUriOwnerRecord>(&bytes, "rrdp_uri_owner")?;
|
||||
record.validate_internal()?;
|
||||
Ok(Some(record))
|
||||
}
|
||||
|
||||
pub fn delete_rrdp_uri_owner_record(&self, rsync_uri: &str) -> StorageResult<()> {
|
||||
let cf = self.cf(CF_RRDP_URI_OWNER)?;
|
||||
let key = rrdp_uri_owner_key(rsync_uri);
|
||||
self.db.delete_cf(cf, key.as_bytes()).map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn write_batch(&self, batch: WriteBatch) -> StorageResult<()> {
|
||||
self.db.write(batch).map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
52
src/repository/storage/tests.rs
Normal file
52
src/repository/storage/tests.rs
Normal file
@@ -0,0 +1,52 @@
|
||||
// Storage tests are grouped by the public storage surface they protect.
|
||||
use super::*;
|
||||
use rocksdb::{DB, Options};
|
||||
use sha2::Digest;
|
||||
|
||||
fn sha256_hex(bytes: &[u8]) -> String {
|
||||
hex::encode(sha2::Sha256::digest(bytes))
|
||||
}
|
||||
|
||||
fn pack_time(seconds: i64) -> PackTime {
|
||||
PackTime::from_utc_offset_datetime(
|
||||
time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(seconds),
|
||||
)
|
||||
}
|
||||
|
||||
fn sample_repository_view_entry(uri: &str, bytes: &[u8]) -> RepositoryViewEntry {
|
||||
RepositoryViewEntry {
|
||||
rsync_uri: uri.to_string(),
|
||||
current_hash: Some(sha256_hex(bytes)),
|
||||
repository_source: Some("fixture".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_raw_by_hash_entry(bytes: Vec<u8>) -> RawByHashEntry {
|
||||
RawByHashEntry::from_bytes(sha256_hex(&bytes), bytes)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opening_an_unsupported_state_schema_fails_without_rewriting() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let mut options = Options::default();
|
||||
options.create_if_missing(true);
|
||||
options.create_missing_column_families(true);
|
||||
let legacy = DB::open_cf(&options, td.path(), ["legacy_private_state"])
|
||||
.expect("create legacy state database");
|
||||
drop(legacy);
|
||||
|
||||
match RocksStore::open(td.path()) {
|
||||
Err(StorageError::InvalidData {
|
||||
entity: "state_schema",
|
||||
detail,
|
||||
}) => assert!(detail.contains("legacy_private_state")),
|
||||
Err(error) => panic!("unexpected schema error: {error}"),
|
||||
Ok(_) => panic!("legacy schema must be rejected"),
|
||||
}
|
||||
}
|
||||
|
||||
include!("tests_parts/repository.rs");
|
||||
include!("tests_parts/rrdp.rs");
|
||||
include!("tests_parts/object_loading.rs");
|
||||
234
src/repository/storage/tests_parts/object_loading.rs
Normal file
234
src/repository/storage/tests_parts/object_loading.rs
Normal file
@@ -0,0 +1,234 @@
|
||||
// Storage test group: object loading.
|
||||
|
||||
#[test]
|
||||
fn load_current_object_with_hash_by_uri_uses_internal_blob_cf_without_raw_entry() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
let rsync_uri = "rsync://example.test/repo/blob-only.roa";
|
||||
let bytes = b"blob-only-current-object".to_vec();
|
||||
let hash = sha256_hex(&bytes);
|
||||
|
||||
store
|
||||
.put_blob_bytes_batch(&[(hash.clone(), bytes.clone())])
|
||||
.expect("put blob bytes");
|
||||
store
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
rsync_uri: rsync_uri.to_string(),
|
||||
current_hash: Some(hash.clone()),
|
||||
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
})
|
||||
.expect("put view");
|
||||
|
||||
let got = store
|
||||
.load_current_object_with_hash_by_uri(rsync_uri)
|
||||
.expect("load current object")
|
||||
.expect("current object exists");
|
||||
assert_eq!(got.current_hash_hex, hash);
|
||||
assert_eq!(got.current_hash, compute_sha256_32(&bytes));
|
||||
assert_eq!(got.bytes, bytes);
|
||||
assert!(
|
||||
store
|
||||
.get_raw_by_hash_entry(&got.current_hash_hex)
|
||||
.expect("get raw entry")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_current_object_bytes_by_uri_uses_internal_blob_cf_without_raw_entry() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
let rsync_uri = "rsync://example.test/repo/blob-only-bytes.roa";
|
||||
let bytes = b"blob-only-current-object-bytes".to_vec();
|
||||
let hash = sha256_hex(&bytes);
|
||||
|
||||
store
|
||||
.put_blob_bytes_batch(&[(hash, bytes.clone())])
|
||||
.expect("put blob bytes");
|
||||
store
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
rsync_uri: rsync_uri.to_string(),
|
||||
current_hash: Some(sha256_hex(&bytes)),
|
||||
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
})
|
||||
.expect("put view");
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.load_current_object_bytes_by_uri(rsync_uri)
|
||||
.expect("load current object bytes"),
|
||||
Some(bytes)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pack_file_can_lazy_load_bytes_from_external_raw_store() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let raw_store = std::sync::Arc::new(
|
||||
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store"),
|
||||
);
|
||||
let bytes = b"lazy-pack-file".to_vec();
|
||||
let sha256_hex = sha256_hex(&bytes);
|
||||
raw_store
|
||||
.put_raw_entry(&RawByHashEntry::from_bytes(
|
||||
sha256_hex.clone(),
|
||||
bytes.clone(),
|
||||
))
|
||||
.expect("put raw entry");
|
||||
|
||||
let file = PackFile::from_lazy_external_raw_store(
|
||||
"rsync://example.test/repo/a.roa",
|
||||
sha256_hex,
|
||||
compute_sha256_32(&bytes),
|
||||
raw_store,
|
||||
);
|
||||
|
||||
assert_eq!(file.bytes().expect("lazy bytes"), bytes.as_slice());
|
||||
assert_eq!(file.bytes_cloned().expect("cloned bytes"), bytes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pack_file_can_lazy_load_bytes_from_external_repo_bytes_store() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let repo_bytes_store = std::sync::Arc::new(
|
||||
ExternalRepoBytesDb::open(td.path().join("repo-bytes.db")).expect("open repo bytes"),
|
||||
);
|
||||
let bytes = b"repo-object-pack-file".to_vec();
|
||||
let sha256_hex = sha256_hex(&bytes);
|
||||
repo_bytes_store
|
||||
.put_blob_bytes_batch(&[(sha256_hex.clone(), bytes.clone())])
|
||||
.expect("put repo bytes");
|
||||
|
||||
let file = PackFile::from_lazy_repo_bytes(
|
||||
"rsync://example.test/repo/a.roa",
|
||||
sha256_hex,
|
||||
compute_sha256_32(&bytes),
|
||||
repo_bytes_store,
|
||||
);
|
||||
|
||||
assert_eq!(file.bytes().expect("lazy repo bytes"), bytes.as_slice());
|
||||
assert_eq!(file.bytes_cloned().expect("cloned repo bytes"), bytes);
|
||||
assert_eq!(file.compute_sha256().expect("compute sha256"), file.sha256);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_only_checkpoint_isolated_from_source_work_db() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let source_path = td.path().join("source-work-db");
|
||||
let checkpoint_path = td.path().join("checkpoint-work-db");
|
||||
let uri = "rsync://example.test/repo/a.roa";
|
||||
let bytes = b"checkpoint-object";
|
||||
let hash = sha256_hex(bytes);
|
||||
{
|
||||
let source = RocksStore::open(&source_path).expect("open source");
|
||||
source
|
||||
.put_blob_bytes_batch(&[(hash.clone(), bytes.to_vec())])
|
||||
.expect("put blob");
|
||||
source
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
rsync_uri: uri.to_string(),
|
||||
current_hash: Some(hash),
|
||||
repository_source: Some("fixture".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
})
|
||||
.expect("put view");
|
||||
}
|
||||
|
||||
RocksStore::create_read_only_checkpoint(&source_path, &checkpoint_path)
|
||||
.expect("create checkpoint");
|
||||
let checkpoint = RocksStore::open(&checkpoint_path).expect("open checkpoint");
|
||||
checkpoint
|
||||
.delete_repository_view_entry(uri)
|
||||
.expect("delete checkpoint entry");
|
||||
drop(checkpoint);
|
||||
|
||||
let source = RocksStore::open(&source_path).expect("reopen source");
|
||||
assert!(
|
||||
source
|
||||
.get_repository_view_entry(uri)
|
||||
.expect("get source entry")
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_only_external_repo_bytes_rejects_writes() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let path = td.path().join("repo-bytes.db");
|
||||
let hash = sha256_hex(b"repo-object");
|
||||
{
|
||||
let writable = ExternalRepoBytesDb::open(&path).expect("open writable repo bytes");
|
||||
writable
|
||||
.put_blob_bytes_batch(&[(hash.clone(), b"repo-object".to_vec())])
|
||||
.expect("seed repo bytes");
|
||||
}
|
||||
let read_only = ExternalRepoBytesDb::open_read_only(&path).expect("open read-only repo bytes");
|
||||
assert_eq!(
|
||||
read_only
|
||||
.get_blob_bytes(&hash)
|
||||
.expect("read repo bytes")
|
||||
.expect("blob"),
|
||||
b"repo-object"
|
||||
);
|
||||
assert!(
|
||||
read_only
|
||||
.put_blob_bytes_batch(&[(hash, b"repo-object".to_vec())])
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_only_store_accepts_only_idempotent_existing_blob_apply() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let work_db_path = td.path().join("work-db");
|
||||
let repo_bytes_path = td.path().join("repo-bytes.db");
|
||||
let bytes = b"repo-object".to_vec();
|
||||
let hash = sha256_hex(&bytes);
|
||||
{
|
||||
let repo_bytes = ExternalRepoBytesDb::open(&repo_bytes_path).expect("open repo bytes");
|
||||
repo_bytes
|
||||
.put_blob_bytes_batch(&[(hash.clone(), bytes.clone())])
|
||||
.expect("seed repo bytes");
|
||||
}
|
||||
let store =
|
||||
RocksStore::open_with_external_repo_bytes_read_only(&work_db_path, &repo_bytes_path)
|
||||
.expect("open read-only store");
|
||||
store
|
||||
.put_blob_bytes_batch(&[(hash.clone(), bytes)])
|
||||
.expect("idempotent apply");
|
||||
assert!(
|
||||
store
|
||||
.put_blob_bytes_batch(&[(hash, b"different".to_vec())])
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repository_blob_verification_detects_missing_external_blob() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let repo_bytes_path = td.path().join("repo-bytes.db");
|
||||
ExternalRepoBytesDb::open(&repo_bytes_path).expect("create repo bytes");
|
||||
let store = RocksStore::open_with_external_repo_bytes_read_only(
|
||||
&td.path().join("work-db"),
|
||||
&repo_bytes_path,
|
||||
)
|
||||
.expect("open work db");
|
||||
store
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
rsync_uri: "rsync://example.test/repo/missing.roa".to_string(),
|
||||
current_hash: Some(sha256_hex(b"missing")),
|
||||
repository_source: Some("fixture".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
})
|
||||
.expect("put view");
|
||||
let error = store
|
||||
.verify_current_repository_blobs(16)
|
||||
.expect_err("missing blob must fail");
|
||||
assert!(error.to_string().contains("missing.roa"), "{error}");
|
||||
}
|
||||
391
src/repository/storage/tests_parts/repository.rs
Normal file
391
src/repository/storage/tests_parts/repository.rs
Normal file
@@ -0,0 +1,391 @@
|
||||
// Storage test group: repository.
|
||||
|
||||
fn sample_rrdp_source_member_record(
|
||||
notify_uri: &str,
|
||||
rsync_uri: &str,
|
||||
serial: u64,
|
||||
) -> RrdpSourceMemberRecord {
|
||||
RrdpSourceMemberRecord {
|
||||
notify_uri: notify_uri.to_string(),
|
||||
rsync_uri: rsync_uri.to_string(),
|
||||
current_hash: Some(sha256_hex(rsync_uri.as_bytes())),
|
||||
object_type: Some("cer".to_string()),
|
||||
present: true,
|
||||
last_confirmed_session_id: "session-1".to_string(),
|
||||
last_confirmed_serial: serial,
|
||||
last_changed_at: pack_time(serial as i64),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repository_view_and_raw_by_hash_roundtrip() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
|
||||
let entry1 = sample_repository_view_entry("rsync://example.test/repo/a.cer", b"object-a");
|
||||
let entry2 = sample_repository_view_entry("rsync://example.test/repo/sub/b.roa", b"object-b");
|
||||
store
|
||||
.put_repository_view_entry(&entry1)
|
||||
.expect("put repository view entry1");
|
||||
store
|
||||
.put_repository_view_entry(&entry2)
|
||||
.expect("put repository view entry2");
|
||||
|
||||
let got1 = store
|
||||
.get_repository_view_entry(&entry1.rsync_uri)
|
||||
.expect("get repository view entry1")
|
||||
.expect("entry1 exists");
|
||||
assert_eq!(got1, entry1);
|
||||
|
||||
let got_prefix = store
|
||||
.list_repository_view_entries_with_prefix("rsync://example.test/repo/sub/")
|
||||
.expect("list repository view prefix");
|
||||
assert_eq!(got_prefix, vec![entry2.clone()]);
|
||||
|
||||
store
|
||||
.delete_repository_view_entry(&entry1.rsync_uri)
|
||||
.expect("delete repository view entry1");
|
||||
assert!(
|
||||
store
|
||||
.get_repository_view_entry(&entry1.rsync_uri)
|
||||
.expect("get deleted repository view entry1")
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let raw = sample_raw_by_hash_entry(b"raw-der-object".to_vec());
|
||||
store
|
||||
.put_raw_by_hash_entry(&raw)
|
||||
.expect("put raw_by_hash entry");
|
||||
let got_raw = store
|
||||
.get_raw_by_hash_entry(&raw.sha256_hex)
|
||||
.expect("get raw_by_hash entry")
|
||||
.expect("raw entry exists");
|
||||
assert_eq!(got_raw, raw);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_by_hash_routes_to_external_raw_store_when_configured() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let main_db = td.path().join("main-db");
|
||||
let raw_db = td.path().join("raw-store.db");
|
||||
|
||||
let raw = sample_raw_by_hash_entry(b"external-raw".to_vec());
|
||||
{
|
||||
let store =
|
||||
RocksStore::open_with_external_raw_store(&main_db, &raw_db).expect("open store");
|
||||
store.put_raw_by_hash_entry(&raw).expect("put external raw");
|
||||
|
||||
let got = store
|
||||
.get_raw_by_hash_entry(&raw.sha256_hex)
|
||||
.expect("get external raw")
|
||||
.expect("raw exists");
|
||||
assert_eq!(got, raw);
|
||||
}
|
||||
|
||||
let main_store = RocksStore::open(&main_db).expect("open main only");
|
||||
assert!(
|
||||
main_store
|
||||
.get_raw_by_hash_entry(&raw.sha256_hex)
|
||||
.expect("read main store")
|
||||
.is_none(),
|
||||
"main db should not contain raw entry when external raw store is configured"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_blob_bytes_batch_uses_internal_blob_cf_without_raw_entry() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
let bytes = b"internal-blob-only".to_vec();
|
||||
let hash = sha256_hex(&bytes);
|
||||
|
||||
store
|
||||
.put_blob_bytes_batch(&[(hash.clone(), bytes.clone())])
|
||||
.expect("put blob bytes");
|
||||
|
||||
assert_eq!(
|
||||
store.get_blob_bytes(&hash).expect("get blob bytes"),
|
||||
Some(bytes.clone())
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_raw_by_hash_entry(&hash)
|
||||
.expect("get raw entry")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_blob_bytes_batch_routes_to_external_raw_store_without_raw_entry() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open_with_external_raw_store(
|
||||
&td.path().join("main-db"),
|
||||
&td.path().join("raw-store.db"),
|
||||
)
|
||||
.expect("open store");
|
||||
let bytes = b"external-blob-only".to_vec();
|
||||
let hash = sha256_hex(&bytes);
|
||||
|
||||
store
|
||||
.put_blob_bytes_batch(&[(hash.clone(), bytes.clone())])
|
||||
.expect("put external blob bytes");
|
||||
|
||||
assert_eq!(store.get_blob_bytes(&hash).unwrap(), Some(bytes));
|
||||
assert!(store.get_raw_by_hash_entry(&hash).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repo_bytes_db_is_physically_separate_from_external_raw_store() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let main_db = td.path().join("main-db");
|
||||
let raw_db = td.path().join("raw-store.db");
|
||||
let repo_bytes_db = td.path().join("repo-bytes.db");
|
||||
let store =
|
||||
RocksStore::open_with_external_stores(&main_db, Some(&raw_db), Some(&repo_bytes_db))
|
||||
.expect("open store");
|
||||
let repo_bytes = b"repo-object".to_vec();
|
||||
let repo_hash = sha256_hex(&repo_bytes);
|
||||
let raw = sample_raw_by_hash_entry(b"raw-evidence".to_vec());
|
||||
|
||||
store
|
||||
.put_blob_bytes_batch(&[(repo_hash.clone(), repo_bytes.clone())])
|
||||
.expect("put repo bytes");
|
||||
store.put_raw_by_hash_entry(&raw).expect("put raw evidence");
|
||||
|
||||
assert_eq!(store.get_blob_bytes(&repo_hash).unwrap(), Some(repo_bytes));
|
||||
assert_eq!(
|
||||
store.get_raw_by_hash_entry(&raw.sha256_hex).unwrap(),
|
||||
Some(raw.clone())
|
||||
);
|
||||
drop(store);
|
||||
|
||||
let raw_only = RocksStore::open_with_external_raw_store(&td.path().join("raw-reader"), &raw_db)
|
||||
.expect("open raw only");
|
||||
assert!(
|
||||
raw_only.get_blob_bytes(&repo_hash).unwrap().is_none(),
|
||||
"repo object bytes must not be written into raw-store.db"
|
||||
);
|
||||
|
||||
let repo_only =
|
||||
RocksStore::open_with_external_repo_bytes(&td.path().join("repo-reader"), &repo_bytes_db)
|
||||
.expect("open repo bytes only");
|
||||
assert_eq!(
|
||||
repo_only.get_blob_bytes(&repo_hash).unwrap(),
|
||||
Some(b"repo-object".to_vec())
|
||||
);
|
||||
assert!(
|
||||
repo_only.get_blob_bytes(&raw.sha256_hex).unwrap().is_none(),
|
||||
"raw evidence bytes must not be written into repo-bytes.db"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memory_snapshot_includes_work_db_and_external_stores() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open_with_external_stores(
|
||||
&td.path().join("main-db"),
|
||||
Some(&td.path().join("raw-store.db")),
|
||||
Some(&td.path().join("repo-bytes.db")),
|
||||
)
|
||||
.expect("open store");
|
||||
|
||||
let snapshot = store.memory_snapshot();
|
||||
let labels: Vec<&str> = snapshot
|
||||
.databases
|
||||
.iter()
|
||||
.map(|db| db.label.as_str())
|
||||
.collect();
|
||||
assert_eq!(labels, vec!["work-db", "raw-store.db", "repo-bytes.db"]);
|
||||
assert!(
|
||||
snapshot.databases[0]
|
||||
.column_families
|
||||
.iter()
|
||||
.any(|cf| cf.name == CF_REPOSITORY_VIEW)
|
||||
);
|
||||
serde_json::to_value(&snapshot).expect("serialize memory snapshot");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_blob_bytes_batch_accepts_empty_batch_with_external_raw_store() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open_with_external_raw_store(
|
||||
&td.path().join("main-db"),
|
||||
&td.path().join("raw-store.db"),
|
||||
)
|
||||
.expect("open store");
|
||||
|
||||
store
|
||||
.put_blob_bytes_batch(&[])
|
||||
.expect("empty external blob batch should be a no-op");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_blob_bytes_internal_falls_back_to_raw_entry_when_blob_missing() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
let raw = sample_raw_by_hash_entry(b"raw-fallback".to_vec());
|
||||
|
||||
store.put_raw_by_hash_entry(&raw).expect("put raw entry");
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.get_blob_bytes(&raw.sha256_hex)
|
||||
.expect("get blob bytes via raw fallback"),
|
||||
Some(raw.bytes.clone())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_blob_bytes_batch_internal_prefers_blob_cf_and_falls_back_to_raw_entry() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
|
||||
let blob_bytes = b"blob-cf-object".to_vec();
|
||||
let blob_hash = sha256_hex(&blob_bytes);
|
||||
store
|
||||
.put_blob_bytes_batch(&[(blob_hash.clone(), blob_bytes.clone())])
|
||||
.expect("put blob bytes");
|
||||
|
||||
let raw = sample_raw_by_hash_entry(b"raw-fallback-batch".to_vec());
|
||||
store.put_raw_by_hash_entry(&raw).expect("put raw fallback");
|
||||
|
||||
let batch = store
|
||||
.get_blob_bytes_batch(&[blob_hash.clone(), raw.sha256_hex.clone(), "00".repeat(32)])
|
||||
.expect("get blob bytes batch");
|
||||
assert_eq!(batch, vec![Some(blob_bytes), Some(raw.bytes.clone()), None]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_blob_bytes_batch_routes_to_external_raw_store_without_raw_entry() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open_with_external_raw_store(
|
||||
&td.path().join("main-db"),
|
||||
&td.path().join("raw-store.db"),
|
||||
)
|
||||
.expect("open store");
|
||||
let bytes = b"external-batch-blob".to_vec();
|
||||
let hash = sha256_hex(&bytes);
|
||||
|
||||
store
|
||||
.put_blob_bytes_batch(&[(hash.clone(), bytes.clone())])
|
||||
.expect("put external blob bytes");
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.get_blob_bytes_batch(&[hash, "00".repeat(32)])
|
||||
.expect("get external blob batch"),
|
||||
vec![Some(bytes), None]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_blob_bytes_rejects_invalid_hash_for_internal_store() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
|
||||
let err = store
|
||||
.get_blob_bytes("not-a-valid-hash")
|
||||
.expect_err("invalid hash must fail");
|
||||
assert!(matches!(err, StorageError::InvalidData { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_blob_bytes_batch_rejects_invalid_hash_for_internal_store() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
|
||||
let err = store
|
||||
.get_blob_bytes_batch(&["not-a-valid-hash".to_string()])
|
||||
.expect_err("invalid hash must fail");
|
||||
assert!(matches!(err, StorageError::InvalidData { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_blob_bytes_batch_returns_empty_for_empty_request_internal() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
|
||||
assert!(
|
||||
store
|
||||
.get_blob_bytes_batch(&[])
|
||||
.expect("empty blob batch request")
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_blob_bytes_batch_accepts_empty_batch() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
|
||||
store
|
||||
.put_blob_bytes_batch(&[])
|
||||
.expect("empty blob batch should be a no-op");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_blob_bytes_batch_rejects_empty_bytes() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
|
||||
let err = store
|
||||
.put_blob_bytes_batch(&[(sha256_hex(b"valid"), Vec::new())])
|
||||
.expect_err("empty bytes must fail");
|
||||
assert!(matches!(err, StorageError::InvalidData { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_raw_by_hash_entry_internal_preserves_blob_bytes() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
let bytes = b"blob-persists-after-raw-delete".to_vec();
|
||||
let hash = sha256_hex(&bytes);
|
||||
let raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone());
|
||||
|
||||
store
|
||||
.put_blob_bytes_batch(&[(hash.clone(), bytes.clone())])
|
||||
.expect("put blob bytes");
|
||||
store.put_raw_by_hash_entry(&raw).expect("put raw entry");
|
||||
|
||||
store
|
||||
.delete_raw_by_hash_entry(&hash)
|
||||
.expect("delete raw entry only");
|
||||
|
||||
assert!(store.get_raw_by_hash_entry(&hash).unwrap().is_none());
|
||||
assert_eq!(store.get_blob_bytes(&hash).unwrap(), Some(bytes));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_raw_by_hash_entry_rejects_invalid_hash() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
|
||||
let err = store
|
||||
.delete_raw_by_hash_entry("not-a-valid-hash")
|
||||
.expect_err("invalid hash must fail");
|
||||
assert!(matches!(err, StorageError::InvalidData { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_raw_by_hash_entry_routes_to_external_raw_store() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open_with_external_raw_store(
|
||||
&td.path().join("main-db"),
|
||||
&td.path().join("raw-store.db"),
|
||||
)
|
||||
.expect("open store");
|
||||
let raw = sample_raw_by_hash_entry(b"external-delete".to_vec());
|
||||
|
||||
store.put_raw_by_hash_entry(&raw).expect("put raw entry");
|
||||
store
|
||||
.delete_raw_by_hash_entry(&raw.sha256_hex)
|
||||
.expect("delete external raw entry");
|
||||
|
||||
assert!(
|
||||
store
|
||||
.get_raw_by_hash_entry(&raw.sha256_hex)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(store.get_blob_bytes(&raw.sha256_hex).unwrap().is_none());
|
||||
}
|
||||
262
src/repository/storage/tests_parts/rrdp.rs
Normal file
262
src/repository/storage/tests_parts/rrdp.rs
Normal file
@@ -0,0 +1,262 @@
|
||||
// Storage test group: rrdp.
|
||||
|
||||
#[test]
|
||||
fn projection_batch_roundtrip_writes_repository_view_member_and_owner_records() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(dir.path()).expect("open store");
|
||||
|
||||
let view = RepositoryViewEntry {
|
||||
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
|
||||
current_hash: Some(hex::encode([1u8; 32])),
|
||||
repository_source: Some("https://example.test/notify.xml".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
};
|
||||
let member = RrdpSourceMemberRecord {
|
||||
notify_uri: "https://example.test/notify.xml".to_string(),
|
||||
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
|
||||
current_hash: Some(hex::encode([1u8; 32])),
|
||||
object_type: Some("roa".to_string()),
|
||||
present: true,
|
||||
last_confirmed_session_id: "session-1".to_string(),
|
||||
last_confirmed_serial: 7,
|
||||
last_changed_at: pack_time(1),
|
||||
};
|
||||
let owner = RrdpUriOwnerRecord {
|
||||
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
|
||||
notify_uri: "https://example.test/notify.xml".to_string(),
|
||||
current_hash: Some(hex::encode([1u8; 32])),
|
||||
last_confirmed_session_id: "session-1".to_string(),
|
||||
last_confirmed_serial: 7,
|
||||
last_changed_at: pack_time(1),
|
||||
owner_state: RrdpUriOwnerState::Active,
|
||||
};
|
||||
|
||||
store
|
||||
.put_projection_batch(
|
||||
std::slice::from_ref(&view),
|
||||
std::slice::from_ref(&member),
|
||||
std::slice::from_ref(&owner),
|
||||
)
|
||||
.expect("write projection batch");
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.get_repository_view_entry(&view.rsync_uri)
|
||||
.expect("get view")
|
||||
.expect("present view"),
|
||||
view
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.get_rrdp_source_member_record(&member.notify_uri, &member.rsync_uri)
|
||||
.expect("get member")
|
||||
.expect("present member"),
|
||||
member
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.get_rrdp_uri_owner_record(&owner.rsync_uri)
|
||||
.expect("get owner")
|
||||
.expect("present owner"),
|
||||
owner
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_rrdp_source_member_helpers_filter_present_records() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
|
||||
let notify_uri = "https://rrdp.example.test/notification.xml";
|
||||
let mut present_a =
|
||||
sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/a.cer", 1);
|
||||
let mut withdrawn_b =
|
||||
sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/b.roa", 2);
|
||||
withdrawn_b.present = false;
|
||||
let present_c =
|
||||
sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/c.crl", 3);
|
||||
let other_source = sample_rrdp_source_member_record(
|
||||
"https://other.example.test/notification.xml",
|
||||
"rsync://other.example.test/repo/x.cer",
|
||||
4,
|
||||
);
|
||||
present_a.last_confirmed_serial = 10;
|
||||
|
||||
store
|
||||
.put_rrdp_source_member_record(&present_a)
|
||||
.expect("put present a");
|
||||
store
|
||||
.put_rrdp_source_member_record(&withdrawn_b)
|
||||
.expect("put withdrawn b");
|
||||
store
|
||||
.put_rrdp_source_member_record(&present_c)
|
||||
.expect("put present c");
|
||||
store
|
||||
.put_rrdp_source_member_record(&other_source)
|
||||
.expect("put other source");
|
||||
|
||||
let members = store
|
||||
.list_current_rrdp_source_members(notify_uri)
|
||||
.expect("list current members");
|
||||
assert_eq!(
|
||||
members
|
||||
.iter()
|
||||
.map(|record| record.rsync_uri.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
"rsync://example.test/repo/a.cer",
|
||||
"rsync://example.test/repo/c.crl",
|
||||
]
|
||||
);
|
||||
|
||||
assert!(
|
||||
store
|
||||
.is_current_rrdp_source_member(notify_uri, &present_a.rsync_uri)
|
||||
.expect("current a")
|
||||
);
|
||||
assert!(
|
||||
!store
|
||||
.is_current_rrdp_source_member(notify_uri, &withdrawn_b.rsync_uri)
|
||||
.expect("withdrawn b")
|
||||
);
|
||||
assert!(
|
||||
!store
|
||||
.is_current_rrdp_source_member(notify_uri, &other_source.rsync_uri)
|
||||
.expect("other source")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_current_object_bytes_by_uri_uses_repository_view_and_raw_by_hash() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
|
||||
let present_bytes = b"present-object".to_vec();
|
||||
let present_hash = sha256_hex(&present_bytes);
|
||||
let mut present_raw = RawByHashEntry::from_bytes(present_hash.clone(), present_bytes.clone());
|
||||
present_raw
|
||||
.origin_uris
|
||||
.push("rsync://example.test/repo/present.roa".to_string());
|
||||
present_raw.object_type = Some("roa".to_string());
|
||||
store
|
||||
.put_raw_by_hash_entry(&present_raw)
|
||||
.expect("put present raw");
|
||||
store
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
rsync_uri: "rsync://example.test/repo/present.roa".to_string(),
|
||||
current_hash: Some(present_hash),
|
||||
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
})
|
||||
.expect("put present view");
|
||||
|
||||
let replaced_bytes = b"replaced-object".to_vec();
|
||||
let replaced_hash = sha256_hex(&replaced_bytes);
|
||||
let mut replaced_raw =
|
||||
RawByHashEntry::from_bytes(replaced_hash.clone(), replaced_bytes.clone());
|
||||
replaced_raw
|
||||
.origin_uris
|
||||
.push("rsync://example.test/repo/replaced.cer".to_string());
|
||||
replaced_raw.object_type = Some("cer".to_string());
|
||||
store
|
||||
.put_raw_by_hash_entry(&replaced_raw)
|
||||
.expect("put replaced raw");
|
||||
store
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
rsync_uri: "rsync://example.test/repo/replaced.cer".to_string(),
|
||||
current_hash: Some(replaced_hash),
|
||||
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
|
||||
object_type: Some("cer".to_string()),
|
||||
state: RepositoryViewState::Replaced,
|
||||
})
|
||||
.expect("put replaced view");
|
||||
|
||||
store
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
rsync_uri: "rsync://example.test/repo/withdrawn.crl".to_string(),
|
||||
current_hash: Some(sha256_hex(b"withdrawn")),
|
||||
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
|
||||
object_type: Some("crl".to_string()),
|
||||
state: RepositoryViewState::Withdrawn,
|
||||
})
|
||||
.expect("put withdrawn view");
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.load_current_object_bytes_by_uri("rsync://example.test/repo/present.roa")
|
||||
.expect("load present"),
|
||||
Some(present_bytes)
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.load_current_object_bytes_by_uri("rsync://example.test/repo/replaced.cer")
|
||||
.expect("load replaced"),
|
||||
Some(replaced_bytes)
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.load_current_object_bytes_by_uri("rsync://example.test/repo/withdrawn.crl")
|
||||
.expect("load withdrawn"),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.load_current_object_bytes_by_uri("rsync://example.test/repo/missing.roa")
|
||||
.expect("load missing"),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_current_object_bytes_by_uri_errors_when_raw_by_hash_is_missing() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
let rsync_uri = "rsync://example.test/repo/missing.cer";
|
||||
|
||||
store
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
rsync_uri: rsync_uri.to_string(),
|
||||
current_hash: Some(hex::encode([0x11; 32])),
|
||||
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
|
||||
object_type: Some("cer".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
})
|
||||
.expect("put view");
|
||||
let err = store
|
||||
.load_current_object_bytes_by_uri(rsync_uri)
|
||||
.expect_err("missing raw_by_hash should error");
|
||||
assert!(matches!(err, StorageError::InvalidData { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_current_object_with_hash_by_uri_returns_hash_and_bytes() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
let rsync_uri = "rsync://example.test/repo/present.roa";
|
||||
let bytes = b"present-object".to_vec();
|
||||
let hash = sha256_hex(&bytes);
|
||||
|
||||
let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone());
|
||||
raw.origin_uris.push(rsync_uri.to_string());
|
||||
raw.object_type = Some("roa".to_string());
|
||||
store.put_raw_by_hash_entry(&raw).expect("put raw");
|
||||
store
|
||||
.put_repository_view_entry(&RepositoryViewEntry {
|
||||
rsync_uri: rsync_uri.to_string(),
|
||||
current_hash: Some(hash.clone()),
|
||||
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
})
|
||||
.expect("put view");
|
||||
|
||||
let got = store
|
||||
.load_current_object_with_hash_by_uri(rsync_uri)
|
||||
.expect("load current object")
|
||||
.expect("current object exists");
|
||||
assert_eq!(got.current_hash_hex, hash);
|
||||
assert_eq!(got.current_hash, compute_sha256_32(&bytes));
|
||||
assert_eq!(got.bytes, bytes);
|
||||
}
|
||||
32
src/repository/storage/verification.rs
Normal file
32
src/repository/storage/verification.rs
Normal file
@@ -0,0 +1,32 @@
|
||||
// Repository blob integrity verification helper.
|
||||
|
||||
fn verify_repository_blob_batch(
|
||||
store: &RocksStore,
|
||||
batch: &[(String, String)],
|
||||
summary: &mut RepositoryBlobVerificationSummary,
|
||||
) -> StorageResult<()> {
|
||||
let hashes = batch
|
||||
.iter()
|
||||
.map(|(_, hash)| hash.clone())
|
||||
.collect::<Vec<_>>();
|
||||
let blobs = store.get_blob_bytes_batch(&hashes)?;
|
||||
for ((uri, expected_hash), blob) in batch.iter().zip(blobs.into_iter()) {
|
||||
let bytes = blob.as_ref().ok_or(StorageError::InvalidData {
|
||||
entity: "repository_blob_verification",
|
||||
detail: format!("blob missing for URI {uri} (hash={expected_hash})"),
|
||||
})?;
|
||||
let actual_hash = hex::encode(compute_sha256_32(bytes));
|
||||
if !actual_hash.eq_ignore_ascii_case(expected_hash) {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: "repository_blob_verification",
|
||||
detail: format!(
|
||||
"blob hash mismatch for URI {uri}: expected={expected_hash}, actual={actual_hash}"
|
||||
),
|
||||
});
|
||||
}
|
||||
summary.current_objects += 1;
|
||||
summary.bytes_verified += bytes.len() as u64;
|
||||
}
|
||||
summary.batches += 1;
|
||||
Ok(())
|
||||
}
|
||||
3
src/repository/sync/mod.rs
Normal file
3
src/repository/sync/mod.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
pub mod repo;
|
||||
pub mod rrdp;
|
||||
pub(crate) mod store_projection;
|
||||
516
src/repository/sync/repo.rs
Normal file
516
src/repository/sync/repo.rs
Normal file
@@ -0,0 +1,516 @@
|
||||
#![allow(clippy::too_many_arguments)]
|
||||
|
||||
use crate::output::analysis::timing::TimingHandle;
|
||||
use crate::output::audit::AuditDownloadKind;
|
||||
use crate::output::audit_downloads::DownloadLogHandle;
|
||||
use crate::output::report::{RfcRef, Warning};
|
||||
use crate::repository::current_repo_index::CurrentRepoIndexHandle;
|
||||
use crate::repository::fetch::rsync::{RsyncFetchError, RsyncFetcher};
|
||||
use crate::repository::storage::RocksStore;
|
||||
use crate::repository::sync::rrdp::sync_from_notification_with_timing_and_download_log;
|
||||
use crate::repository::sync::rrdp::{
|
||||
Fetcher as HttpFetcher, RrdpFetchError, RrdpOrigin, RrdpResourceKind, RrdpSyncError,
|
||||
};
|
||||
use crate::repository::sync::store_projection::{
|
||||
build_repository_view_present_entry, build_repository_view_withdrawn_entry,
|
||||
prepare_repo_bytes_batch_owned,
|
||||
};
|
||||
use crate::validation::policy::{Policy, SyncPreference};
|
||||
use std::collections::HashSet;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum RepoSyncSource {
|
||||
Rrdp,
|
||||
Rsync,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum RepoSyncPhase {
|
||||
RrdpOk,
|
||||
RrdpFailedRsyncOk,
|
||||
RsyncOnlyOk,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RepoSyncResult {
|
||||
pub source: RepoSyncSource,
|
||||
pub phase: RepoSyncPhase,
|
||||
pub objects_written: usize,
|
||||
pub warnings: Vec<Warning>,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum RepoSyncError {
|
||||
#[error("RRDP sync failed: {0}")]
|
||||
Rrdp(#[from] RrdpSyncError),
|
||||
|
||||
#[error("rsync fallback failed: {0}")]
|
||||
Rsync(#[from] RsyncFetchError),
|
||||
|
||||
#[error("storage error: {0}")]
|
||||
Storage(String),
|
||||
}
|
||||
|
||||
/// Sync a publication point into the current repository view.
|
||||
///
|
||||
/// v1 behavior:
|
||||
/// - If `rrdp_notification_uri` is present and `policy.sync_preference` is `rrdp_then_rsync`,
|
||||
/// try RRDP snapshot sync first (RFC 8182 §3.4.1-§3.4.3).
|
||||
/// - On RRDP failure, fall back to rsync (RFC 8182 §3.4.5).
|
||||
/// - If `sync_preference` is `rsync_only` or there is no RRDP URI, use rsync.
|
||||
pub fn sync_publication_point(
|
||||
store: &RocksStore,
|
||||
policy: &Policy,
|
||||
rrdp_notification_uri: Option<&str>,
|
||||
rsync_base_uri: &str,
|
||||
http_fetcher: &dyn HttpFetcher,
|
||||
rsync_fetcher: &dyn RsyncFetcher,
|
||||
timing: Option<&TimingHandle>,
|
||||
download_log: Option<&DownloadLogHandle>,
|
||||
) -> Result<RepoSyncResult, RepoSyncError> {
|
||||
match (policy.sync_preference, rrdp_notification_uri) {
|
||||
(SyncPreference::RrdpThenRsync, Some(notification_uri)) => {
|
||||
match try_rrdp_sync_with_retry(
|
||||
store,
|
||||
notification_uri,
|
||||
None,
|
||||
http_fetcher,
|
||||
timing,
|
||||
download_log,
|
||||
) {
|
||||
Ok(written) => {
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("repo_sync_rrdp_ok_total", 1);
|
||||
t.record_count("repo_sync_rrdp_objects_written_total", written as u64);
|
||||
}
|
||||
crate::logging::progress::emit!(
|
||||
"repo_sync_rrdp_ok",
|
||||
serde_json::json!({
|
||||
"notify_uri": notification_uri,
|
||||
"objects_written": written,
|
||||
}),
|
||||
);
|
||||
Ok(RepoSyncResult {
|
||||
source: RepoSyncSource::Rrdp,
|
||||
phase: RepoSyncPhase::RrdpOk,
|
||||
objects_written: written,
|
||||
warnings: Vec::new(),
|
||||
})
|
||||
}
|
||||
Err(err) => {
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("repo_sync_rrdp_failed_total", 1);
|
||||
}
|
||||
crate::logging::progress::emit!(
|
||||
"rrdp_fallback_rsync",
|
||||
serde_json::json!({
|
||||
"notify_uri": notification_uri,
|
||||
"rsync_base_uri": rsync_base_uri,
|
||||
"rrdp_error": err.to_string(),
|
||||
}),
|
||||
);
|
||||
crate::logging::progress::emit!(
|
||||
"rrdp_failed_fallback_rsync",
|
||||
serde_json::json!({
|
||||
"notify_uri": notification_uri,
|
||||
"rsync_base_uri": rsync_base_uri,
|
||||
"rrdp_error": err.to_string(),
|
||||
}),
|
||||
);
|
||||
let mut rfc_refs = vec![RfcRef("RFC 8182 §3.4.5")];
|
||||
if matches!(&err, RrdpSyncError::Rrdp(e) if e.is_cross_origin_violation()) {
|
||||
rfc_refs.push(RfcRef("RFC 9674 §3.2"));
|
||||
}
|
||||
let warnings = vec![
|
||||
Warning::new(format!("RRDP failed; falling back to rsync: {err}"))
|
||||
.with_rfc_refs(&rfc_refs)
|
||||
.with_context(notification_uri),
|
||||
];
|
||||
let written = rsync_sync_into_current_store(
|
||||
store,
|
||||
rsync_base_uri,
|
||||
None,
|
||||
rsync_fetcher,
|
||||
timing,
|
||||
download_log,
|
||||
)?;
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("repo_sync_rsync_peer_aligned_profile_total", 1);
|
||||
}
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("repo_sync_rsync_fallback_ok_total", 1);
|
||||
t.record_count("repo_sync_rsync_objects_written_total", written as u64);
|
||||
}
|
||||
Ok(RepoSyncResult {
|
||||
source: RepoSyncSource::Rsync,
|
||||
phase: RepoSyncPhase::RrdpFailedRsyncOk,
|
||||
objects_written: written,
|
||||
warnings,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
let written = rsync_sync_into_current_store(
|
||||
store,
|
||||
rsync_base_uri,
|
||||
None,
|
||||
rsync_fetcher,
|
||||
timing,
|
||||
download_log,
|
||||
)?;
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("repo_sync_rsync_peer_aligned_profile_total", 1);
|
||||
}
|
||||
crate::logging::progress::emit!(
|
||||
"repo_sync_rsync_direct",
|
||||
serde_json::json!({
|
||||
"rsync_base_uri": rsync_base_uri,
|
||||
"objects_written": written,
|
||||
}),
|
||||
);
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("repo_sync_rsync_direct_total", 1);
|
||||
t.record_count("repo_sync_rsync_objects_written_total", written as u64);
|
||||
}
|
||||
Ok(RepoSyncResult {
|
||||
source: RepoSyncSource::Rsync,
|
||||
phase: RepoSyncPhase::RsyncOnlyOk,
|
||||
objects_written: written,
|
||||
warnings: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn try_rrdp_sync(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||
http_fetcher: &dyn HttpFetcher,
|
||||
timing: Option<&TimingHandle>,
|
||||
download_log: Option<&DownloadLogHandle>,
|
||||
) -> Result<usize, RrdpSyncError> {
|
||||
let notification_origin = RrdpOrigin::parse(notification_uri, RrdpResourceKind::Notification)?;
|
||||
let notification_xml = {
|
||||
let _step = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_notification"));
|
||||
let _total = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_phase("rrdp_fetch_notification_total"));
|
||||
let mut dl_span = download_log
|
||||
.map(|dl| dl.span_download(AuditDownloadKind::RrdpNotification, notification_uri));
|
||||
match http_fetcher.fetch_rrdp(
|
||||
RrdpResourceKind::Notification,
|
||||
notification_uri,
|
||||
¬ification_origin,
|
||||
) {
|
||||
Ok(v) => {
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_notification_fetch_ok_total", 1);
|
||||
}
|
||||
if let Some(s) = dl_span.as_mut() {
|
||||
s.set_bytes(v.len() as u64);
|
||||
s.set_ok();
|
||||
}
|
||||
v
|
||||
}
|
||||
Err(RrdpFetchError::Fetch(e)) => {
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_notification_fetch_fail_total", 1);
|
||||
}
|
||||
if let Some(s) = dl_span.as_mut() {
|
||||
s.set_err(e.clone());
|
||||
}
|
||||
return Err(RrdpSyncError::Fetch(e));
|
||||
}
|
||||
Err(RrdpFetchError::Rrdp(e)) => {
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_notification_fetch_fail_total", 1);
|
||||
if e.is_cross_origin_violation() {
|
||||
t.record_count("rrdp_rejected_cross_origin_total", 1);
|
||||
}
|
||||
}
|
||||
if let Some(s) = dl_span.as_mut() {
|
||||
s.set_err(e.to_string());
|
||||
}
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
};
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count(
|
||||
"rrdp_notification_bytes_total",
|
||||
notification_xml.len() as u64,
|
||||
);
|
||||
}
|
||||
|
||||
sync_from_notification_with_timing_and_download_log(
|
||||
store,
|
||||
notification_uri,
|
||||
current_repo_index,
|
||||
¬ification_xml,
|
||||
http_fetcher,
|
||||
timing,
|
||||
download_log,
|
||||
)
|
||||
}
|
||||
|
||||
fn is_retryable_http_fetch_error(msg: &str) -> bool {
|
||||
if msg.contains("http request failed:") || msg.contains("http read body failed:") {
|
||||
return true;
|
||||
}
|
||||
let Some(rest) = msg.strip_prefix("http status ") else {
|
||||
return false;
|
||||
};
|
||||
let code = rest
|
||||
.split_whitespace()
|
||||
.next()
|
||||
.and_then(|s| s.parse::<u16>().ok())
|
||||
.unwrap_or(0);
|
||||
code == 408 || code == 429 || (500..600).contains(&code)
|
||||
}
|
||||
|
||||
fn try_rrdp_sync_with_retry(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||
http_fetcher: &dyn HttpFetcher,
|
||||
timing: Option<&TimingHandle>,
|
||||
download_log: Option<&DownloadLogHandle>,
|
||||
) -> Result<usize, RrdpSyncError> {
|
||||
let attempt = 1usize;
|
||||
crate::logging::progress::emit!(
|
||||
"rrdp_sync_attempt",
|
||||
serde_json::json!({
|
||||
"notify_uri": notification_uri,
|
||||
"attempt": attempt,
|
||||
}),
|
||||
);
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_retry_attempt_total", 1);
|
||||
}
|
||||
|
||||
match try_rrdp_sync(
|
||||
store,
|
||||
notification_uri,
|
||||
current_repo_index,
|
||||
http_fetcher,
|
||||
timing,
|
||||
download_log,
|
||||
) {
|
||||
Ok(written) => {
|
||||
crate::logging::progress::emit!(
|
||||
"rrdp_sync_success",
|
||||
serde_json::json!({
|
||||
"notify_uri": notification_uri,
|
||||
"attempt": attempt,
|
||||
"objects_written": written,
|
||||
}),
|
||||
);
|
||||
Ok(written)
|
||||
}
|
||||
Err(err) => {
|
||||
let retryable = match &err {
|
||||
RrdpSyncError::Fetch(msg) => is_retryable_http_fetch_error(msg),
|
||||
_ => false,
|
||||
};
|
||||
crate::logging::progress::emit!(
|
||||
"rrdp_sync_failed",
|
||||
serde_json::json!({
|
||||
"notify_uri": notification_uri,
|
||||
"attempt": attempt,
|
||||
"retryable": retryable,
|
||||
"error": err.to_string(),
|
||||
}),
|
||||
);
|
||||
if let Some(t) = timing.as_ref() {
|
||||
match &err {
|
||||
RrdpSyncError::Fetch(_) => t.record_count("rrdp_failed_fetch_total", 1),
|
||||
RrdpSyncError::Rrdp(_) => t.record_count("rrdp_failed_protocol_total", 1),
|
||||
RrdpSyncError::Storage(_) => t.record_count("rrdp_failed_storage_total", 1),
|
||||
}
|
||||
}
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn run_rrdp_transport(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||
http_fetcher: &dyn HttpFetcher,
|
||||
timing: Option<&TimingHandle>,
|
||||
download_log: Option<&DownloadLogHandle>,
|
||||
) -> Result<usize, RrdpSyncError> {
|
||||
try_rrdp_sync_with_retry(
|
||||
store,
|
||||
notification_uri,
|
||||
current_repo_index,
|
||||
http_fetcher,
|
||||
timing,
|
||||
download_log,
|
||||
)
|
||||
}
|
||||
|
||||
fn rsync_sync_into_current_store(
|
||||
store: &RocksStore,
|
||||
rsync_base_uri: &str,
|
||||
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||
rsync_fetcher: &dyn RsyncFetcher,
|
||||
timing: Option<&TimingHandle>,
|
||||
download_log: Option<&DownloadLogHandle>,
|
||||
) -> Result<usize, RepoSyncError> {
|
||||
let started = std::time::Instant::now();
|
||||
let sync_scope_uri = rsync_fetcher.dedup_key(rsync_base_uri);
|
||||
crate::logging::progress::emit!(
|
||||
"rsync_sync_start",
|
||||
serde_json::json!({
|
||||
"rsync_base_uri": rsync_base_uri,
|
||||
"sync_scope_uri": &sync_scope_uri,
|
||||
}),
|
||||
);
|
||||
let _s = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_rrdp_repo_step(rsync_base_uri, "rsync_fetch_objects"));
|
||||
let _p = timing.as_ref().map(|t| t.span_phase("rsync_fetch_total"));
|
||||
let mut dl_span =
|
||||
download_log.map(|dl| dl.span_download(AuditDownloadKind::Rsync, rsync_base_uri));
|
||||
let mut new_set: HashSet<String> = HashSet::new();
|
||||
let mut fetched_objects: Vec<(String, Vec<u8>)> = Vec::new();
|
||||
let (object_count, bytes_total) =
|
||||
match rsync_fetcher.visit_objects(rsync_base_uri, &mut |uri, bytes| {
|
||||
new_set.insert(uri.clone());
|
||||
fetched_objects.push((uri, bytes));
|
||||
Ok(())
|
||||
}) {
|
||||
Ok(v) => {
|
||||
if let Some(s) = dl_span.as_mut() {
|
||||
s.set_objects(v.0 as u64, v.1);
|
||||
s.set_bytes(v.1);
|
||||
s.set_ok();
|
||||
}
|
||||
v
|
||||
}
|
||||
Err(e) => {
|
||||
if let Some(s) = dl_span.as_mut() {
|
||||
s.set_err(e.to_string());
|
||||
}
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
crate::logging::progress::emit!(
|
||||
"rsync_sync_fetch_done",
|
||||
serde_json::json!({
|
||||
"rsync_base_uri": rsync_base_uri,
|
||||
"sync_scope_uri": &sync_scope_uri,
|
||||
"object_count": object_count,
|
||||
"bytes_total": bytes_total,
|
||||
"duration_ms": started.elapsed().as_millis() as u64,
|
||||
}),
|
||||
);
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rsync_objects_fetched_total", object_count as u64);
|
||||
t.record_count("rsync_objects_bytes_total", bytes_total);
|
||||
}
|
||||
drop(_p);
|
||||
|
||||
let existing_view = store
|
||||
.list_repository_view_entries_with_prefix(&sync_scope_uri)
|
||||
.map_err(|e| RepoSyncError::Storage(e.to_string()))?;
|
||||
|
||||
let _proj = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_phase("rsync_write_current_store_total"));
|
||||
let prepared_bytes =
|
||||
prepare_repo_bytes_batch_owned(fetched_objects).map_err(RepoSyncError::Storage)?;
|
||||
let mut repository_view_entries = Vec::new();
|
||||
for entry in existing_view {
|
||||
if !new_set.contains(&entry.rsync_uri) {
|
||||
repository_view_entries.push(build_repository_view_withdrawn_entry(
|
||||
&sync_scope_uri,
|
||||
&entry.rsync_uri,
|
||||
entry.current_hash,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
for uri in &new_set {
|
||||
let current_hash = prepared_bytes
|
||||
.uri_to_hash
|
||||
.get(uri)
|
||||
.cloned()
|
||||
.ok_or_else(|| {
|
||||
RepoSyncError::Storage(format!("missing raw_by_hash mapping for {uri}"))
|
||||
})?;
|
||||
repository_view_entries.push(build_repository_view_present_entry(
|
||||
&sync_scope_uri,
|
||||
uri,
|
||||
¤t_hash,
|
||||
));
|
||||
}
|
||||
|
||||
store
|
||||
.put_blob_bytes_batch(&prepared_bytes.blobs_to_write)
|
||||
.map_err(|e| RepoSyncError::Storage(e.to_string()))?;
|
||||
store
|
||||
.put_projection_batch(&repository_view_entries, &[], &[])
|
||||
.map_err(|e| RepoSyncError::Storage(e.to_string()))?;
|
||||
if let Some(index) = current_repo_index {
|
||||
index
|
||||
.write()
|
||||
.map_err(|_| RepoSyncError::Storage("current repo index lock poisoned".to_string()))?
|
||||
.apply_repository_view_entries(&repository_view_entries)
|
||||
.map_err(RepoSyncError::Storage)?;
|
||||
}
|
||||
|
||||
let total_duration_ms = started.elapsed().as_millis() as u64;
|
||||
crate::logging::progress::emit!(
|
||||
"rsync_sync_done",
|
||||
serde_json::json!({
|
||||
"rsync_base_uri": rsync_base_uri,
|
||||
"sync_scope_uri": &sync_scope_uri,
|
||||
"object_count": object_count,
|
||||
"bytes_total": bytes_total,
|
||||
"duration_ms": total_duration_ms,
|
||||
}),
|
||||
);
|
||||
if (total_duration_ms as f64) / 1000.0 >= crate::logging::progress::slow_threshold_secs() {
|
||||
crate::logging::progress::emit!(
|
||||
"rsync_sync_slow",
|
||||
serde_json::json!({
|
||||
"rsync_base_uri": rsync_base_uri,
|
||||
"sync_scope_uri": &sync_scope_uri,
|
||||
"object_count": object_count,
|
||||
"bytes_total": bytes_total,
|
||||
"duration_ms": total_duration_ms,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
Ok(object_count)
|
||||
}
|
||||
|
||||
pub(crate) fn run_rsync_transport(
|
||||
store: &RocksStore,
|
||||
rsync_base_uri: &str,
|
||||
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||
rsync_fetcher: &dyn RsyncFetcher,
|
||||
timing: Option<&TimingHandle>,
|
||||
download_log: Option<&DownloadLogHandle>,
|
||||
) -> Result<usize, RepoSyncError> {
|
||||
rsync_sync_into_current_store(
|
||||
store,
|
||||
rsync_base_uri,
|
||||
current_repo_index,
|
||||
rsync_fetcher,
|
||||
timing,
|
||||
download_log,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "repo/tests.rs"]
|
||||
mod tests;
|
||||
2
src/repository/sync/repo/tests.rs
Normal file
2
src/repository/sync/repo/tests.rs
Normal file
@@ -0,0 +1,2 @@
|
||||
// Repository sync tests cover public RRDP/rsync transport behavior.
|
||||
include!("tests_parts/setup_and_sync.rs");
|
||||
371
src/repository/sync/repo/tests_parts/setup_and_sync.rs
Normal file
371
src/repository/sync/repo/tests_parts/setup_and_sync.rs
Normal file
@@ -0,0 +1,371 @@
|
||||
// Repository sync test group: setup and sync.
|
||||
|
||||
use super::*;
|
||||
use crate::output::analysis::timing::{TimingHandle, TimingMeta};
|
||||
use crate::repository::fetch::rsync::LocalDirRsyncFetcher;
|
||||
use crate::repository::storage::RepositoryViewState;
|
||||
use crate::repository::sync::rrdp::Fetcher as HttpFetcher;
|
||||
use crate::repository::sync::store_projection::{build_repository_view_present_entry, compute_sha256_hex};
|
||||
use sha2::Digest;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
struct DummyHttpFetcher;
|
||||
|
||||
impl HttpFetcher for DummyHttpFetcher {
|
||||
fn fetch(&self, _url: &str) -> Result<Vec<u8>, String> {
|
||||
panic!("http fetcher must not be used in rsync-only mode")
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_current_object(store: &RocksStore, uri: &str, expected: &[u8]) {
|
||||
assert_eq!(
|
||||
store
|
||||
.load_current_object_bytes_by_uri(uri)
|
||||
.expect("load current object"),
|
||||
Some(expected.to_vec())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rsync_sync_uses_fetcher_dedup_scope_for_repository_view_projection() {
|
||||
struct ScopeFetcher;
|
||||
impl RsyncFetcher for ScopeFetcher {
|
||||
fn fetch_objects(
|
||||
&self,
|
||||
_rsync_base_uri: &str,
|
||||
) -> Result<Vec<(String, Vec<u8>)>, RsyncFetchError> {
|
||||
Ok(vec![(
|
||||
"rsync://example.net/repo/child/a.mft".to_string(),
|
||||
b"manifest".to_vec(),
|
||||
)])
|
||||
}
|
||||
|
||||
fn dedup_key(&self, _rsync_base_uri: &str) -> String {
|
||||
"rsync://example.net/repo/".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
let seeded = build_repository_view_present_entry(
|
||||
"rsync://example.net/repo/",
|
||||
"rsync://example.net/repo/sibling/old.roa",
|
||||
&compute_sha256_hex(b"old"),
|
||||
);
|
||||
store
|
||||
.put_projection_batch(&[seeded], &[], &[])
|
||||
.expect("seed repository view");
|
||||
|
||||
let fetcher = ScopeFetcher;
|
||||
let written = rsync_sync_into_current_store(
|
||||
&store,
|
||||
"rsync://example.net/repo/child/",
|
||||
None,
|
||||
&fetcher,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("sync ok");
|
||||
assert_eq!(written, 1);
|
||||
|
||||
let entries = store
|
||||
.list_repository_view_entries_with_prefix("rsync://example.net/repo/")
|
||||
.expect("list repository view");
|
||||
let sibling = entries
|
||||
.iter()
|
||||
.find(|entry| entry.rsync_uri == "rsync://example.net/repo/sibling/old.roa")
|
||||
.expect("sibling entry exists");
|
||||
assert_eq!(sibling.state, RepositoryViewState::Withdrawn);
|
||||
let child = entries
|
||||
.iter()
|
||||
.find(|entry| entry.rsync_uri == "rsync://example.net/repo/child/a.mft")
|
||||
.expect("child entry exists");
|
||||
assert_eq!(child.state, RepositoryViewState::Present);
|
||||
}
|
||||
|
||||
fn timing_to_json(temp_dir: &std::path::Path, timing: &TimingHandle) -> serde_json::Value {
|
||||
let timing_path = temp_dir.join("timing_retry.json");
|
||||
timing.write_json(&timing_path, 50).expect("write json");
|
||||
serde_json::from_slice(&std::fs::read(&timing_path).expect("read json")).expect("parse json")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rsync_sync_writes_current_store_and_records_counts() {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
|
||||
let repo_dir = temp.path().join("repo");
|
||||
std::fs::create_dir_all(repo_dir.join("sub")).expect("mkdir");
|
||||
std::fs::write(repo_dir.join("a.mft"), b"mft").expect("write");
|
||||
std::fs::write(repo_dir.join("sub").join("b.roa"), b"roa").expect("write");
|
||||
std::fs::write(repo_dir.join("sub").join("c.cer"), b"cer").expect("write");
|
||||
|
||||
let store_dir = temp.path().join("db");
|
||||
let store = RocksStore::open(&store_dir).expect("open rocksdb");
|
||||
|
||||
let timing = TimingHandle::new(TimingMeta {
|
||||
recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(),
|
||||
validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(),
|
||||
tal_url: None,
|
||||
db_path: Some(store_dir.to_string_lossy().into_owned()),
|
||||
});
|
||||
|
||||
let policy = Policy {
|
||||
sync_preference: SyncPreference::RsyncOnly,
|
||||
..Policy::default()
|
||||
};
|
||||
let http = DummyHttpFetcher;
|
||||
let rsync = LocalDirRsyncFetcher::new(&repo_dir);
|
||||
|
||||
let download_log = DownloadLogHandle::new();
|
||||
let out = sync_publication_point(
|
||||
&store,
|
||||
&policy,
|
||||
None,
|
||||
"rsync://example.test/repo/",
|
||||
&http,
|
||||
&rsync,
|
||||
Some(&timing),
|
||||
Some(&download_log),
|
||||
)
|
||||
.expect("sync ok");
|
||||
|
||||
assert_eq!(out.source, RepoSyncSource::Rsync);
|
||||
assert_eq!(out.objects_written, 3);
|
||||
|
||||
let events = download_log.snapshot_events();
|
||||
assert_eq!(events.len(), 1);
|
||||
assert_eq!(events[0].kind, AuditDownloadKind::Rsync);
|
||||
assert!(events[0].success);
|
||||
assert_eq!(events[0].bytes, Some(9));
|
||||
let objects = events[0].objects.as_ref().expect("objects stat");
|
||||
assert_eq!(objects.objects_count, 3);
|
||||
assert_eq!(objects.objects_bytes_total, 9);
|
||||
|
||||
assert_current_object(&store, "rsync://example.test/repo/a.mft", b"mft");
|
||||
assert_current_object(&store, "rsync://example.test/repo/sub/b.roa", b"roa");
|
||||
assert_current_object(&store, "rsync://example.test/repo/sub/c.cer", b"cer");
|
||||
|
||||
let view = store
|
||||
.get_repository_view_entry("rsync://example.test/repo/a.mft")
|
||||
.expect("get repository view")
|
||||
.expect("repository view entry present");
|
||||
assert_eq!(
|
||||
view.current_hash.as_deref(),
|
||||
Some(hex::encode(sha2::Sha256::digest(b"mft")).as_str())
|
||||
);
|
||||
assert_eq!(
|
||||
view.repository_source.as_deref(),
|
||||
Some("rsync://example.test/repo/")
|
||||
);
|
||||
|
||||
let current_bytes = store
|
||||
.load_current_object_bytes_by_uri("rsync://example.test/repo/sub/b.roa")
|
||||
.expect("load current bytes")
|
||||
.expect("current object bytes exist");
|
||||
assert_eq!(current_bytes, b"roa".to_vec());
|
||||
assert!(
|
||||
store
|
||||
.get_raw_by_hash_entry(hex::encode(sha2::Sha256::digest(b"roa")).as_str())
|
||||
.expect("get raw_by_hash")
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let timing_path = temp.path().join("timing.json");
|
||||
timing.write_json(&timing_path, 5).expect("write json");
|
||||
let v: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(&timing_path).expect("read json"))
|
||||
.expect("parse json");
|
||||
let counts = v.get("counts").expect("counts");
|
||||
assert_eq!(
|
||||
counts
|
||||
.get("rsync_objects_fetched_total")
|
||||
.and_then(|v| v.as_u64()),
|
||||
Some(3)
|
||||
);
|
||||
assert_eq!(
|
||||
counts
|
||||
.get("rsync_objects_bytes_total")
|
||||
.and_then(|v| v.as_u64()),
|
||||
Some(3 * 3)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rsync_second_sync_marks_missing_repository_view_entries_withdrawn() {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
|
||||
let repo_dir = temp.path().join("repo");
|
||||
std::fs::create_dir_all(repo_dir.join("sub")).expect("mkdir");
|
||||
std::fs::write(repo_dir.join("a.mft"), b"mft-v1").expect("write a");
|
||||
std::fs::write(repo_dir.join("sub").join("b.roa"), b"roa-v1").expect("write b");
|
||||
|
||||
let store_dir = temp.path().join("db");
|
||||
let store = RocksStore::open(&store_dir).expect("open rocksdb");
|
||||
let policy = Policy {
|
||||
sync_preference: SyncPreference::RsyncOnly,
|
||||
..Policy::default()
|
||||
};
|
||||
let http = DummyHttpFetcher;
|
||||
let rsync = LocalDirRsyncFetcher::new(&repo_dir);
|
||||
|
||||
sync_publication_point(
|
||||
&store,
|
||||
&policy,
|
||||
None,
|
||||
"rsync://example.test/repo/",
|
||||
&http,
|
||||
&rsync,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("first sync ok");
|
||||
|
||||
std::fs::remove_file(repo_dir.join("sub").join("b.roa")).expect("remove b");
|
||||
std::fs::write(repo_dir.join("c.crl"), b"crl-v2").expect("write c");
|
||||
|
||||
sync_publication_point(
|
||||
&store,
|
||||
&policy,
|
||||
None,
|
||||
"rsync://example.test/repo/",
|
||||
&http,
|
||||
&rsync,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("second sync ok");
|
||||
|
||||
let withdrawn = store
|
||||
.get_repository_view_entry("rsync://example.test/repo/sub/b.roa")
|
||||
.expect("get withdrawn repo view")
|
||||
.expect("withdrawn entry exists");
|
||||
assert_eq!(
|
||||
withdrawn.state,
|
||||
crate::repository::storage::RepositoryViewState::Withdrawn
|
||||
);
|
||||
assert_eq!(
|
||||
withdrawn.repository_source.as_deref(),
|
||||
Some("rsync://example.test/repo/")
|
||||
);
|
||||
|
||||
let added = store
|
||||
.get_repository_view_entry("rsync://example.test/repo/c.crl")
|
||||
.expect("get added repo view")
|
||||
.expect("added entry exists");
|
||||
assert_eq!(added.state, crate::repository::storage::RepositoryViewState::Present);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rrdp_fetch_error_falls_back_to_rsync_without_retry() {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let store_dir = temp.path().join("db");
|
||||
let store = RocksStore::open(&store_dir).expect("open rocksdb");
|
||||
|
||||
let timing = TimingHandle::new(TimingMeta {
|
||||
recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(),
|
||||
validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(),
|
||||
tal_url: None,
|
||||
db_path: Some(store_dir.to_string_lossy().into_owned()),
|
||||
});
|
||||
|
||||
let notification_uri = "https://example.test/notification.xml";
|
||||
let published_uri = "rsync://example.test/repo/a.mft";
|
||||
let published_bytes = b"x";
|
||||
struct AlwaysFailHttp {
|
||||
notification_calls: AtomicUsize,
|
||||
}
|
||||
|
||||
impl HttpFetcher for AlwaysFailHttp {
|
||||
fn fetch(&self, _uri: &str) -> Result<Vec<u8>, String> {
|
||||
self.notification_calls.fetch_add(1, Ordering::SeqCst);
|
||||
Err("http request failed: simulated transient".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
struct SingleObjectRsync {
|
||||
uri: String,
|
||||
bytes: Vec<u8>,
|
||||
}
|
||||
impl RsyncFetcher for SingleObjectRsync {
|
||||
fn fetch_objects(
|
||||
&self,
|
||||
_rsync_base_uri: &str,
|
||||
) -> Result<Vec<(String, Vec<u8>)>, RsyncFetchError> {
|
||||
Ok(vec![(self.uri.clone(), self.bytes.clone())])
|
||||
}
|
||||
}
|
||||
|
||||
let http = AlwaysFailHttp {
|
||||
notification_calls: AtomicUsize::new(0),
|
||||
};
|
||||
|
||||
let policy = Policy {
|
||||
sync_preference: SyncPreference::RrdpThenRsync,
|
||||
..Policy::default()
|
||||
};
|
||||
|
||||
let download_log = DownloadLogHandle::new();
|
||||
let out = sync_publication_point(
|
||||
&store,
|
||||
&policy,
|
||||
Some(notification_uri),
|
||||
"rsync://example.test/repo/",
|
||||
&http,
|
||||
&SingleObjectRsync {
|
||||
uri: published_uri.to_string(),
|
||||
bytes: published_bytes.to_vec(),
|
||||
},
|
||||
Some(&timing),
|
||||
Some(&download_log),
|
||||
)
|
||||
.expect("sync ok");
|
||||
|
||||
assert_eq!(out.source, RepoSyncSource::Rsync);
|
||||
assert_current_object(&store, published_uri, published_bytes);
|
||||
assert_eq!(http.notification_calls.load(Ordering::SeqCst), 1);
|
||||
|
||||
let events = download_log.snapshot_events();
|
||||
assert_eq!(events.len(), 2, "expected 1x notification + 1x rsync");
|
||||
assert_eq!(
|
||||
events
|
||||
.iter()
|
||||
.filter(|e| e.kind == AuditDownloadKind::RrdpNotification)
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
events
|
||||
.iter()
|
||||
.filter(|e| e.kind == AuditDownloadKind::RrdpNotification && !e.success)
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
events
|
||||
.iter()
|
||||
.filter(|e| e.kind == AuditDownloadKind::Rsync)
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
|
||||
let v = timing_to_json(temp.path(), &timing);
|
||||
let counts = v.get("counts").expect("counts");
|
||||
assert_eq!(
|
||||
counts
|
||||
.get("rrdp_retry_attempt_total")
|
||||
.and_then(|v| v.as_u64()),
|
||||
Some(1)
|
||||
);
|
||||
assert_eq!(
|
||||
counts
|
||||
.get("repo_sync_rrdp_failed_total")
|
||||
.and_then(|v| v.as_u64()),
|
||||
Some(1)
|
||||
);
|
||||
assert_eq!(
|
||||
counts
|
||||
.get("repo_sync_rsync_fallback_ok_total")
|
||||
.and_then(|v| v.as_u64()),
|
||||
Some(1)
|
||||
);
|
||||
}
|
||||
56
src/repository/sync/rrdp.rs
Normal file
56
src/repository/sync/rrdp.rs
Normal file
@@ -0,0 +1,56 @@
|
||||
use crate::output::analysis::timing::TimingHandle;
|
||||
use crate::output::audit::AuditDownloadKind;
|
||||
use crate::output::audit_downloads::DownloadLogHandle;
|
||||
use crate::repository::current_repo_index::CurrentRepoIndexHandle;
|
||||
use crate::repository::storage::{
|
||||
RepositoryViewEntry, RepositoryViewState, RocksStore, RrdpDeltaOp, RrdpSourceSyncState,
|
||||
};
|
||||
mod snapshot_apply;
|
||||
|
||||
use crate::repository::sync::store_projection::{
|
||||
build_repository_view_present_entry, build_repository_view_withdrawn_entry,
|
||||
current_rrdp_owner_is, ensure_rrdp_uri_can_be_owned_by, put_repository_view_present,
|
||||
put_repository_view_withdrawn, put_rrdp_source_member_present,
|
||||
put_rrdp_source_member_withdrawn, put_rrdp_uri_owner_active, put_rrdp_uri_owner_withdrawn,
|
||||
update_rrdp_source_record_on_success, upsert_repo_blob_bytes,
|
||||
};
|
||||
use base64::Engine;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::Digest;
|
||||
use std::fmt;
|
||||
use std::io::Write;
|
||||
use url::Url;
|
||||
use uuid::Uuid;
|
||||
|
||||
const RRDP_XMLNS: &str = "http://www.ripe.net/rpki/rrdp";
|
||||
const RRDP_SNAPSHOT_APPLY_BATCH_SIZE: usize = 1024;
|
||||
|
||||
/// The RRDP object currently being fetched. Keeping this in protocol errors
|
||||
/// makes an origin rejection actionable without treating it as a transient
|
||||
/// transport failure.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum RrdpResourceKind {
|
||||
Notification,
|
||||
Snapshot,
|
||||
Delta,
|
||||
}
|
||||
|
||||
impl fmt::Display for RrdpResourceKind {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(match self {
|
||||
Self::Notification => "notification",
|
||||
Self::Snapshot => "snapshot",
|
||||
Self::Delta => "delta",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
include!("rrdp/models_and_parsing.rs");
|
||||
include!("rrdp/snapshot_sync.rs");
|
||||
include!("rrdp/notification_sync.rs");
|
||||
include!("rrdp/delta.rs");
|
||||
include!("rrdp/parse_helpers.rs");
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "rrdp/tests.rs"]
|
||||
mod tests;
|
||||
272
src/repository/sync/rrdp/delta.rs
Normal file
272
src/repository/sync/rrdp/delta.rs
Normal file
@@ -0,0 +1,272 @@
|
||||
// Current-repository hydration and delta application.
|
||||
|
||||
fn hydrate_current_repo_index_from_rrdp_members(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||
) -> Result<usize, RrdpSyncError> {
|
||||
let Some(index) = current_repo_index else {
|
||||
return Ok(0);
|
||||
};
|
||||
|
||||
let members = store
|
||||
.list_current_rrdp_source_members(notification_uri)
|
||||
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
|
||||
if members.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let mut entries = Vec::with_capacity(members.len());
|
||||
for member in members {
|
||||
let current_hash = member.current_hash.ok_or_else(|| {
|
||||
RrdpSyncError::Storage(format!(
|
||||
"rrdp source member missing current_hash for current object {}",
|
||||
member.rsync_uri
|
||||
))
|
||||
})?;
|
||||
entries.push(RepositoryViewEntry {
|
||||
rsync_uri: member.rsync_uri,
|
||||
current_hash: Some(current_hash),
|
||||
repository_source: Some(notification_uri.to_string()),
|
||||
object_type: member.object_type,
|
||||
state: RepositoryViewState::Present,
|
||||
});
|
||||
}
|
||||
|
||||
index
|
||||
.write()
|
||||
.map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))?
|
||||
.apply_repository_view_entries(&entries)
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
Ok(entries.len())
|
||||
}
|
||||
|
||||
fn apply_delta(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||
delta_xml: &[u8],
|
||||
expected_hash_sha256: [u8; 32],
|
||||
expected_session_id: Uuid,
|
||||
expected_serial: u64,
|
||||
) -> Result<usize, RrdpSyncError> {
|
||||
let computed = sha2::Sha256::digest(delta_xml);
|
||||
if computed.as_slice() != expected_hash_sha256.as_slice() {
|
||||
return Err(RrdpError::DeltaHashMismatch.into());
|
||||
}
|
||||
|
||||
let delta = parse_delta_file(delta_xml)?;
|
||||
if delta.session_id != expected_session_id {
|
||||
return Err(RrdpError::DeltaSessionIdMismatch {
|
||||
expected: expected_session_id.to_string(),
|
||||
got: delta.session_id.to_string(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
if delta.serial != expected_serial {
|
||||
return Err(RrdpError::DeltaSerialMismatch {
|
||||
expected: expected_serial,
|
||||
got: delta.serial,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
|
||||
enum DeltaProjectionEffect {
|
||||
Upsert {
|
||||
rsync_uri: String,
|
||||
bytes: Vec<u8>,
|
||||
},
|
||||
Delete {
|
||||
rsync_uri: String,
|
||||
previous_hash: String,
|
||||
},
|
||||
}
|
||||
|
||||
let session_id = expected_session_id.to_string();
|
||||
let mut ops: Vec<RrdpDeltaOp> = Vec::with_capacity(delta.elements.len());
|
||||
let mut projection: Vec<DeltaProjectionEffect> = Vec::with_capacity(delta.elements.len());
|
||||
for e in delta.elements {
|
||||
match e {
|
||||
DeltaElement::Publish {
|
||||
uri,
|
||||
hash_sha256: Some(old_hash),
|
||||
bytes,
|
||||
} => {
|
||||
let is_member = store
|
||||
.is_current_rrdp_source_member(notification_uri, uri.as_str())
|
||||
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
|
||||
if !is_member {
|
||||
return Err(RrdpError::DeltaTargetNotFromRepository { rsync_uri: uri }.into());
|
||||
}
|
||||
ensure_rrdp_uri_can_be_owned_by(store, notification_uri, uri.as_str())
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
let old_bytes = store
|
||||
.load_current_object_bytes_by_uri(uri.as_str())
|
||||
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?
|
||||
.ok_or_else(|| RrdpError::DeltaTargetMissing {
|
||||
rsync_uri: uri.clone(),
|
||||
})?;
|
||||
let old_computed = sha2::Sha256::digest(old_bytes.as_slice());
|
||||
if old_computed.as_slice() != old_hash.as_slice() {
|
||||
return Err(RrdpError::DeltaTargetHashMismatch { rsync_uri: uri }.into());
|
||||
}
|
||||
|
||||
ops.push(RrdpDeltaOp::Upsert {
|
||||
rsync_uri: uri.clone(),
|
||||
bytes: bytes.clone(),
|
||||
});
|
||||
projection.push(DeltaProjectionEffect::Upsert {
|
||||
rsync_uri: uri,
|
||||
bytes,
|
||||
});
|
||||
}
|
||||
DeltaElement::Publish {
|
||||
uri,
|
||||
hash_sha256: None,
|
||||
bytes,
|
||||
} => {
|
||||
let is_member = store
|
||||
.is_current_rrdp_source_member(notification_uri, uri.as_str())
|
||||
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
|
||||
if is_member {
|
||||
return Err(
|
||||
RrdpError::DeltaPublishWithoutHashForExisting { rsync_uri: uri }.into(),
|
||||
);
|
||||
}
|
||||
ensure_rrdp_uri_can_be_owned_by(store, notification_uri, uri.as_str())
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
ops.push(RrdpDeltaOp::Upsert {
|
||||
rsync_uri: uri.clone(),
|
||||
bytes: bytes.clone(),
|
||||
});
|
||||
projection.push(DeltaProjectionEffect::Upsert {
|
||||
rsync_uri: uri,
|
||||
bytes,
|
||||
});
|
||||
}
|
||||
DeltaElement::Withdraw { uri, hash_sha256 } => {
|
||||
let is_member = store
|
||||
.is_current_rrdp_source_member(notification_uri, uri.as_str())
|
||||
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
|
||||
if !is_member {
|
||||
return Err(RrdpError::DeltaTargetNotFromRepository { rsync_uri: uri }.into());
|
||||
}
|
||||
ensure_rrdp_uri_can_be_owned_by(store, notification_uri, uri.as_str())
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
let old_bytes = store
|
||||
.load_current_object_bytes_by_uri(uri.as_str())
|
||||
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?
|
||||
.ok_or_else(|| RrdpError::DeltaTargetMissing {
|
||||
rsync_uri: uri.clone(),
|
||||
})?;
|
||||
let old_computed = sha2::Sha256::digest(old_bytes.as_slice());
|
||||
if old_computed.as_slice() != hash_sha256.as_slice() {
|
||||
return Err(RrdpError::DeltaTargetHashMismatch { rsync_uri: uri }.into());
|
||||
}
|
||||
let previous_hash = hex::encode(old_computed);
|
||||
ops.push(RrdpDeltaOp::Delete {
|
||||
rsync_uri: uri.clone(),
|
||||
});
|
||||
projection.push(DeltaProjectionEffect::Delete {
|
||||
rsync_uri: uri,
|
||||
previous_hash,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for effect in projection {
|
||||
match effect {
|
||||
DeltaProjectionEffect::Upsert { rsync_uri, bytes } => {
|
||||
let current_hash =
|
||||
upsert_repo_blob_bytes(store, &bytes).map_err(RrdpSyncError::Storage)?;
|
||||
put_repository_view_present(store, notification_uri, &rsync_uri, ¤t_hash)
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
if let Some(index) = current_repo_index {
|
||||
let entry = build_repository_view_present_entry(
|
||||
notification_uri,
|
||||
&rsync_uri,
|
||||
¤t_hash,
|
||||
);
|
||||
index
|
||||
.write()
|
||||
.map_err(|_| {
|
||||
RrdpSyncError::Storage("current repo index lock poisoned".to_string())
|
||||
})?
|
||||
.apply_repository_view_entries(&[entry])
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
}
|
||||
put_rrdp_source_member_present(
|
||||
store,
|
||||
notification_uri,
|
||||
&session_id,
|
||||
expected_serial,
|
||||
&rsync_uri,
|
||||
¤t_hash,
|
||||
)
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
put_rrdp_uri_owner_active(
|
||||
store,
|
||||
notification_uri,
|
||||
&session_id,
|
||||
expected_serial,
|
||||
&rsync_uri,
|
||||
¤t_hash,
|
||||
)
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
}
|
||||
DeltaProjectionEffect::Delete {
|
||||
rsync_uri,
|
||||
previous_hash,
|
||||
} => {
|
||||
put_rrdp_source_member_withdrawn(
|
||||
store,
|
||||
notification_uri,
|
||||
&session_id,
|
||||
expected_serial,
|
||||
&rsync_uri,
|
||||
Some(previous_hash.clone()),
|
||||
)
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
if current_rrdp_owner_is(store, notification_uri, &rsync_uri)
|
||||
.map_err(RrdpSyncError::Storage)?
|
||||
{
|
||||
put_repository_view_withdrawn(
|
||||
store,
|
||||
notification_uri,
|
||||
&rsync_uri,
|
||||
Some(previous_hash.clone()),
|
||||
)
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
if let Some(index) = current_repo_index {
|
||||
let entry = build_repository_view_withdrawn_entry(
|
||||
notification_uri,
|
||||
&rsync_uri,
|
||||
Some(previous_hash.clone()),
|
||||
);
|
||||
index
|
||||
.write()
|
||||
.map_err(|_| {
|
||||
RrdpSyncError::Storage(
|
||||
"current repo index lock poisoned".to_string(),
|
||||
)
|
||||
})?
|
||||
.apply_repository_view_entries(&[entry])
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
}
|
||||
put_rrdp_uri_owner_withdrawn(
|
||||
store,
|
||||
notification_uri,
|
||||
&session_id,
|
||||
expected_serial,
|
||||
&rsync_uri,
|
||||
Some(previous_hash),
|
||||
)
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ops.len())
|
||||
}
|
||||
639
src/repository/sync/rrdp/models_and_parsing.rs
Normal file
639
src/repository/sync/rrdp/models_and_parsing.rs
Normal file
@@ -0,0 +1,639 @@
|
||||
// RRDP error/state models and notification/delta parsing.
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum RrdpError {
|
||||
#[error("RRDP XML must be US-ASCII encoded (RFC 8182 §3.5.1.3, §3.5.2.3), got non-ASCII bytes")]
|
||||
NotAscii,
|
||||
|
||||
#[error("RRDP XML parse error: {0} (RFC 8182 §3.5.1.3, §3.5.2.3)")]
|
||||
Xml(String),
|
||||
|
||||
#[error(
|
||||
"RRDP root element must be <notification>, <snapshot>, or <delta>, got <{0}> (RFC 8182 §3.5.1.3, §3.5.2.3, §3.5.3.3)"
|
||||
)]
|
||||
UnexpectedRoot(String),
|
||||
|
||||
#[error("RRDP XML namespace must be {RRDP_XMLNS}, got {0} (RFC 8182 §3.5.1.3, §3.5.2.3)")]
|
||||
InvalidNamespace(String),
|
||||
|
||||
#[error("RRDP version must be 1, got {0} (RFC 8182 §3.5.1.3, §3.5.2.3)")]
|
||||
InvalidVersion(String),
|
||||
|
||||
#[error("RRDP session_id invalid UUID: {0} (RFC 8182 §3.5.1.3)")]
|
||||
InvalidSessionId(String),
|
||||
|
||||
#[error("RRDP serial invalid unsigned integer: {0} (RFC 8182 §3.5.1.3)")]
|
||||
InvalidSerial(String),
|
||||
|
||||
#[error("notification must contain exactly one <snapshot> element (RFC 8182 §3.5.1.3)")]
|
||||
SnapshotCountInvalid,
|
||||
|
||||
#[error("snapshot/@uri missing (RFC 8182 §3.5.1.3)")]
|
||||
SnapshotUriMissing,
|
||||
|
||||
#[error("snapshot/@hash missing (RFC 8182 §3.5.1.3)")]
|
||||
SnapshotHashMissing,
|
||||
|
||||
#[error("snapshot/@hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.1.3)")]
|
||||
SnapshotHashInvalid(String),
|
||||
|
||||
#[error("delta/@serial missing in notification (RFC 8182 §3.5.1.3)")]
|
||||
DeltaRefSerialMissing,
|
||||
|
||||
#[error("delta/@uri missing in notification (RFC 8182 §3.5.1.3)")]
|
||||
DeltaRefUriMissing,
|
||||
|
||||
#[error("delta/@hash missing in notification (RFC 8182 §3.5.1.3)")]
|
||||
DeltaRefHashMissing,
|
||||
|
||||
#[error("delta/@hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.1.3)")]
|
||||
DeltaRefHashInvalid(String),
|
||||
|
||||
#[error("delta/@serial duplicates in notification: {0} (RFC 8182 §3.5.1.3)")]
|
||||
DeltaRefSerialDuplicate(u64),
|
||||
|
||||
#[error(
|
||||
"delta/@serial must be <= notification/@serial: delta={delta_serial} notification={notification_serial} (RFC 8182 §3.5.1.3)"
|
||||
)]
|
||||
DeltaRefSerialTooHigh {
|
||||
delta_serial: u64,
|
||||
notification_serial: u64,
|
||||
},
|
||||
|
||||
#[error(
|
||||
"notification contains deltas but does not end at notification/@serial: max_delta={max_delta_serial} notification={notification_serial} (RFC 8182 §3.5.1.3)"
|
||||
)]
|
||||
DeltaRefChainDoesNotEndAtNotificationSerial {
|
||||
max_delta_serial: u64,
|
||||
notification_serial: u64,
|
||||
},
|
||||
|
||||
#[error(
|
||||
"notification delta chain not contiguous: missing serial={missing_serial} (range {min_serial}..{notification_serial}) (RFC 8182 §3.5.1.3)"
|
||||
)]
|
||||
DeltaRefChainNotContiguous {
|
||||
min_serial: u64,
|
||||
notification_serial: u64,
|
||||
missing_serial: u64,
|
||||
},
|
||||
|
||||
#[error("snapshot file hash mismatch (RFC 8182 §3.5.1.3)")]
|
||||
SnapshotHashMismatch,
|
||||
|
||||
#[error("snapshot session_id mismatch: expected {expected}, got {got} (RFC 8182 §3.5.2.3)")]
|
||||
SnapshotSessionIdMismatch { expected: String, got: String },
|
||||
|
||||
#[error("snapshot serial mismatch: expected {expected}, got {got} (RFC 8182 §3.5.2.3)")]
|
||||
SnapshotSerialMismatch { expected: u64, got: u64 },
|
||||
|
||||
#[error("delta file hash mismatch (RFC 8182 §3.4.2; RFC 8182 §3.5.1.3)")]
|
||||
DeltaHashMismatch,
|
||||
|
||||
#[error("delta session_id mismatch: expected {expected}, got {got} (RFC 8182 §3.5.3.3)")]
|
||||
DeltaSessionIdMismatch { expected: String, got: String },
|
||||
|
||||
#[error("delta serial mismatch: expected {expected}, got {got} (RFC 8182 §3.5.3.3)")]
|
||||
DeltaSerialMismatch { expected: u64, got: u64 },
|
||||
|
||||
#[error("notification serial moved backwards: old={old} new={new} (RFC 8182 §3.4.1)")]
|
||||
NotificationSerialRollback { old: u64, new: u64 },
|
||||
|
||||
#[error("delta publish without @hash for existing object: {rsync_uri} (RFC 8182 §3.4.2)")]
|
||||
DeltaPublishWithoutHashForExisting { rsync_uri: String },
|
||||
|
||||
#[error(
|
||||
"delta withdraw/replace target not from this repository server: {rsync_uri} (RFC 8182 §3.4.2)"
|
||||
)]
|
||||
DeltaTargetNotFromRepository { rsync_uri: String },
|
||||
|
||||
#[error("delta withdraw/replace target missing in local cache: {rsync_uri} (RFC 8182 §3.4.2)")]
|
||||
DeltaTargetMissing { rsync_uri: String },
|
||||
|
||||
#[error("delta withdraw/replace target hash mismatch: {rsync_uri} (RFC 8182 §3.4.2)")]
|
||||
DeltaTargetHashMismatch { rsync_uri: String },
|
||||
|
||||
#[error("publish/@uri missing (RFC 8182 §3.5.2.3)")]
|
||||
PublishUriMissing,
|
||||
|
||||
#[error("publish element missing base64 content (RFC 8182 §3.5.2.3)")]
|
||||
PublishContentMissing,
|
||||
|
||||
#[error("publish base64 decode failed (RFC 8182 §3.5.2.3): {0}")]
|
||||
PublishBase64(String),
|
||||
|
||||
#[error("delta file <publish> missing @uri (RFC 8182 §3.5.3.3)")]
|
||||
DeltaPublishUriMissing,
|
||||
|
||||
#[error("delta file <publish> base64 content missing (RFC 8182 §3.5.3.3)")]
|
||||
DeltaPublishContentMissing,
|
||||
|
||||
#[error("delta file <publish> base64 decode failed (RFC 8182 §3.5.3.3): {0}")]
|
||||
DeltaPublishBase64(String),
|
||||
|
||||
#[error(
|
||||
"delta file <publish> @hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.3.3)"
|
||||
)]
|
||||
DeltaPublishHashInvalid(String),
|
||||
|
||||
#[error("delta file <withdraw> missing @uri (RFC 8182 §3.5.3.3)")]
|
||||
DeltaWithdrawUriMissing,
|
||||
|
||||
#[error("delta file <withdraw> missing @hash (RFC 8182 §3.5.3.3)")]
|
||||
DeltaWithdrawHashMissing,
|
||||
|
||||
#[error(
|
||||
"delta file <withdraw> @hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.3.3)"
|
||||
)]
|
||||
DeltaWithdrawHashInvalid(String),
|
||||
|
||||
#[error("delta file <withdraw> must not contain text content (RFC 8182 §3.5.3.3)")]
|
||||
DeltaWithdrawUnexpectedContent,
|
||||
|
||||
#[error("delta file must contain at least one publish/withdraw element (RFC 8182 §3.5.3.3)")]
|
||||
DeltaNoElements,
|
||||
|
||||
#[error("delta file contains unexpected element <{0}> (RFC 8182 §3.5.3.3)")]
|
||||
DeltaUnexpectedElement(String),
|
||||
|
||||
#[error("invalid RRDP {resource} URI: {detail} (RFC 9674 §3.2)")]
|
||||
InvalidRrdpUri {
|
||||
resource: RrdpResourceKind,
|
||||
detail: String,
|
||||
},
|
||||
|
||||
#[error(
|
||||
"RRDP {resource} URI crosses notification origin: notification={notification_origin}, resource={resource_origin} (RFC 9674 §3.2)"
|
||||
)]
|
||||
CrossOriginReference {
|
||||
resource: RrdpResourceKind,
|
||||
notification_origin: String,
|
||||
resource_origin: String,
|
||||
},
|
||||
|
||||
#[error(
|
||||
"RRDP {resource} redirect crosses notification origin: notification={notification_origin}, redirect={redirect_origin} (RFC 9674 §3.2)"
|
||||
)]
|
||||
CrossOriginRedirect {
|
||||
resource: RrdpResourceKind,
|
||||
notification_origin: String,
|
||||
redirect_origin: String,
|
||||
},
|
||||
|
||||
#[error("RRDP {resource} redirect has no Location header (RFC 9674 §3.2)")]
|
||||
RedirectLocationMissing { resource: RrdpResourceKind },
|
||||
|
||||
#[error("RRDP {resource} redirect Location is invalid: {location} (RFC 9674 §3.2)")]
|
||||
InvalidRedirectLocation {
|
||||
resource: RrdpResourceKind,
|
||||
location: String,
|
||||
},
|
||||
|
||||
#[error("RRDP {resource} redirect limit exceeded ({max}) (RFC 9674 §3.2)")]
|
||||
RedirectLimitExceeded {
|
||||
resource: RrdpResourceKind,
|
||||
max: usize,
|
||||
},
|
||||
}
|
||||
|
||||
impl RrdpError {
|
||||
pub fn is_cross_origin_violation(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::CrossOriginReference { .. } | Self::CrossOriginRedirect { .. }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// RFC 9674 §3.2 compares scheme, host name, and port. `Url` normalizes an
|
||||
/// omitted well-known port, so `https://example/` and `https://example:443/`
|
||||
/// intentionally have the same effective origin.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RrdpOrigin {
|
||||
scheme: String,
|
||||
host: String,
|
||||
port: u16,
|
||||
}
|
||||
|
||||
impl fmt::Display for RrdpOrigin {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}://{}:{}", self.scheme, self.host, self.port)
|
||||
}
|
||||
}
|
||||
|
||||
impl RrdpOrigin {
|
||||
pub fn parse(uri: &str, resource: RrdpResourceKind) -> Result<Self, RrdpError> {
|
||||
let url = Url::parse(uri).map_err(|e| RrdpError::InvalidRrdpUri {
|
||||
resource,
|
||||
detail: e.to_string(),
|
||||
})?;
|
||||
Self::from_url(&url, resource)
|
||||
}
|
||||
|
||||
pub fn from_url(url: &Url, resource: RrdpResourceKind) -> Result<Self, RrdpError> {
|
||||
if !matches!(url.scheme(), "http" | "https") {
|
||||
return Err(RrdpError::InvalidRrdpUri {
|
||||
resource,
|
||||
detail: format!("unsupported scheme {}", url.scheme()),
|
||||
});
|
||||
}
|
||||
if !url.username().is_empty() || url.password().is_some() {
|
||||
return Err(RrdpError::InvalidRrdpUri {
|
||||
resource,
|
||||
detail: "userinfo is not permitted".to_string(),
|
||||
});
|
||||
}
|
||||
let host = url.host_str().ok_or_else(|| RrdpError::InvalidRrdpUri {
|
||||
resource,
|
||||
detail: "host is missing".to_string(),
|
||||
})?;
|
||||
let port = url
|
||||
.port_or_known_default()
|
||||
.ok_or_else(|| RrdpError::InvalidRrdpUri {
|
||||
resource,
|
||||
detail: "port is missing or unknown".to_string(),
|
||||
})?;
|
||||
Ok(Self {
|
||||
scheme: url.scheme().to_ascii_lowercase(),
|
||||
host: host.to_ascii_lowercase(),
|
||||
port,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum RrdpSyncError {
|
||||
#[error("{0}")]
|
||||
Rrdp(#[from] RrdpError),
|
||||
|
||||
#[error("fetch failed: {0}")]
|
||||
Fetch(String),
|
||||
|
||||
#[error("storage error: {0}")]
|
||||
Storage(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum RrdpFetchError {
|
||||
#[error("{0}")]
|
||||
Rrdp(#[from] RrdpError),
|
||||
|
||||
#[error("{0}")]
|
||||
Fetch(String),
|
||||
}
|
||||
|
||||
pub type RrdpSyncResult<T> = Result<T, RrdpSyncError>;
|
||||
|
||||
pub trait Fetcher: Send + Sync {
|
||||
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String>;
|
||||
|
||||
fn fetch_to_writer(&self, uri: &str, out: &mut dyn Write) -> Result<u64, String> {
|
||||
let bytes = self.fetch(uri)?;
|
||||
out.write_all(&bytes)
|
||||
.map_err(|e| format!("write sink failed: {e}"))?;
|
||||
Ok(bytes.len() as u64)
|
||||
}
|
||||
|
||||
/// RRDP-only fetch path. Implementations that do not perform HTTP may use
|
||||
/// the compatibility default; HTTP implementations must enforce redirect
|
||||
/// origin checks before issuing the next request.
|
||||
fn fetch_rrdp(
|
||||
&self,
|
||||
_resource: RrdpResourceKind,
|
||||
uri: &str,
|
||||
_notification_origin: &RrdpOrigin,
|
||||
) -> Result<Vec<u8>, RrdpFetchError> {
|
||||
self.fetch(uri).map_err(RrdpFetchError::Fetch)
|
||||
}
|
||||
|
||||
fn fetch_rrdp_to_writer(
|
||||
&self,
|
||||
_resource: RrdpResourceKind,
|
||||
uri: &str,
|
||||
_notification_origin: &RrdpOrigin,
|
||||
out: &mut dyn Write,
|
||||
) -> Result<u64, RrdpFetchError> {
|
||||
self.fetch_to_writer(uri, out)
|
||||
.map_err(RrdpFetchError::Fetch)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RrdpState {
|
||||
pub session_id: String,
|
||||
pub serial: u64,
|
||||
}
|
||||
|
||||
impl RrdpState {
|
||||
pub fn encode(&self) -> Result<Vec<u8>, String> {
|
||||
serde_cbor::to_vec(self).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub fn decode(bytes: &[u8]) -> Result<Self, String> {
|
||||
serde_cbor::from_slice(bytes).map_err(|e| e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn load_rrdp_local_state(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
) -> Result<Option<RrdpState>, String> {
|
||||
if let Some(record) = store
|
||||
.get_rrdp_source_record(notification_uri)
|
||||
.map_err(|e| e.to_string())?
|
||||
&& let (Some(session_id), Some(serial)) = (record.last_session_id, record.last_serial) {
|
||||
return Ok(Some(RrdpState { session_id, serial }));
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
pub(crate) fn persist_rrdp_local_state(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
state: &RrdpState,
|
||||
sync_state: RrdpSourceSyncState,
|
||||
last_snapshot_uri: Option<&str>,
|
||||
last_snapshot_hash_hex: Option<&str>,
|
||||
) -> Result<(), String> {
|
||||
update_rrdp_source_record_on_success(
|
||||
store,
|
||||
notification_uri,
|
||||
state.session_id.as_str(),
|
||||
state.serial,
|
||||
sync_state,
|
||||
last_snapshot_uri,
|
||||
last_snapshot_hash_hex,
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct NotificationSnapshot {
|
||||
pub session_id: Uuid,
|
||||
pub serial: u64,
|
||||
pub snapshot_uri: String,
|
||||
pub snapshot_hash_sha256: [u8; 32],
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct NotificationDeltaRef {
|
||||
pub serial: u64,
|
||||
pub uri: String,
|
||||
pub hash_sha256: [u8; 32],
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Notification {
|
||||
pub session_id: Uuid,
|
||||
pub serial: u64,
|
||||
pub snapshot_uri: String,
|
||||
pub snapshot_hash_sha256: [u8; 32],
|
||||
/// Deltas referenced by the notification file, sorted by serial ascending.
|
||||
///
|
||||
/// If present, this list is guaranteed to be contiguous and to end at `serial`
|
||||
/// (RFC 8182 §3.5.1.3).
|
||||
pub deltas: Vec<NotificationDeltaRef>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum DeltaElement {
|
||||
Publish {
|
||||
uri: String,
|
||||
hash_sha256: Option<[u8; 32]>,
|
||||
bytes: Vec<u8>,
|
||||
},
|
||||
Withdraw {
|
||||
uri: String,
|
||||
hash_sha256: [u8; 32],
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct DeltaFile {
|
||||
pub session_id: Uuid,
|
||||
pub serial: u64,
|
||||
pub elements: Vec<DeltaElement>,
|
||||
}
|
||||
|
||||
pub fn parse_notification(xml: &[u8]) -> Result<Notification, RrdpError> {
|
||||
let doc = parse_rrdp_xml(xml)?;
|
||||
let root = doc.root_element();
|
||||
if root.tag_name().name() != "notification" {
|
||||
return Err(RrdpError::UnexpectedRoot(
|
||||
root.tag_name().name().to_string(),
|
||||
));
|
||||
}
|
||||
validate_root_common(&root)?;
|
||||
|
||||
let session_id = parse_uuid_attr(&root, "session_id")?;
|
||||
let serial = parse_u64_attr(&root, "serial")?;
|
||||
|
||||
let snapshots: Vec<_> = root
|
||||
.children()
|
||||
.filter(|n| n.is_element() && n.tag_name().name() == "snapshot")
|
||||
.collect();
|
||||
if snapshots.len() != 1 {
|
||||
return Err(RrdpError::SnapshotCountInvalid);
|
||||
}
|
||||
let snapshot = snapshots[0];
|
||||
|
||||
let snapshot_uri = snapshot
|
||||
.attribute("uri")
|
||||
.ok_or(RrdpError::SnapshotUriMissing)?
|
||||
.to_string();
|
||||
let snapshot_hash_hex = snapshot
|
||||
.attribute("hash")
|
||||
.ok_or(RrdpError::SnapshotHashMissing)?;
|
||||
let snapshot_hash_sha256 = parse_sha256_hex_snapshot(snapshot_hash_hex)?;
|
||||
|
||||
let mut deltas: Vec<NotificationDeltaRef> = Vec::new();
|
||||
for d in root
|
||||
.children()
|
||||
.filter(|n| n.is_element() && n.tag_name().name() == "delta")
|
||||
{
|
||||
let delta_serial = d
|
||||
.attribute("serial")
|
||||
.ok_or(RrdpError::DeltaRefSerialMissing)?;
|
||||
let delta_serial = parse_u64_str(delta_serial)?;
|
||||
if delta_serial > serial {
|
||||
return Err(RrdpError::DeltaRefSerialTooHigh {
|
||||
delta_serial,
|
||||
notification_serial: serial,
|
||||
});
|
||||
}
|
||||
let uri = d.attribute("uri").ok_or(RrdpError::DeltaRefUriMissing)?;
|
||||
let hash = d.attribute("hash").ok_or(RrdpError::DeltaRefHashMissing)?;
|
||||
let hash_sha256 = parse_sha256_hex_delta_ref(hash)?;
|
||||
|
||||
deltas.push(NotificationDeltaRef {
|
||||
serial: delta_serial,
|
||||
uri: uri.to_string(),
|
||||
hash_sha256,
|
||||
});
|
||||
}
|
||||
|
||||
if !deltas.is_empty() {
|
||||
let mut seen: std::collections::HashSet<u64> = std::collections::HashSet::new();
|
||||
let mut min_serial = u64::MAX;
|
||||
let mut max_serial = 0u64;
|
||||
for d in &deltas {
|
||||
if !seen.insert(d.serial) {
|
||||
return Err(RrdpError::DeltaRefSerialDuplicate(d.serial));
|
||||
}
|
||||
min_serial = min_serial.min(d.serial);
|
||||
max_serial = max_serial.max(d.serial);
|
||||
}
|
||||
if max_serial != serial {
|
||||
return Err(RrdpError::DeltaRefChainDoesNotEndAtNotificationSerial {
|
||||
max_delta_serial: max_serial,
|
||||
notification_serial: serial,
|
||||
});
|
||||
}
|
||||
for s in min_serial..=serial {
|
||||
if !seen.contains(&s) {
|
||||
return Err(RrdpError::DeltaRefChainNotContiguous {
|
||||
min_serial,
|
||||
notification_serial: serial,
|
||||
missing_serial: s,
|
||||
});
|
||||
}
|
||||
}
|
||||
deltas.sort_by_key(|d| d.serial);
|
||||
}
|
||||
|
||||
Ok(Notification {
|
||||
session_id,
|
||||
serial,
|
||||
snapshot_uri,
|
||||
snapshot_hash_sha256,
|
||||
deltas,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn parse_notification_snapshot(xml: &[u8]) -> Result<NotificationSnapshot, RrdpError> {
|
||||
let n = parse_notification(xml)?;
|
||||
Ok(NotificationSnapshot {
|
||||
session_id: n.session_id,
|
||||
serial: n.serial,
|
||||
snapshot_uri: n.snapshot_uri,
|
||||
snapshot_hash_sha256: n.snapshot_hash_sha256,
|
||||
})
|
||||
}
|
||||
|
||||
/// Validate every URI embedded in a notification before reading mutable local
|
||||
/// RRDP state or fetching a snapshot/delta. RFC 9674 §3.2 requires the
|
||||
/// notification, snapshot, and all delta URIs to share one origin.
|
||||
pub fn validate_notification_references(
|
||||
notification_uri: &str,
|
||||
notification: &Notification,
|
||||
) -> Result<RrdpOrigin, RrdpError> {
|
||||
let notification_origin = RrdpOrigin::parse(notification_uri, RrdpResourceKind::Notification)?;
|
||||
validate_reference_origin(
|
||||
¬ification_origin,
|
||||
¬ification.snapshot_uri,
|
||||
RrdpResourceKind::Snapshot,
|
||||
)?;
|
||||
for delta in ¬ification.deltas {
|
||||
validate_reference_origin(¬ification_origin, &delta.uri, RrdpResourceKind::Delta)?;
|
||||
}
|
||||
Ok(notification_origin)
|
||||
}
|
||||
|
||||
fn validate_reference_origin(
|
||||
notification_origin: &RrdpOrigin,
|
||||
uri: &str,
|
||||
resource: RrdpResourceKind,
|
||||
) -> Result<(), RrdpError> {
|
||||
let resource_origin = RrdpOrigin::parse(uri, resource)?;
|
||||
if &resource_origin != notification_origin {
|
||||
crate::logging::progress::emit!(
|
||||
"rrdp_cross_origin_rejected",
|
||||
serde_json::json!({
|
||||
"resource": resource.to_string(),
|
||||
"notification_origin": notification_origin.to_string(),
|
||||
"resource_origin": resource_origin.to_string(),
|
||||
"reason": "direct_reference",
|
||||
"rfc": "RFC 9674 §3.2",
|
||||
}),
|
||||
);
|
||||
return Err(RrdpError::CrossOriginReference {
|
||||
resource,
|
||||
notification_origin: notification_origin.to_string(),
|
||||
resource_origin: resource_origin.to_string(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn parse_delta_file(xml: &[u8]) -> Result<DeltaFile, RrdpError> {
|
||||
let doc = parse_rrdp_xml(xml)?;
|
||||
let root = doc.root_element();
|
||||
if root.tag_name().name() != "delta" {
|
||||
return Err(RrdpError::UnexpectedRoot(
|
||||
root.tag_name().name().to_string(),
|
||||
));
|
||||
}
|
||||
validate_root_common(&root)?;
|
||||
|
||||
let session_id = parse_uuid_attr(&root, "session_id")?;
|
||||
let serial = parse_u64_attr(&root, "serial")?;
|
||||
|
||||
let mut elements: Vec<DeltaElement> = Vec::new();
|
||||
for child in root.children().filter(|n| n.is_element()) {
|
||||
match child.tag_name().name() {
|
||||
"publish" => {
|
||||
let uri = child
|
||||
.attribute("uri")
|
||||
.ok_or(RrdpError::DeltaPublishUriMissing)?
|
||||
.to_string();
|
||||
let hash_sha256 = child
|
||||
.attribute("hash")
|
||||
.map(parse_sha256_hex_delta_publish)
|
||||
.transpose()?;
|
||||
|
||||
let content_b64 =
|
||||
collect_element_text(&child).ok_or(RrdpError::DeltaPublishContentMissing)?;
|
||||
let content_b64 = strip_all_ascii_whitespace(&content_b64);
|
||||
if content_b64.is_empty() {
|
||||
return Err(RrdpError::DeltaPublishContentMissing);
|
||||
}
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(content_b64.as_bytes())
|
||||
.map_err(|e| RrdpError::DeltaPublishBase64(e.to_string()))?;
|
||||
|
||||
elements.push(DeltaElement::Publish {
|
||||
uri,
|
||||
hash_sha256,
|
||||
bytes,
|
||||
});
|
||||
}
|
||||
"withdraw" => {
|
||||
let uri = child
|
||||
.attribute("uri")
|
||||
.ok_or(RrdpError::DeltaWithdrawUriMissing)?
|
||||
.to_string();
|
||||
let hash = child
|
||||
.attribute("hash")
|
||||
.ok_or(RrdpError::DeltaWithdrawHashMissing)?;
|
||||
let hash_sha256 = parse_sha256_hex_delta_withdraw(hash)?;
|
||||
|
||||
if let Some(s) = collect_element_text(&child)
|
||||
&& !strip_all_ascii_whitespace(&s).is_empty() {
|
||||
return Err(RrdpError::DeltaWithdrawUnexpectedContent);
|
||||
}
|
||||
|
||||
elements.push(DeltaElement::Withdraw { uri, hash_sha256 });
|
||||
}
|
||||
other => return Err(RrdpError::DeltaUnexpectedElement(other.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
if elements.is_empty() {
|
||||
return Err(RrdpError::DeltaNoElements);
|
||||
}
|
||||
|
||||
Ok(DeltaFile {
|
||||
session_id,
|
||||
serial,
|
||||
elements,
|
||||
})
|
||||
}
|
||||
375
src/repository/sync/rrdp/notification_sync.rs
Normal file
375
src/repository/sync/rrdp/notification_sync.rs
Normal file
@@ -0,0 +1,375 @@
|
||||
// RRDP notification, delta-chain, and state synchronization.
|
||||
|
||||
pub fn sync_from_notification(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
notification_xml: &[u8],
|
||||
fetcher: &dyn Fetcher,
|
||||
) -> RrdpSyncResult<usize> {
|
||||
sync_from_notification_inner(
|
||||
store,
|
||||
notification_uri,
|
||||
None,
|
||||
notification_xml,
|
||||
fetcher,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn sync_from_notification_with_timing(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
notification_xml: &[u8],
|
||||
fetcher: &dyn Fetcher,
|
||||
timing: Option<&TimingHandle>,
|
||||
) -> RrdpSyncResult<usize> {
|
||||
sync_from_notification_inner(
|
||||
store,
|
||||
notification_uri,
|
||||
None,
|
||||
notification_xml,
|
||||
fetcher,
|
||||
timing,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn sync_from_notification_with_timing_and_download_log(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||
notification_xml: &[u8],
|
||||
fetcher: &dyn Fetcher,
|
||||
timing: Option<&TimingHandle>,
|
||||
download_log: Option<&DownloadLogHandle>,
|
||||
) -> RrdpSyncResult<usize> {
|
||||
sync_from_notification_inner(
|
||||
store,
|
||||
notification_uri,
|
||||
current_repo_index,
|
||||
notification_xml,
|
||||
fetcher,
|
||||
timing,
|
||||
download_log,
|
||||
)
|
||||
}
|
||||
|
||||
fn sync_from_notification_inner(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||
notification_xml: &[u8],
|
||||
fetcher: &dyn Fetcher,
|
||||
timing: Option<&TimingHandle>,
|
||||
download_log: Option<&DownloadLogHandle>,
|
||||
) -> RrdpSyncResult<usize> {
|
||||
let _parse_step = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_rrdp_repo_step(notification_uri, "parse_notification"));
|
||||
let _parse_total = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_phase("rrdp_parse_notification_total"));
|
||||
let notif = parse_notification(notification_xml)?;
|
||||
let notification_origin = match validate_notification_references(notification_uri, ¬if) {
|
||||
Ok(origin) => origin,
|
||||
Err(e) => {
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count(
|
||||
"rrdp_rejected_cross_origin_total",
|
||||
e.is_cross_origin_violation() as u64,
|
||||
);
|
||||
}
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
drop(_parse_step);
|
||||
drop(_parse_total);
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count(
|
||||
"rrdp_notification_delta_refs_total",
|
||||
notif.deltas.len() as u64,
|
||||
);
|
||||
}
|
||||
|
||||
let _read_state_step = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_rrdp_repo_step(notification_uri, "read_state"));
|
||||
let _read_state_total = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_phase("rrdp_read_state_total"));
|
||||
let state = load_rrdp_local_state(store, notification_uri).map_err(RrdpSyncError::Storage)?;
|
||||
drop(_read_state_step);
|
||||
drop(_read_state_total);
|
||||
|
||||
let same_session_state = state
|
||||
.as_ref()
|
||||
.filter(|s| s.session_id == notif.session_id.to_string());
|
||||
|
||||
if let Some(s) = same_session_state {
|
||||
if s.serial == notif.serial {
|
||||
let _hydrate_step = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_rrdp_repo_step(notification_uri, "hydrate_current_index"));
|
||||
let _hydrate_total = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_phase("rrdp_hydrate_current_index_total"));
|
||||
let hydrated = hydrate_current_repo_index_from_rrdp_members(
|
||||
store,
|
||||
notification_uri,
|
||||
current_repo_index,
|
||||
)?;
|
||||
drop(_hydrate_step);
|
||||
drop(_hydrate_total);
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_current_index_hydrated_objects_total", hydrated as u64);
|
||||
}
|
||||
return Ok(0);
|
||||
}
|
||||
if s.serial > notif.serial {
|
||||
return Err(RrdpError::NotificationSerialRollback {
|
||||
old: s.serial,
|
||||
new: notif.serial,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(s) = same_session_state {
|
||||
// RFC 8182 §3.4.1: if session matches, MAY use deltas when a contiguous chain from the
|
||||
// last processed serial to the current serial can be processed (i.e. deltas cover the gap).
|
||||
let want_first = s.serial + 1;
|
||||
let want_last = notif.serial;
|
||||
|
||||
if want_first <= want_last && !notif.deltas.is_empty() {
|
||||
let min_serial = notif.deltas[0].serial;
|
||||
let max_serial = notif.deltas[notif.deltas.len() - 1].serial;
|
||||
|
||||
// `parse_notification` guarantees contiguity and max==notif.serial when deltas exist.
|
||||
if max_serial == notif.serial && want_first >= min_serial {
|
||||
// Fetch all required delta files first so a network failure doesn't leave us with
|
||||
// partially applied deltas and no snapshot fallback.
|
||||
let _fetch_d_step = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_deltas"));
|
||||
let _fetch_d_total = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_phase("rrdp_fetch_deltas_total"));
|
||||
let mut fetched: Vec<(u64, [u8; 32], Vec<u8>)> =
|
||||
Vec::with_capacity((want_last - want_first + 1) as usize);
|
||||
let mut fetch_ok = true;
|
||||
for serial in want_first..=want_last {
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_delta_fetch_attempted_total", 1);
|
||||
}
|
||||
let idx = (serial - min_serial) as usize;
|
||||
let dref = match notif.deltas.get(idx) {
|
||||
Some(v) if v.serial == serial => v,
|
||||
_ => {
|
||||
fetch_ok = false;
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
let mut dl_span = download_log
|
||||
.map(|dl| dl.span_download(AuditDownloadKind::RrdpDelta, &dref.uri));
|
||||
match fetcher.fetch_rrdp(
|
||||
RrdpResourceKind::Delta,
|
||||
&dref.uri,
|
||||
¬ification_origin,
|
||||
) {
|
||||
Ok(bytes) => {
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_delta_fetch_ok_total", 1);
|
||||
t.record_count("rrdp_delta_bytes_total", bytes.len() as u64);
|
||||
}
|
||||
if let Some(s) = dl_span.as_mut() {
|
||||
s.set_bytes(bytes.len() as u64);
|
||||
s.set_ok();
|
||||
}
|
||||
fetched.push((serial, dref.hash_sha256, bytes))
|
||||
}
|
||||
Err(RrdpFetchError::Fetch(e)) => {
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_delta_fetch_fail_total", 1);
|
||||
}
|
||||
if let Some(s) = dl_span.as_mut() {
|
||||
s.set_err(e.clone());
|
||||
}
|
||||
fetch_ok = false;
|
||||
break;
|
||||
}
|
||||
Err(RrdpFetchError::Rrdp(e)) => return Err(e.into()),
|
||||
}
|
||||
}
|
||||
drop(_fetch_d_step);
|
||||
drop(_fetch_d_total);
|
||||
|
||||
if fetch_ok {
|
||||
let _apply_d_step = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_rrdp_repo_step(notification_uri, "apply_deltas"));
|
||||
let _apply_d_total = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_phase("rrdp_apply_deltas_total"));
|
||||
let mut applied_total = 0usize;
|
||||
let mut ok = true;
|
||||
for (serial, expected_hash, bytes) in &fetched {
|
||||
match apply_delta(
|
||||
store,
|
||||
notification_uri,
|
||||
current_repo_index,
|
||||
bytes.as_slice(),
|
||||
*expected_hash,
|
||||
notif.session_id,
|
||||
*serial,
|
||||
) {
|
||||
Ok(n) => applied_total += n,
|
||||
Err(_) => {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(_apply_d_step);
|
||||
drop(_apply_d_total);
|
||||
|
||||
if ok {
|
||||
let _write_state_step = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_rrdp_repo_step(notification_uri, "write_state"));
|
||||
let _write_state_total = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_phase("rrdp_write_state_total"));
|
||||
let new_state = RrdpState {
|
||||
session_id: notif.session_id.to_string(),
|
||||
serial: notif.serial,
|
||||
};
|
||||
persist_rrdp_local_state(
|
||||
store,
|
||||
notification_uri,
|
||||
&new_state,
|
||||
RrdpSourceSyncState::DeltaReady,
|
||||
Some(¬if.snapshot_uri),
|
||||
Some(&hex::encode(notif.snapshot_hash_sha256)),
|
||||
)
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
drop(_write_state_step);
|
||||
drop(_write_state_total);
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_delta_ops_applied_total", applied_total as u64);
|
||||
}
|
||||
let _hydrate_step = timing.as_ref().map(|t| {
|
||||
t.span_rrdp_repo_step(notification_uri, "hydrate_current_index")
|
||||
});
|
||||
let _hydrate_total = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_phase("rrdp_hydrate_current_index_total"));
|
||||
let hydrated = hydrate_current_repo_index_from_rrdp_members(
|
||||
store,
|
||||
notification_uri,
|
||||
current_repo_index,
|
||||
)?;
|
||||
drop(_hydrate_step);
|
||||
drop(_hydrate_total);
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count(
|
||||
"rrdp_current_index_hydrated_objects_total",
|
||||
hydrated as u64,
|
||||
);
|
||||
}
|
||||
return Ok(applied_total);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Snapshot fallback (RFC 8182 §3.4.3).
|
||||
let _fetch_step = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_snapshot"));
|
||||
let _fetch_total = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_phase("rrdp_fetch_snapshot_total"));
|
||||
let mut dl_span = download_log
|
||||
.map(|dl| dl.span_download(AuditDownloadKind::RrdpSnapshot, ¬if.snapshot_uri));
|
||||
let (snapshot_file, _snapshot_bytes) = match fetch_snapshot_into_tempfile(
|
||||
fetcher,
|
||||
¬if.snapshot_uri,
|
||||
¬if.snapshot_hash_sha256,
|
||||
¬ification_origin,
|
||||
) {
|
||||
Ok(v) => {
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_snapshot_fetch_ok_total", 1);
|
||||
t.record_count("rrdp_snapshot_bytes_total", v.1);
|
||||
}
|
||||
if let Some(s) = dl_span.as_mut() {
|
||||
s.set_bytes(v.1);
|
||||
s.set_ok();
|
||||
}
|
||||
v
|
||||
}
|
||||
Err(RrdpSyncError::Fetch(e)) => {
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_snapshot_fetch_fail_total", 1);
|
||||
}
|
||||
if let Some(s) = dl_span.as_mut() {
|
||||
s.set_err(e.clone());
|
||||
}
|
||||
return Err(RrdpSyncError::Fetch(e));
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
drop(_fetch_step);
|
||||
drop(_fetch_total);
|
||||
|
||||
let _apply_step = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_rrdp_repo_step(notification_uri, "apply_snapshot"));
|
||||
let _apply_total = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_phase("rrdp_apply_snapshot_total"));
|
||||
let published = apply_snapshot_from_bufread(
|
||||
store,
|
||||
notification_uri,
|
||||
current_repo_index,
|
||||
std::io::BufReader::new(
|
||||
snapshot_file
|
||||
.reopen()
|
||||
.map_err(|e| RrdpSyncError::Fetch(format!("tempfile reopen failed: {e}")))?,
|
||||
),
|
||||
notif.session_id,
|
||||
notif.serial,
|
||||
)?;
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_snapshot_objects_applied_total", published as u64);
|
||||
}
|
||||
drop(_apply_step);
|
||||
drop(_apply_total);
|
||||
|
||||
let _write_state_step = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_rrdp_repo_step(notification_uri, "write_state"));
|
||||
let _write_state_total = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_phase("rrdp_write_state_total"));
|
||||
let new_state = RrdpState {
|
||||
session_id: notif.session_id.to_string(),
|
||||
serial: notif.serial,
|
||||
};
|
||||
persist_rrdp_local_state(
|
||||
store,
|
||||
notification_uri,
|
||||
&new_state,
|
||||
RrdpSourceSyncState::SnapshotOnly,
|
||||
Some(¬if.snapshot_uri),
|
||||
Some(&hex::encode(notif.snapshot_hash_sha256)),
|
||||
)
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
drop(_write_state_step);
|
||||
drop(_write_state_total);
|
||||
|
||||
Ok(published)
|
||||
}
|
||||
87
src/repository/sync/rrdp/parse_helpers.rs
Normal file
87
src/repository/sync/rrdp/parse_helpers.rs
Normal file
@@ -0,0 +1,87 @@
|
||||
// Shared XML, URI, and hash parsing helpers.
|
||||
|
||||
#[cfg(test)]
|
||||
use snapshot_apply::apply_snapshot;
|
||||
use snapshot_apply::{apply_snapshot_from_bufread, fetch_snapshot_into_tempfile};
|
||||
|
||||
fn parse_rrdp_xml(xml: &[u8]) -> Result<roxmltree::Document<'_>, RrdpError> {
|
||||
if xml.iter().any(|&b| b > 0x7F) {
|
||||
return Err(RrdpError::NotAscii);
|
||||
}
|
||||
let s = std::str::from_utf8(xml).map_err(|e| RrdpError::Xml(e.to_string()))?;
|
||||
roxmltree::Document::parse(s).map_err(|e| RrdpError::Xml(e.to_string()))
|
||||
}
|
||||
|
||||
fn validate_root_common(root: &roxmltree::Node<'_, '_>) -> Result<(), RrdpError> {
|
||||
let ns = root.default_namespace().unwrap_or("").to_string();
|
||||
if ns != RRDP_XMLNS {
|
||||
return Err(RrdpError::InvalidNamespace(ns));
|
||||
}
|
||||
|
||||
let version = root.attribute("version").unwrap_or("");
|
||||
if version != "1" {
|
||||
return Err(RrdpError::InvalidVersion(version.to_string()));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_uuid_attr(root: &roxmltree::Node<'_, '_>, name: &'static str) -> Result<Uuid, RrdpError> {
|
||||
let s = root.attribute(name).unwrap_or("");
|
||||
Uuid::parse_str(s).map_err(|_e| RrdpError::InvalidSessionId(s.to_string()))
|
||||
}
|
||||
|
||||
fn parse_u64_attr(root: &roxmltree::Node<'_, '_>, name: &'static str) -> Result<u64, RrdpError> {
|
||||
let s = root.attribute(name).unwrap_or("");
|
||||
parse_u64_str(s)
|
||||
}
|
||||
|
||||
fn parse_u64_str(s: &str) -> Result<u64, RrdpError> {
|
||||
let v = s
|
||||
.parse::<u64>()
|
||||
.map_err(|_e| RrdpError::InvalidSerial(s.to_string()))?;
|
||||
if v == 0 {
|
||||
return Err(RrdpError::InvalidSerial(s.to_string()));
|
||||
}
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
fn parse_sha256_hex_impl(s: &str, invalid: fn(String) -> RrdpError) -> Result<[u8; 32], RrdpError> {
|
||||
let bytes = hex::decode(s).map_err(|_e| invalid(s.to_string()))?;
|
||||
if bytes.len() != 32 {
|
||||
return Err(invalid(s.to_string()));
|
||||
}
|
||||
let mut out = [0u8; 32];
|
||||
out.copy_from_slice(&bytes);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn parse_sha256_hex_snapshot(s: &str) -> Result<[u8; 32], RrdpError> {
|
||||
parse_sha256_hex_impl(s, RrdpError::SnapshotHashInvalid)
|
||||
}
|
||||
|
||||
fn parse_sha256_hex_delta_ref(s: &str) -> Result<[u8; 32], RrdpError> {
|
||||
parse_sha256_hex_impl(s, RrdpError::DeltaRefHashInvalid)
|
||||
}
|
||||
|
||||
fn parse_sha256_hex_delta_publish(s: &str) -> Result<[u8; 32], RrdpError> {
|
||||
parse_sha256_hex_impl(s, RrdpError::DeltaPublishHashInvalid)
|
||||
}
|
||||
|
||||
fn parse_sha256_hex_delta_withdraw(s: &str) -> Result<[u8; 32], RrdpError> {
|
||||
parse_sha256_hex_impl(s, RrdpError::DeltaWithdrawHashInvalid)
|
||||
}
|
||||
|
||||
fn collect_element_text(node: &roxmltree::Node<'_, '_>) -> Option<String> {
|
||||
let mut out = String::new();
|
||||
for child in node.children() {
|
||||
if child.is_text() {
|
||||
out.push_str(child.text().unwrap_or(""));
|
||||
}
|
||||
}
|
||||
if out.is_empty() { None } else { Some(out) }
|
||||
}
|
||||
|
||||
fn strip_all_ascii_whitespace(s: &str) -> String {
|
||||
s.chars().filter(|c| !c.is_ascii_whitespace()).collect()
|
||||
}
|
||||
454
src/repository/sync/rrdp/snapshot_apply.rs
Normal file
454
src/repository/sync/rrdp/snapshot_apply.rs
Normal file
@@ -0,0 +1,454 @@
|
||||
use base64::Engine;
|
||||
use quick_xml::Reader;
|
||||
use quick_xml::events::Event;
|
||||
use sha2::Digest;
|
||||
use std::io::{BufRead, Seek, SeekFrom, Write};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::repository::current_repo_index::CurrentRepoIndexHandle;
|
||||
use crate::repository::storage::RocksStore;
|
||||
use crate::repository::sync::store_projection::{
|
||||
build_repository_view_present_entry, build_repository_view_withdrawn_entry,
|
||||
build_rrdp_source_member_present_record, build_rrdp_source_member_withdrawn_record,
|
||||
build_rrdp_uri_owner_active_record, build_rrdp_uri_owner_withdrawn_record, compute_sha256_hex,
|
||||
current_rrdp_owner_is, ensure_rrdp_uri_can_be_owned_by, prepare_repo_bytes_batch,
|
||||
};
|
||||
|
||||
use super::{
|
||||
Fetcher, RRDP_SNAPSHOT_APPLY_BATCH_SIZE, RRDP_XMLNS, RrdpError, RrdpFetchError, RrdpOrigin,
|
||||
RrdpResourceKind, RrdpSyncError, parse_u64_str, strip_all_ascii_whitespace,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn apply_snapshot(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||
snapshot_xml: &[u8],
|
||||
expected_session_id: Uuid,
|
||||
expected_serial: u64,
|
||||
) -> Result<usize, RrdpSyncError> {
|
||||
if snapshot_xml.iter().any(|&b| b > 0x7F) {
|
||||
return Err(RrdpError::NotAscii.into());
|
||||
}
|
||||
apply_snapshot_from_bufread(
|
||||
store,
|
||||
notification_uri,
|
||||
current_repo_index,
|
||||
std::io::Cursor::new(snapshot_xml),
|
||||
expected_session_id,
|
||||
expected_serial,
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn apply_snapshot_from_bufread<R: BufRead>(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||
input: R,
|
||||
expected_session_id: Uuid,
|
||||
expected_serial: u64,
|
||||
) -> Result<usize, RrdpSyncError> {
|
||||
let previous_members: Vec<String> = store
|
||||
.list_current_rrdp_source_members(notification_uri)
|
||||
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?
|
||||
.into_iter()
|
||||
.map(|record| record.rsync_uri)
|
||||
.collect();
|
||||
let session_id = expected_session_id.to_string();
|
||||
let mut new_set: std::collections::HashSet<String> = std::collections::HashSet::new();
|
||||
let mut batch_published: Vec<(String, Vec<u8>)> =
|
||||
Vec::with_capacity(RRDP_SNAPSHOT_APPLY_BATCH_SIZE);
|
||||
let mut published_count = 0usize;
|
||||
|
||||
let mut reader = Reader::from_reader(input);
|
||||
reader.config_mut().trim_text(false);
|
||||
let mut buf = Vec::new();
|
||||
let mut root_seen = false;
|
||||
let mut in_publish = false;
|
||||
let mut publish_nested_depth = 0usize;
|
||||
let mut current_publish_uri: Option<String> = None;
|
||||
let mut current_publish_text = String::new();
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(e)) => {
|
||||
let local_name = e.local_name();
|
||||
let local_name = local_name.as_ref();
|
||||
if !root_seen {
|
||||
root_seen = true;
|
||||
if local_name != b"snapshot" {
|
||||
let got = String::from_utf8_lossy(local_name).to_string();
|
||||
return Err(RrdpError::UnexpectedRoot(got).into());
|
||||
}
|
||||
let mut xmlns = String::new();
|
||||
let mut version = String::new();
|
||||
let mut session_id_attr = String::new();
|
||||
let mut serial_attr = String::new();
|
||||
for attr in e.attributes().with_checks(false) {
|
||||
let attr = match attr {
|
||||
Ok(attr) => attr,
|
||||
Err(e) => return Err(RrdpError::Xml(e.to_string()).into()),
|
||||
};
|
||||
let key = attr.key.as_ref();
|
||||
let value = attr
|
||||
.decode_and_unescape_value(reader.decoder())
|
||||
.map_err(|e| RrdpError::Xml(e.to_string()))?
|
||||
.into_owned();
|
||||
match key {
|
||||
b"xmlns" => xmlns = value,
|
||||
b"version" => version = value,
|
||||
b"session_id" => session_id_attr = value,
|
||||
b"serial" => serial_attr = value,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if xmlns != RRDP_XMLNS {
|
||||
return Err(RrdpError::InvalidNamespace(xmlns).into());
|
||||
}
|
||||
if version != "1" {
|
||||
return Err(RrdpError::InvalidVersion(version).into());
|
||||
}
|
||||
let got_session_id = Uuid::parse_str(&session_id_attr)
|
||||
.map_err(|_| RrdpError::InvalidSessionId(session_id_attr.clone()))?;
|
||||
if got_session_id != expected_session_id {
|
||||
return Err(RrdpError::SnapshotSessionIdMismatch {
|
||||
expected: expected_session_id.to_string(),
|
||||
got: got_session_id.to_string(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let got_serial = parse_u64_str(&serial_attr)?;
|
||||
if got_serial != expected_serial {
|
||||
return Err(RrdpError::SnapshotSerialMismatch {
|
||||
expected: expected_serial,
|
||||
got: got_serial,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
} else if in_publish {
|
||||
publish_nested_depth += 1;
|
||||
} else if local_name == b"publish" {
|
||||
let mut uri = None;
|
||||
for attr in e.attributes().with_checks(false) {
|
||||
let attr = match attr {
|
||||
Ok(attr) => attr,
|
||||
Err(e) => return Err(RrdpError::Xml(e.to_string()).into()),
|
||||
};
|
||||
if attr.key.as_ref() == b"uri" {
|
||||
uri = Some(
|
||||
attr.decode_and_unescape_value(reader.decoder())
|
||||
.map_err(|e| RrdpError::Xml(e.to_string()))?
|
||||
.into_owned(),
|
||||
);
|
||||
}
|
||||
}
|
||||
let uri = uri.ok_or(RrdpError::PublishUriMissing)?;
|
||||
ensure_rrdp_uri_can_be_owned_by(store, notification_uri, &uri)
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
in_publish = true;
|
||||
publish_nested_depth = 0;
|
||||
current_publish_uri = Some(uri);
|
||||
current_publish_text.clear();
|
||||
}
|
||||
}
|
||||
Ok(Event::Empty(e)) => {
|
||||
let local_name = e.local_name();
|
||||
let local_name = local_name.as_ref();
|
||||
if !root_seen {
|
||||
let got = String::from_utf8_lossy(local_name).to_string();
|
||||
return Err(RrdpError::UnexpectedRoot(got).into());
|
||||
}
|
||||
if local_name == b"publish" {
|
||||
let mut has_uri = false;
|
||||
for attr in e.attributes().with_checks(false) {
|
||||
let attr = match attr {
|
||||
Ok(attr) => attr,
|
||||
Err(e) => return Err(RrdpError::Xml(e.to_string()).into()),
|
||||
};
|
||||
if attr.key.as_ref() == b"uri" {
|
||||
has_uri = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !has_uri {
|
||||
return Err(RrdpError::PublishUriMissing.into());
|
||||
}
|
||||
return Err(RrdpError::PublishContentMissing.into());
|
||||
}
|
||||
}
|
||||
Ok(Event::Text(e)) => {
|
||||
if in_publish && publish_nested_depth == 0 {
|
||||
let text = reader
|
||||
.decoder()
|
||||
.decode(e.as_ref())
|
||||
.map_err(|e| RrdpError::Xml(e.to_string()))?;
|
||||
current_publish_text.push_str(&text);
|
||||
}
|
||||
}
|
||||
Ok(Event::CData(e)) => {
|
||||
if in_publish && publish_nested_depth == 0 {
|
||||
let text = reader
|
||||
.decoder()
|
||||
.decode(e.as_ref())
|
||||
.map_err(|e| RrdpError::Xml(e.to_string()))?;
|
||||
current_publish_text.push_str(&text);
|
||||
}
|
||||
}
|
||||
Ok(Event::End(e)) => {
|
||||
let local_name = e.local_name();
|
||||
let local_name = local_name.as_ref();
|
||||
if in_publish {
|
||||
if publish_nested_depth > 0 {
|
||||
publish_nested_depth -= 1;
|
||||
} else if local_name == b"publish" {
|
||||
let uri = current_publish_uri
|
||||
.take()
|
||||
.ok_or_else(|| RrdpError::Xml("publish uri missing in state".into()))?;
|
||||
let content_b64 = strip_all_ascii_whitespace(¤t_publish_text);
|
||||
current_publish_text.clear();
|
||||
if content_b64.is_empty() {
|
||||
return Err(RrdpError::PublishContentMissing.into());
|
||||
}
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(content_b64.as_bytes())
|
||||
.map_err(|e| RrdpError::PublishBase64(e.to_string()))?;
|
||||
new_set.insert(uri.clone());
|
||||
batch_published.push((uri, bytes));
|
||||
published_count += 1;
|
||||
if batch_published.len() >= RRDP_SNAPSHOT_APPLY_BATCH_SIZE {
|
||||
flush_snapshot_publish_batch(
|
||||
store,
|
||||
notification_uri,
|
||||
current_repo_index,
|
||||
&session_id,
|
||||
expected_serial,
|
||||
&batch_published,
|
||||
)?;
|
||||
batch_published.clear();
|
||||
}
|
||||
in_publish = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
Ok(Event::Decl(_) | Event::PI(_) | Event::Comment(_) | Event::DocType(_)) => {}
|
||||
Err(e) => return Err(RrdpError::Xml(e.to_string()).into()),
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
|
||||
if !root_seen {
|
||||
return Err(RrdpError::Xml("missing root element".to_string()).into());
|
||||
}
|
||||
if in_publish {
|
||||
return Err(RrdpError::PublishContentMissing.into());
|
||||
}
|
||||
if !batch_published.is_empty() {
|
||||
flush_snapshot_publish_batch(
|
||||
store,
|
||||
notification_uri,
|
||||
current_repo_index,
|
||||
&session_id,
|
||||
expected_serial,
|
||||
&batch_published,
|
||||
)?;
|
||||
batch_published.clear();
|
||||
}
|
||||
|
||||
let mut withdrawn: Vec<(String, Option<String>)> = Vec::new();
|
||||
for old_uri in &previous_members {
|
||||
if new_set.contains(old_uri) {
|
||||
continue;
|
||||
}
|
||||
let previous_hash = store
|
||||
.get_repository_view_entry(old_uri)
|
||||
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?
|
||||
.and_then(|entry| entry.current_hash)
|
||||
.or_else(|| {
|
||||
store
|
||||
.load_current_object_bytes_by_uri(old_uri)
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|bytes| compute_sha256_hex(&bytes))
|
||||
});
|
||||
withdrawn.push((old_uri.clone(), previous_hash));
|
||||
}
|
||||
|
||||
let mut repository_view_entries = Vec::with_capacity(withdrawn.len());
|
||||
let mut member_records = Vec::with_capacity(withdrawn.len());
|
||||
let mut owner_records = Vec::with_capacity(withdrawn.len());
|
||||
for (uri, previous_hash) in withdrawn {
|
||||
member_records.push(build_rrdp_source_member_withdrawn_record(
|
||||
notification_uri,
|
||||
&session_id,
|
||||
expected_serial,
|
||||
&uri,
|
||||
previous_hash.clone(),
|
||||
));
|
||||
if current_rrdp_owner_is(store, notification_uri, &uri).map_err(RrdpSyncError::Storage)? {
|
||||
repository_view_entries.push(build_repository_view_withdrawn_entry(
|
||||
notification_uri,
|
||||
&uri,
|
||||
previous_hash.clone(),
|
||||
));
|
||||
owner_records.push(build_rrdp_uri_owner_withdrawn_record(
|
||||
notification_uri,
|
||||
&session_id,
|
||||
expected_serial,
|
||||
&uri,
|
||||
previous_hash,
|
||||
));
|
||||
}
|
||||
}
|
||||
store
|
||||
.put_projection_batch(&repository_view_entries, &member_records, &owner_records)
|
||||
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
|
||||
if let Some(index) = current_repo_index {
|
||||
index
|
||||
.write()
|
||||
.map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))?
|
||||
.apply_repository_view_entries(&repository_view_entries)
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
}
|
||||
|
||||
Ok(published_count)
|
||||
}
|
||||
|
||||
fn flush_snapshot_publish_batch(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||
session_id: &str,
|
||||
serial: u64,
|
||||
published: &[(String, Vec<u8>)],
|
||||
) -> Result<(), RrdpSyncError> {
|
||||
let prepared_bytes = prepare_repo_bytes_batch(published).map_err(RrdpSyncError::Storage)?;
|
||||
let mut repository_view_entries = Vec::with_capacity(published.len());
|
||||
let mut member_records = Vec::with_capacity(published.len());
|
||||
let mut owner_records = Vec::with_capacity(published.len());
|
||||
|
||||
for (uri, _bytes) in published {
|
||||
let current_hash = prepared_bytes
|
||||
.uri_to_hash
|
||||
.get(uri)
|
||||
.cloned()
|
||||
.ok_or_else(|| {
|
||||
RrdpSyncError::Storage(format!("missing raw_by_hash mapping for {uri}"))
|
||||
})?;
|
||||
repository_view_entries.push(build_repository_view_present_entry(
|
||||
notification_uri,
|
||||
uri,
|
||||
¤t_hash,
|
||||
));
|
||||
member_records.push(build_rrdp_source_member_present_record(
|
||||
notification_uri,
|
||||
session_id,
|
||||
serial,
|
||||
uri,
|
||||
¤t_hash,
|
||||
));
|
||||
owner_records.push(build_rrdp_uri_owner_active_record(
|
||||
notification_uri,
|
||||
session_id,
|
||||
serial,
|
||||
uri,
|
||||
¤t_hash,
|
||||
));
|
||||
}
|
||||
|
||||
store
|
||||
.put_blob_bytes_batch(&prepared_bytes.blobs_to_write)
|
||||
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
|
||||
store
|
||||
.put_projection_batch(&repository_view_entries, &member_records, &owner_records)
|
||||
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
|
||||
if let Some(index) = current_repo_index {
|
||||
index
|
||||
.write()
|
||||
.map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))?
|
||||
.apply_repository_view_entries(&repository_view_entries)
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const SNAPSHOT_NON_ASCII_ERROR: &str = "snapshot body contains non-ASCII bytes";
|
||||
|
||||
struct SnapshotSpoolWriter<'a, W: Write> {
|
||||
inner: &'a mut W,
|
||||
hasher: sha2::Sha256,
|
||||
bytes: u64,
|
||||
}
|
||||
|
||||
impl<'a, W: Write> SnapshotSpoolWriter<'a, W> {
|
||||
fn new(inner: &'a mut W) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
hasher: sha2::Sha256::new(),
|
||||
bytes: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn finalize_hash(self) -> [u8; 32] {
|
||||
let digest = self.hasher.finalize();
|
||||
let mut out = [0u8; 32];
|
||||
out.copy_from_slice(&digest);
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
impl<W: Write> Write for SnapshotSpoolWriter<'_, W> {
|
||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
if buf.iter().any(|&b| b > 0x7F) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
SNAPSHOT_NON_ASCII_ERROR,
|
||||
));
|
||||
}
|
||||
let n = self.inner.write(buf)?;
|
||||
self.hasher.update(&buf[..n]);
|
||||
self.bytes += n as u64;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn fetch_snapshot_into_tempfile(
|
||||
fetcher: &dyn Fetcher,
|
||||
snapshot_uri: &str,
|
||||
expected_hash_sha256: &[u8; 32],
|
||||
notification_origin: &RrdpOrigin,
|
||||
) -> Result<(tempfile::NamedTempFile, u64), RrdpSyncError> {
|
||||
let mut tmp = tempfile::NamedTempFile::new()
|
||||
.map_err(|e| RrdpSyncError::Fetch(format!("tempfile create failed: {e}")))?;
|
||||
let mut spool = SnapshotSpoolWriter::new(tmp.as_file_mut());
|
||||
let bytes_written = match fetcher.fetch_rrdp_to_writer(
|
||||
RrdpResourceKind::Snapshot,
|
||||
snapshot_uri,
|
||||
notification_origin,
|
||||
&mut spool,
|
||||
) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(RrdpFetchError::Fetch(e)) if e.contains(SNAPSHOT_NON_ASCII_ERROR) => {
|
||||
return Err(RrdpError::NotAscii.into());
|
||||
}
|
||||
Err(RrdpFetchError::Fetch(e)) => return Err(RrdpSyncError::Fetch(e)),
|
||||
Err(RrdpFetchError::Rrdp(e)) => return Err(e.into()),
|
||||
};
|
||||
let computed = spool.finalize_hash();
|
||||
if computed.as_slice() != expected_hash_sha256.as_slice() {
|
||||
return Err(RrdpError::SnapshotHashMismatch.into());
|
||||
}
|
||||
tmp.as_file_mut()
|
||||
.flush()
|
||||
.map_err(|e| RrdpSyncError::Fetch(format!("tempfile flush failed: {e}")))?;
|
||||
tmp.as_file_mut()
|
||||
.seek(SeekFrom::Start(0))
|
||||
.map_err(|e| RrdpSyncError::Fetch(format!("tempfile rewind failed: {e}")))?;
|
||||
Ok((tmp, bytes_written))
|
||||
}
|
||||
170
src/repository/sync/rrdp/snapshot_sync.rs
Normal file
170
src/repository/sync/rrdp/snapshot_sync.rs
Normal file
@@ -0,0 +1,170 @@
|
||||
// RRDP snapshot synchronization and timing instrumentation.
|
||||
|
||||
pub fn sync_from_notification_snapshot(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
notification_xml: &[u8],
|
||||
fetcher: &dyn Fetcher,
|
||||
) -> RrdpSyncResult<usize> {
|
||||
sync_from_notification_snapshot_inner(
|
||||
store,
|
||||
notification_uri,
|
||||
notification_xml,
|
||||
fetcher,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn sync_from_notification_snapshot_with_timing(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
notification_xml: &[u8],
|
||||
fetcher: &dyn Fetcher,
|
||||
timing: Option<&TimingHandle>,
|
||||
) -> RrdpSyncResult<usize> {
|
||||
sync_from_notification_snapshot_inner(
|
||||
store,
|
||||
notification_uri,
|
||||
notification_xml,
|
||||
fetcher,
|
||||
timing,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn sync_from_notification_snapshot_with_timing_and_download_log(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
notification_xml: &[u8],
|
||||
fetcher: &dyn Fetcher,
|
||||
timing: Option<&TimingHandle>,
|
||||
download_log: Option<&DownloadLogHandle>,
|
||||
) -> RrdpSyncResult<usize> {
|
||||
sync_from_notification_snapshot_inner(
|
||||
store,
|
||||
notification_uri,
|
||||
notification_xml,
|
||||
fetcher,
|
||||
timing,
|
||||
download_log,
|
||||
)
|
||||
}
|
||||
|
||||
fn sync_from_notification_snapshot_inner(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
notification_xml: &[u8],
|
||||
fetcher: &dyn Fetcher,
|
||||
timing: Option<&TimingHandle>,
|
||||
download_log: Option<&DownloadLogHandle>,
|
||||
) -> RrdpSyncResult<usize> {
|
||||
let _parse_step = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_rrdp_repo_step(notification_uri, "parse_notification_snapshot"));
|
||||
let _parse_total = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_phase("rrdp_parse_notification_total"));
|
||||
let notif = parse_notification(notification_xml)?;
|
||||
let notification_origin = match validate_notification_references(notification_uri, ¬if) {
|
||||
Ok(origin) => origin,
|
||||
Err(e) => {
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count(
|
||||
"rrdp_rejected_cross_origin_total",
|
||||
e.is_cross_origin_violation() as u64,
|
||||
);
|
||||
}
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
drop(_parse_step);
|
||||
drop(_parse_total);
|
||||
|
||||
let _fetch_step = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_snapshot"));
|
||||
let _fetch_total = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_phase("rrdp_fetch_snapshot_total"));
|
||||
let mut dl_span = download_log
|
||||
.map(|dl| dl.span_download(AuditDownloadKind::RrdpSnapshot, ¬if.snapshot_uri));
|
||||
let (snapshot_file, _snapshot_bytes) = match fetch_snapshot_into_tempfile(
|
||||
fetcher,
|
||||
¬if.snapshot_uri,
|
||||
¬if.snapshot_hash_sha256,
|
||||
¬ification_origin,
|
||||
) {
|
||||
Ok(v) => {
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_snapshot_fetch_ok_total", 1);
|
||||
t.record_count("rrdp_snapshot_bytes_total", v.1);
|
||||
}
|
||||
if let Some(s) = dl_span.as_mut() {
|
||||
s.set_bytes(v.1);
|
||||
s.set_ok();
|
||||
}
|
||||
v
|
||||
}
|
||||
Err(RrdpSyncError::Fetch(e)) => {
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_snapshot_fetch_fail_total", 1);
|
||||
}
|
||||
if let Some(s) = dl_span.as_mut() {
|
||||
s.set_err(e.clone());
|
||||
}
|
||||
return Err(RrdpSyncError::Fetch(e));
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
drop(_fetch_step);
|
||||
drop(_fetch_total);
|
||||
|
||||
let _apply_step = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_rrdp_repo_step(notification_uri, "apply_snapshot"));
|
||||
let _apply_total = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_phase("rrdp_apply_snapshot_total"));
|
||||
let published = apply_snapshot_from_bufread(
|
||||
store,
|
||||
notification_uri,
|
||||
None,
|
||||
std::io::BufReader::new(
|
||||
snapshot_file
|
||||
.reopen()
|
||||
.map_err(|e| RrdpSyncError::Fetch(format!("tempfile reopen failed: {e}")))?,
|
||||
),
|
||||
notif.session_id,
|
||||
notif.serial,
|
||||
)?;
|
||||
if let Some(t) = timing.as_ref() {
|
||||
t.record_count("rrdp_snapshot_objects_applied_total", published as u64);
|
||||
}
|
||||
drop(_apply_step);
|
||||
drop(_apply_total);
|
||||
|
||||
let _write_state_step = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_rrdp_repo_step(notification_uri, "write_state"));
|
||||
let _write_state_total = timing
|
||||
.as_ref()
|
||||
.map(|t| t.span_phase("rrdp_write_state_total"));
|
||||
let state = RrdpState {
|
||||
session_id: notif.session_id.to_string(),
|
||||
serial: notif.serial,
|
||||
};
|
||||
persist_rrdp_local_state(
|
||||
store,
|
||||
notification_uri,
|
||||
&state,
|
||||
RrdpSourceSyncState::SnapshotOnly,
|
||||
Some(¬if.snapshot_uri),
|
||||
Some(&hex::encode(notif.snapshot_hash_sha256)),
|
||||
)
|
||||
.map_err(RrdpSyncError::Storage)?;
|
||||
drop(_write_state_step);
|
||||
drop(_write_state_total);
|
||||
|
||||
Ok(published)
|
||||
}
|
||||
5
src/repository/sync/rrdp/tests.rs
Normal file
5
src/repository/sync/rrdp/tests.rs
Normal file
@@ -0,0 +1,5 @@
|
||||
// RRDP tests are grouped by parsing, delta application, and synchronization.
|
||||
include!("tests_parts/parsing.rs");
|
||||
include!("tests_parts/delta_apply.rs");
|
||||
include!("tests_parts/sync.rs");
|
||||
include!("tests_parts/edge_cases.rs");
|
||||
359
src/repository/sync/rrdp/tests_parts/delta_apply.rs
Normal file
359
src/repository/sync/rrdp/tests_parts/delta_apply.rs
Normal file
@@ -0,0 +1,359 @@
|
||||
// RRDP test group: delta apply.
|
||||
|
||||
#[test]
|
||||
fn apply_delta_applies_publish_replace_and_withdraw_with_membership_checks() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
|
||||
let sid = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
|
||||
// Start from snapshot state with a + b
|
||||
let snapshot_uri = "https://example.net/snapshot.xml";
|
||||
let snapshot = snapshot_xml(
|
||||
sid,
|
||||
1,
|
||||
&[
|
||||
("rsync://example.net/repo/a.mft", b"a1"),
|
||||
("rsync://example.net/repo/b.roa", b"b1"),
|
||||
],
|
||||
);
|
||||
let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot));
|
||||
let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash);
|
||||
let fetcher = MapFetcher {
|
||||
map: HashMap::from([(snapshot_uri.to_string(), snapshot)]),
|
||||
};
|
||||
sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("sync snapshot");
|
||||
|
||||
let old_b = store
|
||||
.load_current_object_bytes_by_uri("rsync://example.net/repo/b.roa")
|
||||
.expect("load current b")
|
||||
.expect("b present");
|
||||
let old_b_hash = hex::encode(sha2::Sha256::digest(old_b.as_slice()));
|
||||
|
||||
let withdraw_a_hash = hex::encode(sha2::Sha256::digest(b"a1".as_slice()));
|
||||
let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2");
|
||||
let replace_b_b64 = base64::engine::general_purpose::STANDARD.encode(b"b2");
|
||||
|
||||
let delta = delta_xml(
|
||||
sid,
|
||||
2,
|
||||
&[
|
||||
&format!(
|
||||
r#"<withdraw uri="rsync://example.net/repo/a.mft" hash="{withdraw_a_hash}"/>"#
|
||||
),
|
||||
&format!(
|
||||
r#"<publish uri="rsync://example.net/repo/b.roa" hash="{old_b_hash}">{replace_b_b64}</publish>"#
|
||||
),
|
||||
&format!(r#"<publish uri="rsync://example.net/repo/c.crl">{publish_c_b64}</publish>"#),
|
||||
],
|
||||
);
|
||||
let delta_hash = sha2::Sha256::digest(&delta);
|
||||
let mut expected_hash = [0u8; 32];
|
||||
expected_hash.copy_from_slice(delta_hash.as_slice());
|
||||
|
||||
let applied = apply_delta(
|
||||
&store,
|
||||
notif_uri,
|
||||
None,
|
||||
&delta,
|
||||
expected_hash,
|
||||
Uuid::parse_str(sid).unwrap(),
|
||||
2,
|
||||
)
|
||||
.expect("apply delta");
|
||||
assert_eq!(applied, 3);
|
||||
|
||||
assert_eq!(
|
||||
store
|
||||
.load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft")
|
||||
.expect("load current a"),
|
||||
None,
|
||||
"a withdrawn"
|
||||
);
|
||||
let b = store
|
||||
.load_current_object_bytes_by_uri("rsync://example.net/repo/b.roa")
|
||||
.expect("load current b")
|
||||
.expect("b present");
|
||||
assert_eq!(b, b"b2");
|
||||
let c = store
|
||||
.load_current_object_bytes_by_uri("rsync://example.net/repo/c.crl")
|
||||
.expect("load current c")
|
||||
.expect("c present");
|
||||
assert_eq!(c, b"c2");
|
||||
|
||||
assert!(
|
||||
!store
|
||||
.is_current_rrdp_source_member(notif_uri, "rsync://example.net/repo/a.mft")
|
||||
.expect("is member"),
|
||||
"a removed from rrdp repo index"
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.is_current_rrdp_source_member(notif_uri, "rsync://example.net/repo/c.crl")
|
||||
.expect("is member"),
|
||||
"c added to rrdp repo index"
|
||||
);
|
||||
|
||||
let a_view = store
|
||||
.get_repository_view_entry("rsync://example.net/repo/a.mft")
|
||||
.expect("get a view")
|
||||
.expect("a view exists");
|
||||
assert_eq!(a_view.state, crate::repository::storage::RepositoryViewState::Withdrawn);
|
||||
let b_view = store
|
||||
.get_repository_view_entry("rsync://example.net/repo/b.roa")
|
||||
.expect("get b view")
|
||||
.expect("b view exists");
|
||||
assert_eq!(b_view.state, crate::repository::storage::RepositoryViewState::Present);
|
||||
assert_eq!(
|
||||
b_view.current_hash.as_deref(),
|
||||
Some(hex::encode(sha2::Sha256::digest(b"b2")).as_str())
|
||||
);
|
||||
let c_owner = store
|
||||
.get_rrdp_uri_owner_record("rsync://example.net/repo/c.crl")
|
||||
.expect("get c owner")
|
||||
.expect("c owner exists");
|
||||
assert_eq!(
|
||||
c_owner.owner_state,
|
||||
crate::repository::storage::RrdpUriOwnerState::Active
|
||||
);
|
||||
let a_member = store
|
||||
.get_rrdp_source_member_record(notif_uri, "rsync://example.net/repo/a.mft")
|
||||
.expect("get a member")
|
||||
.expect("a member exists");
|
||||
assert!(!a_member.present);
|
||||
let current_members = store
|
||||
.list_current_rrdp_source_members(notif_uri)
|
||||
.expect("list current members");
|
||||
assert_eq!(
|
||||
current_members
|
||||
.iter()
|
||||
.map(|record| record.rsync_uri.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
"rsync://example.net/repo/b.roa",
|
||||
"rsync://example.net/repo/c.crl",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_delta_rejects_hash_mismatch() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
|
||||
let delta = delta_xml(
|
||||
sid.to_string().as_str(),
|
||||
1,
|
||||
&[r#"<publish uri="rsync://example.net/repo/a.mft">QQ==</publish>"#],
|
||||
);
|
||||
let mut wrong = [0u8; 32];
|
||||
wrong[0] = 1;
|
||||
let err = apply_delta(&store, notif_uri, None, &delta, wrong, sid, 1).unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpSyncError::Rrdp(RrdpError::DeltaHashMismatch)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_delta_rejects_withdraw_of_non_member() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
|
||||
let withdraw_hash = "00".repeat(32);
|
||||
let delta = delta_xml(
|
||||
sid.to_string().as_str(),
|
||||
1,
|
||||
&[&format!(
|
||||
r#"<withdraw uri="rsync://example.net/repo/a.mft" hash="{withdraw_hash}"/>"#
|
||||
)],
|
||||
);
|
||||
let delta_hash = sha2::Sha256::digest(&delta);
|
||||
let mut expected_hash = [0u8; 32];
|
||||
expected_hash.copy_from_slice(delta_hash.as_slice());
|
||||
|
||||
let err = apply_delta(&store, notif_uri, None, &delta, expected_hash, sid, 1).unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpSyncError::Rrdp(RrdpError::DeltaTargetNotFromRepository { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_delta_rejects_publish_without_hash_for_existing_object() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
|
||||
let sid = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
|
||||
// Seed snapshot with a.mft.
|
||||
let snapshot_uri = "https://example.net/snapshot.xml";
|
||||
let snapshot = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]);
|
||||
let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot));
|
||||
let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash);
|
||||
let fetcher = MapFetcher {
|
||||
map: HashMap::from([(snapshot_uri.to_string(), snapshot)]),
|
||||
};
|
||||
sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("seed");
|
||||
|
||||
// Replace publish for an existing URI must have @hash.
|
||||
let publish_b64 = base64::engine::general_purpose::STANDARD.encode(b"a2");
|
||||
let delta = delta_xml(
|
||||
sid,
|
||||
2,
|
||||
&[&format!(
|
||||
r#"<publish uri="rsync://example.net/repo/a.mft">{publish_b64}</publish>"#
|
||||
)],
|
||||
);
|
||||
let delta_hash = sha2::Sha256::digest(&delta);
|
||||
let mut expected_hash = [0u8; 32];
|
||||
expected_hash.copy_from_slice(delta_hash.as_slice());
|
||||
|
||||
let err = apply_delta(
|
||||
&store,
|
||||
notif_uri,
|
||||
None,
|
||||
&delta,
|
||||
expected_hash,
|
||||
Uuid::parse_str(sid).unwrap(),
|
||||
2,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpSyncError::Rrdp(RrdpError::DeltaPublishWithoutHashForExisting { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_delta_rejects_target_missing_and_hash_mismatch() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
|
||||
let sid = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
|
||||
// Seed snapshot with a.mft.
|
||||
let snapshot_uri = "https://example.net/snapshot.xml";
|
||||
let snapshot = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]);
|
||||
let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot));
|
||||
let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash);
|
||||
let fetcher = MapFetcher {
|
||||
map: HashMap::from([(snapshot_uri.to_string(), snapshot)]),
|
||||
};
|
||||
sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("seed");
|
||||
|
||||
let old_bytes = store
|
||||
.load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft")
|
||||
.expect("get")
|
||||
.expect("present");
|
||||
let old_hash = hex::encode(sha2::Sha256::digest(old_bytes.as_slice()));
|
||||
|
||||
// Hash mismatch on withdraw.
|
||||
let wrong_hash = "11".repeat(32);
|
||||
let delta = delta_xml(
|
||||
sid,
|
||||
2,
|
||||
&[&format!(
|
||||
r#"<withdraw uri="rsync://example.net/repo/a.mft" hash="{wrong_hash}"/>"#
|
||||
)],
|
||||
);
|
||||
let delta_hash = sha2::Sha256::digest(&delta);
|
||||
let mut expected_hash = [0u8; 32];
|
||||
expected_hash.copy_from_slice(delta_hash.as_slice());
|
||||
let err = apply_delta(
|
||||
&store,
|
||||
notif_uri,
|
||||
None,
|
||||
&delta,
|
||||
expected_hash,
|
||||
Uuid::parse_str(sid).unwrap(),
|
||||
2,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpSyncError::Rrdp(RrdpError::DeltaTargetHashMismatch { .. })
|
||||
));
|
||||
|
||||
// Target missing in local cache (index still says it's a member).
|
||||
store
|
||||
.delete_repository_view_entry("rsync://example.net/repo/a.mft")
|
||||
.expect("delete current repository view entry");
|
||||
let delta = delta_xml(
|
||||
sid,
|
||||
2,
|
||||
&[&format!(
|
||||
r#"<withdraw uri="rsync://example.net/repo/a.mft" hash="{old_hash}"/>"#
|
||||
)],
|
||||
);
|
||||
let delta_hash = sha2::Sha256::digest(&delta);
|
||||
let mut expected_hash = [0u8; 32];
|
||||
expected_hash.copy_from_slice(delta_hash.as_slice());
|
||||
let err = apply_delta(
|
||||
&store,
|
||||
notif_uri,
|
||||
None,
|
||||
&delta,
|
||||
expected_hash,
|
||||
Uuid::parse_str(sid).unwrap(),
|
||||
2,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpSyncError::Rrdp(RrdpError::DeltaTargetMissing { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_delta_rejects_session_and_serial_mismatch() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
|
||||
let sid = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
|
||||
let publish_b64 = base64::engine::general_purpose::STANDARD.encode(b"x");
|
||||
let delta = delta_xml(
|
||||
sid,
|
||||
2,
|
||||
&[&format!(
|
||||
r#"<publish uri="rsync://example.net/repo/x.cer">{publish_b64}</publish>"#
|
||||
)],
|
||||
);
|
||||
let delta_hash = sha2::Sha256::digest(&delta);
|
||||
let mut expected_hash = [0u8; 32];
|
||||
expected_hash.copy_from_slice(delta_hash.as_slice());
|
||||
|
||||
// Session mismatch.
|
||||
let other_sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440001").unwrap();
|
||||
let err =
|
||||
apply_delta(&store, notif_uri, None, &delta, expected_hash, other_sid, 2).unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpSyncError::Rrdp(RrdpError::DeltaSessionIdMismatch { .. })
|
||||
));
|
||||
|
||||
// Serial mismatch.
|
||||
let err = apply_delta(
|
||||
&store,
|
||||
notif_uri,
|
||||
None,
|
||||
&delta,
|
||||
expected_hash,
|
||||
Uuid::parse_str(sid).unwrap(),
|
||||
3,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpSyncError::Rrdp(RrdpError::DeltaSerialMismatch { .. })
|
||||
));
|
||||
}
|
||||
214
src/repository/sync/rrdp/tests_parts/edge_cases.rs
Normal file
214
src/repository/sync/rrdp/tests_parts/edge_cases.rs
Normal file
@@ -0,0 +1,214 @@
|
||||
// RRDP test group: edge cases.
|
||||
|
||||
#[test]
|
||||
fn load_rrdp_local_state_uses_source_record_only() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
assert_eq!(
|
||||
load_rrdp_local_state(&store, notif_uri).expect("load empty"),
|
||||
None
|
||||
);
|
||||
|
||||
update_rrdp_source_record_on_success(
|
||||
&store,
|
||||
notif_uri,
|
||||
"source-session",
|
||||
9,
|
||||
crate::repository::storage::RrdpSourceSyncState::DeltaReady,
|
||||
Some("https://example.net/snapshot.xml"),
|
||||
Some(&hex::encode([0x11; 32])),
|
||||
)
|
||||
.expect("write source record");
|
||||
|
||||
let got = load_rrdp_local_state(&store, notif_uri)
|
||||
.expect("load source preferred")
|
||||
.expect("source present");
|
||||
assert_eq!(
|
||||
got,
|
||||
RrdpState {
|
||||
session_id: "source-session".to_string(),
|
||||
serial: 9,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_from_notification_falls_back_to_snapshot_if_missing_required_deltas() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
|
||||
let sid = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
|
||||
// Seed state serial=1 with a only.
|
||||
let snapshot_uri_1 = "https://example.net/snapshot-1.xml";
|
||||
let snapshot_1 = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]);
|
||||
let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1));
|
||||
let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1);
|
||||
let fetcher_1 = MapFetcher {
|
||||
map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]),
|
||||
};
|
||||
sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed");
|
||||
|
||||
// Notification serial=3 only includes delta serial=3 (contiguous per RFC, but missing
|
||||
// serial=2 relative to our local state, so we must use snapshot).
|
||||
let snapshot_uri_3 = "https://example.net/snapshot-3.xml";
|
||||
let snapshot_3 = snapshot_xml(sid, 3, &[("rsync://example.net/repo/z.roa", b"z3")]);
|
||||
let snapshot_hash_3 = hex::encode(sha2::Sha256::digest(&snapshot_3));
|
||||
let delta_3_hash_hex = "11".repeat(32);
|
||||
let notif_3 = notification_xml_with_deltas(
|
||||
sid,
|
||||
3,
|
||||
snapshot_uri_3,
|
||||
&snapshot_hash_3,
|
||||
&[(
|
||||
"d3",
|
||||
3,
|
||||
"https://example.net/delta-3.xml",
|
||||
&delta_3_hash_hex,
|
||||
)],
|
||||
);
|
||||
|
||||
let fetcher = MapFetcher {
|
||||
map: HashMap::from([(snapshot_uri_3.to_string(), snapshot_3)]),
|
||||
};
|
||||
|
||||
let published = sync_from_notification(&store, notif_uri, ¬if_3, &fetcher).expect("sync");
|
||||
assert_eq!(published, 1);
|
||||
assert!(
|
||||
store
|
||||
.load_current_object_bytes_by_uri("rsync://example.net/repo/z.roa")
|
||||
.expect("get current")
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_from_notification_snapshot_rejects_snapshot_hash_mismatch() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
|
||||
let sid = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let serial = 1u64;
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
let snapshot_uri = "https://example.net/snapshot.xml";
|
||||
|
||||
let snapshot = snapshot_xml(sid, serial, &[("rsync://example.net/repo/a.mft", b"x")]);
|
||||
let notif = notification_xml(sid, serial, snapshot_uri, &"00".repeat(32));
|
||||
|
||||
let fetcher = MapFetcher {
|
||||
map: HashMap::from([(snapshot_uri.to_string(), snapshot)]),
|
||||
};
|
||||
let err = sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpSyncError::Rrdp(RrdpError::SnapshotHashMismatch)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_snapshot_rejects_session_id_and_serial_mismatch() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
|
||||
let expected_sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
|
||||
let got_sid = "550e8400-e29b-41d4-a716-446655440001";
|
||||
|
||||
let snapshot = snapshot_xml(got_sid, 2, &[("rsync://example.net/repo/a.mft", b"x")]);
|
||||
let err = apply_snapshot(&store, notif_uri, None, &snapshot, expected_sid, 2).unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpSyncError::Rrdp(RrdpError::SnapshotSessionIdMismatch { .. })
|
||||
));
|
||||
|
||||
let snapshot = snapshot_xml(
|
||||
expected_sid.to_string().as_str(),
|
||||
3,
|
||||
&[("rsync://example.net/repo/a.mft", b"x")],
|
||||
);
|
||||
let err = apply_snapshot(&store, notif_uri, None, &snapshot, expected_sid, 2).unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpSyncError::Rrdp(RrdpError::SnapshotSerialMismatch { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strip_all_ascii_whitespace_removes_newlines_and_spaces() {
|
||||
assert_eq!(strip_all_ascii_whitespace(" a \n b\tc "), "abc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_snapshot_reports_publish_errors() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
|
||||
|
||||
// Missing publish/@uri
|
||||
let xml = format!(
|
||||
r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{sid}" serial="1"><publish>AA==</publish></snapshot>"#
|
||||
)
|
||||
.into_bytes();
|
||||
let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpSyncError::Rrdp(RrdpError::PublishUriMissing)
|
||||
));
|
||||
|
||||
// Missing base64 content (no text nodes).
|
||||
let xml = format!(
|
||||
r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{sid}" serial="1"><publish uri="rsync://example.net/repo/a.cer"></publish></snapshot>"#
|
||||
)
|
||||
.into_bytes();
|
||||
let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpSyncError::Rrdp(RrdpError::PublishContentMissing)
|
||||
));
|
||||
|
||||
// Invalid base64 content.
|
||||
let xml = format!(
|
||||
r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{sid}" serial="1"><publish uri="rsync://example.net/repo/a.cer">!!!</publish></snapshot>"#
|
||||
)
|
||||
.into_bytes();
|
||||
let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpSyncError::Rrdp(RrdpError::PublishBase64(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_snapshot_handles_multiple_publish_batches() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
|
||||
|
||||
let total = RRDP_SNAPSHOT_APPLY_BATCH_SIZE + 7;
|
||||
let mut xml =
|
||||
format!(r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{sid}" serial="1">"#);
|
||||
for i in 0..total {
|
||||
let uri = format!("rsync://example.net/repo/{i:04}.roa");
|
||||
let bytes = format!("payload-{i}").into_bytes();
|
||||
let b64 = base64::engine::general_purpose::STANDARD.encode(bytes);
|
||||
xml.push_str(&format!(r#"<publish uri="{uri}">{b64}</publish>"#));
|
||||
}
|
||||
xml.push_str("</snapshot>");
|
||||
|
||||
let published =
|
||||
apply_snapshot(&store, notif_uri, None, xml.as_bytes(), sid, 1).expect("apply snapshot");
|
||||
assert_eq!(published, total);
|
||||
|
||||
for idx in [0usize, RRDP_SNAPSHOT_APPLY_BATCH_SIZE - 1, total - 1] {
|
||||
let uri = format!("rsync://example.net/repo/{idx:04}.roa");
|
||||
let got = store
|
||||
.load_current_object_bytes_by_uri(&uri)
|
||||
.expect("load object")
|
||||
.expect("object exists");
|
||||
assert_eq!(got, format!("payload-{idx}").into_bytes());
|
||||
}
|
||||
}
|
||||
374
src/repository/sync/rrdp/tests_parts/parsing.rs
Normal file
374
src/repository/sync/rrdp/tests_parts/parsing.rs
Normal file
@@ -0,0 +1,374 @@
|
||||
// RRDP test group: parsing.
|
||||
|
||||
use super::*;
|
||||
use crate::output::analysis::timing::{TimingHandle, TimingMeta};
|
||||
use crate::repository::current_repo_index::CurrentRepoIndex;
|
||||
use crate::repository::storage::RocksStore;
|
||||
use std::collections::HashMap;
|
||||
use std::io::Read;
|
||||
use std::time::Duration;
|
||||
|
||||
struct MapFetcher {
|
||||
map: HashMap<String, Vec<u8>>,
|
||||
}
|
||||
|
||||
impl Fetcher for MapFetcher {
|
||||
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
|
||||
self.map
|
||||
.get(uri)
|
||||
.cloned()
|
||||
.ok_or_else(|| format!("not found: {uri}"))
|
||||
}
|
||||
}
|
||||
|
||||
struct SleepyFetcher {
|
||||
inner: MapFetcher,
|
||||
sleep_uri: String,
|
||||
sleep: Duration,
|
||||
}
|
||||
|
||||
impl Fetcher for SleepyFetcher {
|
||||
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
|
||||
if uri == self.sleep_uri {
|
||||
std::thread::sleep(self.sleep);
|
||||
}
|
||||
self.inner.fetch(uri)
|
||||
}
|
||||
}
|
||||
|
||||
struct WriterOnlyFetcher {
|
||||
map: HashMap<String, Vec<u8>>,
|
||||
}
|
||||
|
||||
impl Fetcher for WriterOnlyFetcher {
|
||||
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
|
||||
Err(format!("unexpected buffered fetch: {uri}"))
|
||||
}
|
||||
|
||||
fn fetch_to_writer(&self, uri: &str, out: &mut dyn std::io::Write) -> Result<u64, String> {
|
||||
let bytes = self
|
||||
.map
|
||||
.get(uri)
|
||||
.ok_or_else(|| format!("not found: {uri}"))?;
|
||||
out.write_all(bytes)
|
||||
.map_err(|e| format!("write sink failed: {e}"))?;
|
||||
Ok(bytes.len() as u64)
|
||||
}
|
||||
}
|
||||
|
||||
struct NonAsciiWriterFetcher;
|
||||
|
||||
impl Fetcher for NonAsciiWriterFetcher {
|
||||
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
|
||||
Err(format!("unexpected buffered fetch: {uri}"))
|
||||
}
|
||||
|
||||
fn fetch_to_writer(&self, _uri: &str, out: &mut dyn std::io::Write) -> Result<u64, String> {
|
||||
out.write_all(&[0x80])
|
||||
.map_err(|e| format!("write sink failed: {e}"))?;
|
||||
Err("snapshot body contains non-ASCII bytes".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_current_object(store: &RocksStore, uri: &str, expected: &[u8]) {
|
||||
assert_eq!(
|
||||
store
|
||||
.load_current_object_bytes_by_uri(uri)
|
||||
.expect("load current object"),
|
||||
Some(expected.to_vec())
|
||||
);
|
||||
}
|
||||
|
||||
fn notification_xml(
|
||||
session_id: &str,
|
||||
serial: u64,
|
||||
snapshot_uri: &str,
|
||||
snapshot_hash: &str,
|
||||
) -> Vec<u8> {
|
||||
format!(
|
||||
r#"<notification xmlns="{RRDP_XMLNS}" version="1" session_id="{session_id}" serial="{serial}"><snapshot uri="{snapshot_uri}" hash="{snapshot_hash}"/></notification>"#
|
||||
)
|
||||
.into_bytes()
|
||||
}
|
||||
|
||||
fn notification_xml_with_deltas(
|
||||
session_id: &str,
|
||||
serial: u64,
|
||||
snapshot_uri: &str,
|
||||
snapshot_hash: &str,
|
||||
deltas: &[(&str, u64, &str, &str)],
|
||||
) -> Vec<u8> {
|
||||
let mut out = format!(
|
||||
r#"<notification xmlns="{RRDP_XMLNS}" version="1" session_id="{session_id}" serial="{serial}"><snapshot uri="{snapshot_uri}" hash="{snapshot_hash}"/>"#
|
||||
);
|
||||
for (_name, delta_serial, uri, hash) in deltas {
|
||||
out.push_str(&format!(
|
||||
r#"<delta serial="{delta_serial}" uri="{uri}" hash="{hash}"/>"#
|
||||
));
|
||||
}
|
||||
out.push_str("</notification>");
|
||||
out.into_bytes()
|
||||
}
|
||||
|
||||
fn snapshot_xml(session_id: &str, serial: u64, published: &[(&str, &[u8])]) -> Vec<u8> {
|
||||
let mut out = format!(
|
||||
r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{session_id}" serial="{serial}">"#
|
||||
);
|
||||
for (uri, bytes) in published {
|
||||
let b64 = base64::engine::general_purpose::STANDARD.encode(bytes);
|
||||
out.push_str(&format!(r#"<publish uri="{uri}">{b64}</publish>"#));
|
||||
}
|
||||
out.push_str("</snapshot>");
|
||||
out.into_bytes()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_snapshot_into_tempfile_streams_and_validates_hash() {
|
||||
let snapshot_uri = "https://example.test/snapshot.xml";
|
||||
let snapshot = b"<snapshot xmlns=\"http://www.ripe.net/rpki/rrdp\" version=\"1\"/>".to_vec();
|
||||
let mut expected_hash = [0u8; 32];
|
||||
expected_hash.copy_from_slice(&sha2::Sha256::digest(&snapshot));
|
||||
let fetcher = WriterOnlyFetcher {
|
||||
map: HashMap::from([(snapshot_uri.to_string(), snapshot.clone())]),
|
||||
};
|
||||
let origin = RrdpOrigin::parse(snapshot_uri, RrdpResourceKind::Notification).expect("origin");
|
||||
|
||||
let (mut file, bytes_written) =
|
||||
fetch_snapshot_into_tempfile(&fetcher, snapshot_uri, &expected_hash, &origin)
|
||||
.expect("fetch snapshot into tempfile");
|
||||
assert_eq!(bytes_written, snapshot.len() as u64);
|
||||
let mut got = Vec::new();
|
||||
file.as_file_mut()
|
||||
.read_to_end(&mut got)
|
||||
.expect("read tempfile");
|
||||
assert_eq!(got, snapshot);
|
||||
|
||||
let mut wrong_hash = expected_hash;
|
||||
wrong_hash[0] ^= 0xff;
|
||||
let err =
|
||||
fetch_snapshot_into_tempfile(&fetcher, snapshot_uri, &wrong_hash, &origin).unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpSyncError::Rrdp(RrdpError::SnapshotHashMismatch)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetch_snapshot_into_tempfile_maps_stream_non_ascii_error() {
|
||||
let err = fetch_snapshot_into_tempfile(
|
||||
&NonAsciiWriterFetcher,
|
||||
"https://example.test/snapshot.xml",
|
||||
&[0u8; 32],
|
||||
&RrdpOrigin::parse(
|
||||
"https://example.test/notification.xml",
|
||||
RrdpResourceKind::Notification,
|
||||
)
|
||||
.expect("origin"),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, RrdpSyncError::Rrdp(RrdpError::NotAscii)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timing_rrdp_repo_step_spans_cover_snapshot_fetch_duration() {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let store_dir = temp.path().join("db");
|
||||
let store = RocksStore::open(&store_dir).expect("open rocksdb");
|
||||
|
||||
let notification_uri = "https://example.test/notification.xml";
|
||||
let snapshot_uri = "https://example.test/snapshot.xml";
|
||||
let published_uri = "rsync://example.test/repo/a.mft";
|
||||
let published_bytes = b"x";
|
||||
let session_id = "550e8400-e29b-41d4-a716-446655440000";
|
||||
|
||||
let snapshot = snapshot_xml(session_id, 1, &[(published_uri, published_bytes)]);
|
||||
let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot));
|
||||
let notif = notification_xml(session_id, 1, snapshot_uri, &snapshot_hash);
|
||||
|
||||
let mut map = HashMap::new();
|
||||
map.insert(snapshot_uri.to_string(), snapshot);
|
||||
let fetcher = SleepyFetcher {
|
||||
inner: MapFetcher { map },
|
||||
sleep_uri: snapshot_uri.to_string(),
|
||||
sleep: Duration::from_millis(25),
|
||||
};
|
||||
|
||||
let timing = TimingHandle::new(TimingMeta {
|
||||
recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(),
|
||||
validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(),
|
||||
tal_url: None,
|
||||
db_path: Some(store_dir.to_string_lossy().into_owned()),
|
||||
});
|
||||
|
||||
sync_from_notification_snapshot_with_timing(
|
||||
&store,
|
||||
notification_uri,
|
||||
¬if,
|
||||
&fetcher,
|
||||
Some(&timing),
|
||||
)
|
||||
.expect("rrdp snapshot sync ok");
|
||||
|
||||
let timing_path = temp.path().join("timing.json");
|
||||
timing.write_json(&timing_path, 200).expect("write timing");
|
||||
let rep: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(&timing_path).expect("read timing"))
|
||||
.expect("parse timing");
|
||||
|
||||
let want = format!("{notification_uri}::fetch_snapshot");
|
||||
let steps = rep
|
||||
.get("top_rrdp_repo_steps")
|
||||
.and_then(|v| v.as_array())
|
||||
.expect("top_rrdp_repo_steps array");
|
||||
let entry = steps
|
||||
.iter()
|
||||
.find(|e| e.get("key").and_then(|k| k.as_str()) == Some(want.as_str()))
|
||||
.unwrap_or_else(|| panic!("missing timing step entry for {want}"));
|
||||
let nanos = entry
|
||||
.get("total_nanos")
|
||||
.and_then(|v| v.as_u64())
|
||||
.expect("total_nanos");
|
||||
assert!(
|
||||
nanos >= 20_000_000,
|
||||
"expected fetch_snapshot timing to include the fetch duration; got {nanos}ns"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_notification_snapshot_rejects_non_ascii() {
|
||||
let mut xml = b"<notification/>".to_vec();
|
||||
xml.push(0x80);
|
||||
let err = parse_notification_snapshot(&xml).unwrap_err();
|
||||
assert!(matches!(err, RrdpError::NotAscii));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_notification_snapshot_parses_valid_minimal_notification() {
|
||||
let sid = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let snapshot_uri = "https://example.net/snapshot.xml";
|
||||
let hash = "00".repeat(32);
|
||||
let xml = notification_xml(sid, 7, snapshot_uri, &hash);
|
||||
let n = parse_notification_snapshot(&xml).expect("parse");
|
||||
assert_eq!(n.session_id, Uuid::parse_str(sid).unwrap());
|
||||
assert_eq!(n.serial, 7);
|
||||
assert_eq!(n.snapshot_uri, snapshot_uri);
|
||||
assert_eq!(hex::encode(n.snapshot_hash_sha256), hash);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_notification_parses_deltas_and_validates_contiguity() {
|
||||
let sid = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let snapshot_uri = "https://example.net/snapshot.xml";
|
||||
let hash = "00".repeat(32);
|
||||
let d_hash_2 = "11".repeat(32);
|
||||
let d_hash_3 = "22".repeat(32);
|
||||
// Provide deltas in reverse order to ensure we sort.
|
||||
let xml = notification_xml_with_deltas(
|
||||
sid,
|
||||
3,
|
||||
snapshot_uri,
|
||||
&hash,
|
||||
&[
|
||||
("d3", 3, "https://example.net/delta-3.xml", &d_hash_3),
|
||||
("d2", 2, "https://example.net/delta-2.xml", &d_hash_2),
|
||||
],
|
||||
);
|
||||
let n = parse_notification(&xml).expect("parse notification");
|
||||
assert_eq!(n.serial, 3);
|
||||
assert_eq!(n.deltas.len(), 2);
|
||||
assert_eq!(n.deltas[0].serial, 2);
|
||||
assert_eq!(n.deltas[1].serial, 3);
|
||||
assert_eq!(n.deltas[0].uri, "https://example.net/delta-2.xml");
|
||||
assert_eq!(hex::encode(n.deltas[1].hash_sha256), d_hash_3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_notification_rejects_non_contiguous_deltas() {
|
||||
let sid = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let snapshot_uri = "https://example.net/snapshot.xml";
|
||||
let hash = "00".repeat(32);
|
||||
let d_hash_1 = "11".repeat(32);
|
||||
let d_hash_3 = "22".repeat(32);
|
||||
// Missing delta serial 2.
|
||||
let xml = notification_xml_with_deltas(
|
||||
sid,
|
||||
3,
|
||||
snapshot_uri,
|
||||
&hash,
|
||||
&[
|
||||
("d3", 3, "https://example.net/delta-3.xml", &d_hash_3),
|
||||
("d1", 1, "https://example.net/delta-1.xml", &d_hash_1),
|
||||
],
|
||||
);
|
||||
let err = parse_notification(&xml).unwrap_err();
|
||||
assert!(matches!(err, RrdpError::DeltaRefChainNotContiguous { .. }));
|
||||
}
|
||||
|
||||
fn delta_xml(session_id: &str, serial: u64, elements: &[&str]) -> Vec<u8> {
|
||||
let mut out = format!(
|
||||
r#"<delta xmlns="{RRDP_XMLNS}" version="1" session_id="{session_id}" serial="{serial}">"#
|
||||
);
|
||||
for e in elements {
|
||||
out.push_str(e);
|
||||
}
|
||||
out.push_str("</delta>");
|
||||
out.into_bytes()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_delta_file_parses_publish_and_withdraw() {
|
||||
let sid = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let serial = 3u64;
|
||||
let publish_bytes = b"abc";
|
||||
let publish_b64 = base64::engine::general_purpose::STANDARD.encode(publish_bytes);
|
||||
let withdraw_hash = "33".repeat(32);
|
||||
|
||||
let xml = delta_xml(
|
||||
sid,
|
||||
serial,
|
||||
&[
|
||||
&format!(r#"<publish uri="rsync://example.net/repo/a.mft">{publish_b64}</publish>"#),
|
||||
&format!(r#"<withdraw uri="rsync://example.net/repo/b.cer" hash="{withdraw_hash}"/>"#),
|
||||
],
|
||||
);
|
||||
|
||||
let d = parse_delta_file(&xml).expect("parse delta");
|
||||
assert_eq!(d.session_id, Uuid::parse_str(sid).unwrap());
|
||||
assert_eq!(d.serial, serial);
|
||||
assert_eq!(d.elements.len(), 2);
|
||||
match &d.elements[0] {
|
||||
DeltaElement::Publish {
|
||||
uri,
|
||||
hash_sha256,
|
||||
bytes,
|
||||
} => {
|
||||
assert_eq!(uri, "rsync://example.net/repo/a.mft");
|
||||
assert_eq!(*hash_sha256, None);
|
||||
assert_eq!(bytes, publish_bytes);
|
||||
}
|
||||
_ => panic!("expected publish"),
|
||||
}
|
||||
match &d.elements[1] {
|
||||
DeltaElement::Withdraw { uri, hash_sha256 } => {
|
||||
assert_eq!(uri, "rsync://example.net/repo/b.cer");
|
||||
assert_eq!(hex::encode(hash_sha256), withdraw_hash);
|
||||
}
|
||||
_ => panic!("expected withdraw"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_delta_file_rejects_withdraw_with_content() {
|
||||
let sid = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let serial = 1u64;
|
||||
let withdraw_hash = "33".repeat(32);
|
||||
let xml = delta_xml(
|
||||
sid,
|
||||
serial,
|
||||
&[&format!(
|
||||
r#"<withdraw uri="rsync://example.net/repo/b.cer" hash="{withdraw_hash}">AA==</withdraw>"#
|
||||
)],
|
||||
);
|
||||
let err = parse_delta_file(&xml).unwrap_err();
|
||||
assert!(matches!(err, RrdpError::DeltaWithdrawUnexpectedContent));
|
||||
}
|
||||
470
src/repository/sync/rrdp/tests_parts/sync.rs
Normal file
470
src/repository/sync/rrdp/tests_parts/sync.rs
Normal file
@@ -0,0 +1,470 @@
|
||||
// RRDP test group: sync.
|
||||
|
||||
#[test]
|
||||
fn sync_from_notification_snapshot_rejects_cross_source_owner_conflict() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
|
||||
let sid_a = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let sid_b = "550e8400-e29b-41d4-a716-446655440001";
|
||||
let uri = "rsync://example.net/repo/a.mft";
|
||||
|
||||
let notif_a_uri = "https://example.net/a/notification.xml";
|
||||
let snapshot_a_uri = "https://example.net/a/snapshot.xml";
|
||||
let snapshot_a = snapshot_xml(sid_a, 1, &[(uri, b"a1")]);
|
||||
let snapshot_a_hash = hex::encode(sha2::Sha256::digest(&snapshot_a));
|
||||
let notif_a = notification_xml(sid_a, 1, snapshot_a_uri, &snapshot_a_hash);
|
||||
let fetcher_a = MapFetcher {
|
||||
map: HashMap::from([(snapshot_a_uri.to_string(), snapshot_a)]),
|
||||
};
|
||||
sync_from_notification_snapshot(&store, notif_a_uri, ¬if_a, &fetcher_a)
|
||||
.expect("seed source a");
|
||||
|
||||
let notif_b_uri = "https://example.net/b/notification.xml";
|
||||
let snapshot_b_uri = "https://example.net/b/snapshot.xml";
|
||||
let snapshot_b = snapshot_xml(sid_b, 1, &[(uri, b"b1")]);
|
||||
let snapshot_b_hash = hex::encode(sha2::Sha256::digest(&snapshot_b));
|
||||
let notif_b = notification_xml(sid_b, 1, snapshot_b_uri, &snapshot_b_hash);
|
||||
let fetcher_b = MapFetcher {
|
||||
map: HashMap::from([(snapshot_b_uri.to_string(), snapshot_b)]),
|
||||
};
|
||||
|
||||
let err = sync_from_notification_snapshot(&store, notif_b_uri, ¬if_b, &fetcher_b)
|
||||
.expect_err("cross-source overwrite must fail");
|
||||
assert!(matches!(err, RrdpSyncError::Storage(_)));
|
||||
assert!(err.to_string().contains("owner conflict"), "{err}");
|
||||
}
|
||||
|
||||
struct PanicOnRrdpFetch;
|
||||
|
||||
impl Fetcher for PanicOnRrdpFetch {
|
||||
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
|
||||
panic!("cross-origin notification must be rejected before fetch: {uri}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cross_origin_snapshot_reference_is_rejected_before_fetch_or_state_write() {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(temp.path()).expect("open rocksdb");
|
||||
let notification_uri = "https://origin.example.test/notification.xml";
|
||||
let notification = notification_xml(
|
||||
"550e8400-e29b-41d4-a716-446655440000",
|
||||
1,
|
||||
"https://foreign.example.test/snapshot.xml",
|
||||
&"00".repeat(32),
|
||||
);
|
||||
|
||||
let err =
|
||||
sync_from_notification_snapshot(&store, notification_uri, ¬ification, &PanicOnRrdpFetch)
|
||||
.expect_err("cross-origin snapshot must be rejected");
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpSyncError::Rrdp(RrdpError::CrossOriginReference {
|
||||
resource: RrdpResourceKind::Snapshot,
|
||||
..
|
||||
})
|
||||
));
|
||||
assert!(
|
||||
load_rrdp_local_state(&store, notification_uri)
|
||||
.expect("read state")
|
||||
.is_none(),
|
||||
"rejection must precede RRDP state writes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cross_origin_delta_reference_is_rejected_even_when_delta_would_not_be_used() {
|
||||
let temp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(temp.path()).expect("open rocksdb");
|
||||
let notification_uri = "https://origin.example.test/notification.xml";
|
||||
let notification = notification_xml_with_deltas(
|
||||
"550e8400-e29b-41d4-a716-446655440000",
|
||||
1,
|
||||
"https://origin.example.test/snapshot.xml",
|
||||
&"00".repeat(32),
|
||||
&[(
|
||||
"delta-1",
|
||||
1,
|
||||
"https://foreign.example.test/delta-1.xml",
|
||||
&"11".repeat(32),
|
||||
)],
|
||||
);
|
||||
|
||||
let err = sync_from_notification(&store, notification_uri, ¬ification, &PanicOnRrdpFetch)
|
||||
.expect_err("cross-origin delta must be rejected before state/fetch");
|
||||
assert!(matches!(
|
||||
err,
|
||||
RrdpSyncError::Rrdp(RrdpError::CrossOriginReference {
|
||||
resource: RrdpResourceKind::Delta,
|
||||
..
|
||||
})
|
||||
));
|
||||
assert!(
|
||||
load_rrdp_local_state(&store, notification_uri)
|
||||
.expect("read state")
|
||||
.is_none(),
|
||||
"all delta references are checked before state reads or writes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rrdp_origin_treats_https_default_port_as_equivalent() {
|
||||
let notification = parse_notification(¬ification_xml(
|
||||
"550e8400-e29b-41d4-a716-446655440000",
|
||||
1,
|
||||
"https://origin.example.test:443/snapshot.xml",
|
||||
&"00".repeat(32),
|
||||
))
|
||||
.expect("notification");
|
||||
validate_notification_references(
|
||||
"https://origin.example.test/notification.xml",
|
||||
¬ification,
|
||||
)
|
||||
.expect("explicit HTTPS default port must remain same-origin");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_from_notification_snapshot_applies_snapshot_and_stores_state() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
|
||||
let sid = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let serial = 9u64;
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
let snapshot_uri = "https://example.net/snapshot.xml";
|
||||
|
||||
let snapshot = snapshot_xml(
|
||||
sid,
|
||||
serial,
|
||||
&[
|
||||
("rsync://example.net/repo/a.mft", b"mft-bytes"),
|
||||
("rsync://example.net/repo/b.roa", b"roa-bytes"),
|
||||
],
|
||||
);
|
||||
let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot));
|
||||
let notif = notification_xml(sid, serial, snapshot_uri, &snapshot_hash);
|
||||
|
||||
let fetcher = MapFetcher {
|
||||
map: HashMap::from([(snapshot_uri.to_string(), snapshot.clone())]),
|
||||
};
|
||||
|
||||
let published =
|
||||
sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("sync");
|
||||
assert_eq!(published, 2);
|
||||
|
||||
assert_current_object(&store, "rsync://example.net/repo/a.mft", b"mft-bytes");
|
||||
assert_current_object(&store, "rsync://example.net/repo/b.roa", b"roa-bytes");
|
||||
|
||||
let state = load_rrdp_local_state(&store, notif_uri)
|
||||
.expect("get rrdp state")
|
||||
.expect("state present");
|
||||
assert_eq!(state.session_id, sid);
|
||||
assert_eq!(state.serial, serial);
|
||||
|
||||
let source = store
|
||||
.get_rrdp_source_record(notif_uri)
|
||||
.expect("get rrdp source")
|
||||
.expect("rrdp source exists");
|
||||
assert_eq!(source.last_session_id.as_deref(), Some(sid));
|
||||
assert_eq!(source.last_serial, Some(serial));
|
||||
assert_eq!(
|
||||
source.sync_state,
|
||||
crate::repository::storage::RrdpSourceSyncState::SnapshotOnly
|
||||
);
|
||||
|
||||
let view = store
|
||||
.get_repository_view_entry("rsync://example.net/repo/a.mft")
|
||||
.expect("get repository view")
|
||||
.expect("repository view exists");
|
||||
assert_eq!(view.state, crate::repository::storage::RepositoryViewState::Present);
|
||||
assert_eq!(view.repository_source.as_deref(), Some(notif_uri));
|
||||
|
||||
let current_bytes = store
|
||||
.load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft")
|
||||
.expect("load current bytes")
|
||||
.expect("current object bytes exist");
|
||||
assert_eq!(current_bytes, b"mft-bytes".to_vec());
|
||||
assert!(
|
||||
store
|
||||
.get_raw_by_hash_entry(hex::encode(sha2::Sha256::digest(b"mft-bytes")).as_str())
|
||||
.expect("get raw_by_hash")
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let member = store
|
||||
.get_rrdp_source_member_record(notif_uri, "rsync://example.net/repo/a.mft")
|
||||
.expect("get member")
|
||||
.expect("member exists");
|
||||
assert!(member.present);
|
||||
let owner = store
|
||||
.get_rrdp_uri_owner_record("rsync://example.net/repo/a.mft")
|
||||
.expect("get owner")
|
||||
.expect("owner exists");
|
||||
assert_eq!(owner.notify_uri, notif_uri);
|
||||
assert_eq!(owner.owner_state, crate::repository::storage::RrdpUriOwnerState::Active);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_from_notification_snapshot_deletes_objects_not_in_new_snapshot() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
|
||||
let sid = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
|
||||
// serial 1: publish a + b
|
||||
let snapshot_uri_1 = "https://example.net/snapshot-1.xml";
|
||||
let snapshot_1 = snapshot_xml(
|
||||
sid,
|
||||
1,
|
||||
&[
|
||||
("rsync://example.net/repo/a.mft", b"a1"),
|
||||
("rsync://example.net/repo/b.roa", b"b1"),
|
||||
],
|
||||
);
|
||||
let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1));
|
||||
let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1);
|
||||
|
||||
let fetcher_1 = MapFetcher {
|
||||
map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]),
|
||||
};
|
||||
sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("sync 1");
|
||||
|
||||
// serial 2: publish b (new bytes) + c, and drop a
|
||||
let snapshot_uri_2 = "https://example.net/snapshot-2.xml";
|
||||
let snapshot_2 = snapshot_xml(
|
||||
sid,
|
||||
2,
|
||||
&[
|
||||
("rsync://example.net/repo/b.roa", b"b2"),
|
||||
("rsync://example.net/repo/c.crl", b"c2"),
|
||||
],
|
||||
);
|
||||
let snapshot_hash_2 = hex::encode(sha2::Sha256::digest(&snapshot_2));
|
||||
let notif_2 = notification_xml(sid, 2, snapshot_uri_2, &snapshot_hash_2);
|
||||
|
||||
let fetcher_2 = MapFetcher {
|
||||
map: HashMap::from([(snapshot_uri_2.to_string(), snapshot_2)]),
|
||||
};
|
||||
sync_from_notification_snapshot(&store, notif_uri, ¬if_2, &fetcher_2).expect("sync 2");
|
||||
|
||||
assert!(
|
||||
store
|
||||
.load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft")
|
||||
.expect("get current a")
|
||||
.is_none(),
|
||||
"a should be deleted by full-state snapshot apply"
|
||||
);
|
||||
|
||||
assert_current_object(&store, "rsync://example.net/repo/b.roa", b"b2");
|
||||
assert_current_object(&store, "rsync://example.net/repo/c.crl", b"c2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_from_notification_uses_deltas_when_available_for_local_state() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
|
||||
let sid = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
|
||||
// Seed state with snapshot serial=1 containing a+b.
|
||||
let snapshot_uri_1 = "https://example.net/snapshot-1.xml";
|
||||
let snapshot_1 = snapshot_xml(
|
||||
sid,
|
||||
1,
|
||||
&[
|
||||
("rsync://example.net/repo/a.mft", b"a1"),
|
||||
("rsync://example.net/repo/b.roa", b"b1"),
|
||||
],
|
||||
);
|
||||
let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1));
|
||||
let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1);
|
||||
let fetcher_1 = MapFetcher {
|
||||
map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]),
|
||||
};
|
||||
sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed");
|
||||
|
||||
// Notification serial=3 with deltas 2 and 3. Snapshot URI is intentionally not fetchable
|
||||
// to assert we really use deltas.
|
||||
let snapshot_uri_3 = "https://example.net/snapshot-3.xml";
|
||||
let snapshot_hash_3 = "00".repeat(32);
|
||||
|
||||
let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2");
|
||||
let delta_2 = delta_xml(
|
||||
sid,
|
||||
2,
|
||||
&[&format!(
|
||||
r#"<publish uri="rsync://example.net/repo/c.crl">{publish_c_b64}</publish>"#
|
||||
)],
|
||||
);
|
||||
let delta_2_hash_hex = hex::encode(sha2::Sha256::digest(&delta_2));
|
||||
|
||||
let c_hash_hex = hex::encode(sha2::Sha256::digest(b"c2".as_slice()));
|
||||
let delta_3 = delta_xml(
|
||||
sid,
|
||||
3,
|
||||
&[&format!(
|
||||
r#"<withdraw uri="rsync://example.net/repo/c.crl" hash="{c_hash_hex}"/>"#
|
||||
)],
|
||||
);
|
||||
let delta_3_hash_hex = hex::encode(sha2::Sha256::digest(&delta_3));
|
||||
|
||||
let notif_3 = notification_xml_with_deltas(
|
||||
sid,
|
||||
3,
|
||||
snapshot_uri_3,
|
||||
&snapshot_hash_3,
|
||||
&[
|
||||
(
|
||||
"d3",
|
||||
3,
|
||||
"https://example.net/delta-3.xml",
|
||||
&delta_3_hash_hex,
|
||||
),
|
||||
(
|
||||
"d2",
|
||||
2,
|
||||
"https://example.net/delta-2.xml",
|
||||
&delta_2_hash_hex,
|
||||
),
|
||||
],
|
||||
);
|
||||
|
||||
let fetcher = MapFetcher {
|
||||
map: HashMap::from([
|
||||
("https://example.net/delta-2.xml".to_string(), delta_2),
|
||||
("https://example.net/delta-3.xml".to_string(), delta_3),
|
||||
]),
|
||||
};
|
||||
|
||||
let applied = sync_from_notification(&store, notif_uri, ¬if_3, &fetcher).expect("sync");
|
||||
assert!(applied > 0);
|
||||
|
||||
// Delta 2 publishes c then delta 3 withdraws it => final state should not contain c.
|
||||
assert!(
|
||||
store
|
||||
.load_current_object_bytes_by_uri("rsync://example.net/repo/c.crl")
|
||||
.expect("get current")
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let state = load_rrdp_local_state(&store, notif_uri)
|
||||
.expect("get rrdp state")
|
||||
.expect("state present");
|
||||
assert_eq!(state.session_id, Uuid::parse_str(sid).unwrap().to_string());
|
||||
assert_eq!(state.serial, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_from_notification_same_serial_hydrates_current_repo_index() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
|
||||
let sid = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
let snapshot_uri = "https://example.net/snapshot.xml";
|
||||
let uri_a = "rsync://example.net/repo/a.mft";
|
||||
let uri_b = "rsync://example.net/repo/b.roa";
|
||||
|
||||
let snapshot = snapshot_xml(sid, 1, &[(uri_a, b"a1"), (uri_b, b"b1")]);
|
||||
let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot));
|
||||
let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash);
|
||||
let fetcher_1 = MapFetcher {
|
||||
map: HashMap::from([(snapshot_uri.to_string(), snapshot)]),
|
||||
};
|
||||
sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher_1).expect("seed");
|
||||
|
||||
let index = CurrentRepoIndex::shared();
|
||||
let no_fetcher = MapFetcher {
|
||||
map: HashMap::new(),
|
||||
};
|
||||
let applied = sync_from_notification_with_timing_and_download_log(
|
||||
&store,
|
||||
notif_uri,
|
||||
Some(&index),
|
||||
¬if,
|
||||
&no_fetcher,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("same serial no-op");
|
||||
assert_eq!(applied, 0);
|
||||
|
||||
let index = index.read().expect("read-lock index");
|
||||
assert_eq!(index.active_uri_count(), 2);
|
||||
assert!(index.get_by_uri(uri_a).is_some());
|
||||
assert!(index.get_by_uri(uri_b).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_from_notification_delta_hydrates_unchanged_current_repo_entries() {
|
||||
let tmp = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(tmp.path()).expect("open rocksdb");
|
||||
|
||||
let sid = "550e8400-e29b-41d4-a716-446655440000";
|
||||
let notif_uri = "https://example.net/notification.xml";
|
||||
let snapshot_uri_1 = "https://example.net/snapshot-1.xml";
|
||||
let uri_a = "rsync://example.net/repo/a.mft";
|
||||
let uri_b = "rsync://example.net/repo/b.roa";
|
||||
let uri_c = "rsync://example.net/repo/c.crl";
|
||||
|
||||
let snapshot_1 = snapshot_xml(sid, 1, &[(uri_a, b"a1"), (uri_b, b"b1")]);
|
||||
let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1));
|
||||
let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1);
|
||||
let fetcher_1 = MapFetcher {
|
||||
map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]),
|
||||
};
|
||||
sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed");
|
||||
|
||||
let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2");
|
||||
let delta_2 = delta_xml(
|
||||
sid,
|
||||
2,
|
||||
&[&format!(
|
||||
r#"<publish uri="{uri_c}">{publish_c_b64}</publish>"#
|
||||
)],
|
||||
);
|
||||
let delta_2_hash_hex = hex::encode(sha2::Sha256::digest(&delta_2));
|
||||
let notif_2 = notification_xml_with_deltas(
|
||||
sid,
|
||||
2,
|
||||
"https://example.net/snapshot-2.xml",
|
||||
&"00".repeat(32),
|
||||
&[(
|
||||
"d2",
|
||||
2,
|
||||
"https://example.net/delta-2.xml",
|
||||
&delta_2_hash_hex,
|
||||
)],
|
||||
);
|
||||
let fetcher_2 = MapFetcher {
|
||||
map: HashMap::from([("https://example.net/delta-2.xml".to_string(), delta_2)]),
|
||||
};
|
||||
|
||||
let index = CurrentRepoIndex::shared();
|
||||
let applied = sync_from_notification_with_timing_and_download_log(
|
||||
&store,
|
||||
notif_uri,
|
||||
Some(&index),
|
||||
¬if_2,
|
||||
&fetcher_2,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.expect("delta sync");
|
||||
assert_eq!(applied, 1);
|
||||
|
||||
let index = index.read().expect("read-lock index");
|
||||
assert_eq!(index.active_uri_count(), 3);
|
||||
assert!(
|
||||
index.get_by_uri(uri_a).is_some(),
|
||||
"unchanged object from the previous serial must be visible"
|
||||
);
|
||||
assert!(
|
||||
index.get_by_uri(uri_b).is_some(),
|
||||
"unchanged object from the previous serial must be visible"
|
||||
);
|
||||
assert!(index.get_by_uri(uri_c).is_some(), "delta publish visible");
|
||||
}
|
||||
510
src/repository/sync/store_projection.rs
Normal file
510
src/repository/sync/store_projection.rs
Normal file
@@ -0,0 +1,510 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use crate::repository::storage::{
|
||||
PackTime, RawByHashEntry, RepositoryViewEntry, RepositoryViewState, RocksStore,
|
||||
RrdpSourceMemberRecord, RrdpSourceRecord, RrdpSourceSyncState, RrdpUriOwnerRecord,
|
||||
RrdpUriOwnerState,
|
||||
};
|
||||
use sha2::Digest;
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub struct PreparedRawByHashBatch {
|
||||
pub uri_to_hash: BTreeMap<String, String>,
|
||||
pub entries_to_write: Vec<RawByHashEntry>,
|
||||
}
|
||||
|
||||
pub struct PreparedRepoBytesBatch {
|
||||
pub uri_to_hash: BTreeMap<String, String>,
|
||||
pub blobs_to_write: Vec<(String, Vec<u8>)>,
|
||||
}
|
||||
|
||||
pub fn prepare_repo_bytes_batch(
|
||||
objects: &[(String, Vec<u8>)],
|
||||
) -> Result<PreparedRepoBytesBatch, String> {
|
||||
prepare_repo_bytes_batch_owned(objects.to_vec())
|
||||
}
|
||||
|
||||
pub fn prepare_repo_bytes_batch_owned(
|
||||
objects: Vec<(String, Vec<u8>)>,
|
||||
) -> Result<PreparedRepoBytesBatch, String> {
|
||||
let mut uri_to_hash: BTreeMap<String, String> = BTreeMap::new();
|
||||
let mut pending: BTreeMap<String, Vec<u8>> = BTreeMap::new();
|
||||
|
||||
for (uri, bytes) in objects {
|
||||
if bytes.is_empty() {
|
||||
return Err(format!("repo bytes for {uri} must not be empty"));
|
||||
}
|
||||
let sha256_hex = compute_sha256_hex(&bytes);
|
||||
uri_to_hash.insert(uri.clone(), sha256_hex.clone());
|
||||
match pending.entry(sha256_hex) {
|
||||
std::collections::btree_map::Entry::Vacant(slot) => {
|
||||
slot.insert(bytes);
|
||||
}
|
||||
std::collections::btree_map::Entry::Occupied(existing) => {
|
||||
if existing.get() != &bytes {
|
||||
return Err(format!(
|
||||
"repo bytes collision for {uri}: same sha256 maps to different bytes"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(PreparedRepoBytesBatch {
|
||||
uri_to_hash,
|
||||
blobs_to_write: pending.into_iter().collect(),
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn prepare_raw_by_hash_evidence_batch(
|
||||
store: &RocksStore,
|
||||
objects: &[(String, Vec<u8>)],
|
||||
) -> Result<PreparedRawByHashBatch, String> {
|
||||
let mut pending: BTreeMap<String, RawByHashEntry> = BTreeMap::new();
|
||||
let mut uri_to_hash: BTreeMap<String, String> = BTreeMap::new();
|
||||
|
||||
for (uri, bytes) in objects {
|
||||
let sha256_hex = compute_sha256_hex(bytes);
|
||||
uri_to_hash.insert(uri.clone(), sha256_hex.clone());
|
||||
let entry = pending
|
||||
.entry(sha256_hex.clone())
|
||||
.or_insert_with(|| RawByHashEntry::from_bytes(sha256_hex.clone(), bytes.clone()));
|
||||
|
||||
if entry.bytes != *bytes {
|
||||
return Err(format!(
|
||||
"raw_by_hash collision for {uri}: same sha256 maps to different bytes"
|
||||
));
|
||||
}
|
||||
if !entry.origin_uris.iter().any(|existing| existing == uri) {
|
||||
entry.origin_uris.push(uri.clone());
|
||||
}
|
||||
if entry.object_type.is_none() {
|
||||
entry.object_type = infer_object_type_from_uri(uri);
|
||||
}
|
||||
}
|
||||
|
||||
let hashes: Vec<String> = pending.keys().cloned().collect();
|
||||
let existing_entries = store
|
||||
.get_raw_by_hash_entries_batch(&hashes)
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut entries_to_write = Vec::new();
|
||||
for (hash, existing_opt) in hashes.into_iter().zip(existing_entries.into_iter()) {
|
||||
let mut pending_entry = pending.remove(&hash).expect("pending raw_by_hash entry");
|
||||
match existing_opt {
|
||||
Some(mut existing) => {
|
||||
if existing.bytes != pending_entry.bytes {
|
||||
return Err(format!(
|
||||
"raw_by_hash collision for hash {hash}: same sha256 maps to different bytes"
|
||||
));
|
||||
}
|
||||
let mut changed = false;
|
||||
for uri in pending_entry.origin_uris.drain(..) {
|
||||
if !existing
|
||||
.origin_uris
|
||||
.iter()
|
||||
.any(|existing_uri| existing_uri == &uri)
|
||||
{
|
||||
existing.origin_uris.push(uri);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
if existing.object_type.is_none() && pending_entry.object_type.is_some() {
|
||||
existing.object_type = pending_entry.object_type;
|
||||
changed = true;
|
||||
}
|
||||
if changed {
|
||||
entries_to_write.push(existing);
|
||||
}
|
||||
}
|
||||
None => entries_to_write.push(pending_entry),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(PreparedRawByHashBatch {
|
||||
uri_to_hash,
|
||||
entries_to_write,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn upsert_repo_blob_bytes(store: &RocksStore, bytes: &[u8]) -> Result<String, String> {
|
||||
if bytes.is_empty() {
|
||||
return Err("repo bytes must not be empty".to_string());
|
||||
}
|
||||
let sha256_hex = compute_sha256_hex(bytes);
|
||||
store
|
||||
.put_blob_bytes_batch(&[(sha256_hex.clone(), bytes.to_vec())])
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(sha256_hex)
|
||||
}
|
||||
|
||||
pub fn infer_object_type_from_uri(uri: &str) -> Option<String> {
|
||||
let ext = uri.rsplit('.').next()?;
|
||||
let ext = ext.to_ascii_lowercase();
|
||||
match ext.as_str() {
|
||||
"cer" | "crl" | "mft" | "roa" | "asa" | "gbr" | "tal" | "xml" => Some(ext),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_repository_view_present_entry(
|
||||
repository_source: &str,
|
||||
rsync_uri: &str,
|
||||
current_hash: &str,
|
||||
) -> RepositoryViewEntry {
|
||||
RepositoryViewEntry {
|
||||
rsync_uri: rsync_uri.to_string(),
|
||||
current_hash: Some(current_hash.to_string()),
|
||||
repository_source: Some(repository_source.to_string()),
|
||||
object_type: infer_object_type_from_uri(rsync_uri),
|
||||
state: RepositoryViewState::Present,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_repository_view_withdrawn_entry(
|
||||
repository_source: &str,
|
||||
rsync_uri: &str,
|
||||
current_hash: Option<String>,
|
||||
) -> RepositoryViewEntry {
|
||||
RepositoryViewEntry {
|
||||
rsync_uri: rsync_uri.to_string(),
|
||||
current_hash,
|
||||
repository_source: Some(repository_source.to_string()),
|
||||
object_type: infer_object_type_from_uri(rsync_uri),
|
||||
state: RepositoryViewState::Withdrawn,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_rrdp_source_member_present_record(
|
||||
notification_uri: &str,
|
||||
session_id: &str,
|
||||
serial: u64,
|
||||
rsync_uri: &str,
|
||||
current_hash: &str,
|
||||
) -> RrdpSourceMemberRecord {
|
||||
RrdpSourceMemberRecord {
|
||||
notify_uri: notification_uri.to_string(),
|
||||
rsync_uri: rsync_uri.to_string(),
|
||||
current_hash: Some(current_hash.to_string()),
|
||||
object_type: infer_object_type_from_uri(rsync_uri),
|
||||
present: true,
|
||||
last_confirmed_session_id: session_id.to_string(),
|
||||
last_confirmed_serial: serial,
|
||||
last_changed_at: now_pack_time(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_rrdp_source_member_withdrawn_record(
|
||||
notification_uri: &str,
|
||||
session_id: &str,
|
||||
serial: u64,
|
||||
rsync_uri: &str,
|
||||
current_hash: Option<String>,
|
||||
) -> RrdpSourceMemberRecord {
|
||||
RrdpSourceMemberRecord {
|
||||
notify_uri: notification_uri.to_string(),
|
||||
rsync_uri: rsync_uri.to_string(),
|
||||
current_hash,
|
||||
object_type: infer_object_type_from_uri(rsync_uri),
|
||||
present: false,
|
||||
last_confirmed_session_id: session_id.to_string(),
|
||||
last_confirmed_serial: serial,
|
||||
last_changed_at: now_pack_time(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_rrdp_uri_owner_active_record(
|
||||
notification_uri: &str,
|
||||
session_id: &str,
|
||||
serial: u64,
|
||||
rsync_uri: &str,
|
||||
current_hash: &str,
|
||||
) -> RrdpUriOwnerRecord {
|
||||
RrdpUriOwnerRecord {
|
||||
rsync_uri: rsync_uri.to_string(),
|
||||
notify_uri: notification_uri.to_string(),
|
||||
current_hash: Some(current_hash.to_string()),
|
||||
last_confirmed_session_id: session_id.to_string(),
|
||||
last_confirmed_serial: serial,
|
||||
last_changed_at: now_pack_time(),
|
||||
owner_state: RrdpUriOwnerState::Active,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_rrdp_uri_owner_withdrawn_record(
|
||||
notification_uri: &str,
|
||||
session_id: &str,
|
||||
serial: u64,
|
||||
rsync_uri: &str,
|
||||
current_hash: Option<String>,
|
||||
) -> RrdpUriOwnerRecord {
|
||||
RrdpUriOwnerRecord {
|
||||
rsync_uri: rsync_uri.to_string(),
|
||||
notify_uri: notification_uri.to_string(),
|
||||
current_hash,
|
||||
last_confirmed_session_id: session_id.to_string(),
|
||||
last_confirmed_serial: serial,
|
||||
last_changed_at: now_pack_time(),
|
||||
owner_state: RrdpUriOwnerState::Withdrawn,
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn upsert_raw_by_hash_evidence(
|
||||
store: &RocksStore,
|
||||
rsync_uri: &str,
|
||||
bytes: &[u8],
|
||||
) -> Result<String, String> {
|
||||
let prepared =
|
||||
prepare_raw_by_hash_evidence_batch(store, &[(rsync_uri.to_string(), bytes.to_vec())])?;
|
||||
store
|
||||
.put_raw_by_hash_entries_batch_unchecked(&prepared.entries_to_write)
|
||||
.map_err(|e| e.to_string())?;
|
||||
let sha256_hex = prepared
|
||||
.uri_to_hash
|
||||
.get(rsync_uri)
|
||||
.cloned()
|
||||
.expect("raw_by_hash mapping for upsert input");
|
||||
Ok(sha256_hex)
|
||||
}
|
||||
|
||||
pub fn put_repository_view_present(
|
||||
store: &RocksStore,
|
||||
repository_source: &str,
|
||||
rsync_uri: &str,
|
||||
current_hash: &str,
|
||||
) -> Result<(), String> {
|
||||
let entry = build_repository_view_present_entry(repository_source, rsync_uri, current_hash);
|
||||
store
|
||||
.put_repository_view_entry(&entry)
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub fn put_repository_view_withdrawn(
|
||||
store: &RocksStore,
|
||||
repository_source: &str,
|
||||
rsync_uri: &str,
|
||||
current_hash: Option<String>,
|
||||
) -> Result<(), String> {
|
||||
let entry = build_repository_view_withdrawn_entry(repository_source, rsync_uri, current_hash);
|
||||
store
|
||||
.put_repository_view_entry(&entry)
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub fn ensure_rrdp_uri_can_be_owned_by(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
rsync_uri: &str,
|
||||
) -> Result<(), String> {
|
||||
let Some(owner) = store
|
||||
.get_rrdp_uri_owner_record(rsync_uri)
|
||||
.map_err(|e| e.to_string())?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
if owner.notify_uri != notification_uri && owner.owner_state == RrdpUriOwnerState::Active {
|
||||
return Err(format!(
|
||||
"RRDP source owner conflict for {rsync_uri}: current owner {} but incoming source {}",
|
||||
owner.notify_uri, notification_uri
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn current_rrdp_owner_is(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
rsync_uri: &str,
|
||||
) -> Result<bool, String> {
|
||||
Ok(matches!(
|
||||
store
|
||||
.get_rrdp_uri_owner_record(rsync_uri)
|
||||
.map_err(|e| e.to_string())?,
|
||||
Some(owner)
|
||||
if owner.notify_uri == notification_uri && owner.owner_state == RrdpUriOwnerState::Active
|
||||
))
|
||||
}
|
||||
|
||||
pub fn put_rrdp_source_member_present(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
session_id: &str,
|
||||
serial: u64,
|
||||
rsync_uri: &str,
|
||||
current_hash: &str,
|
||||
) -> Result<(), String> {
|
||||
let record = build_rrdp_source_member_present_record(
|
||||
notification_uri,
|
||||
session_id,
|
||||
serial,
|
||||
rsync_uri,
|
||||
current_hash,
|
||||
);
|
||||
store
|
||||
.put_rrdp_source_member_record(&record)
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub fn put_rrdp_source_member_withdrawn(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
session_id: &str,
|
||||
serial: u64,
|
||||
rsync_uri: &str,
|
||||
current_hash: Option<String>,
|
||||
) -> Result<(), String> {
|
||||
let record = build_rrdp_source_member_withdrawn_record(
|
||||
notification_uri,
|
||||
session_id,
|
||||
serial,
|
||||
rsync_uri,
|
||||
current_hash,
|
||||
);
|
||||
store
|
||||
.put_rrdp_source_member_record(&record)
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub fn put_rrdp_uri_owner_active(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
session_id: &str,
|
||||
serial: u64,
|
||||
rsync_uri: &str,
|
||||
current_hash: &str,
|
||||
) -> Result<(), String> {
|
||||
let record = build_rrdp_uri_owner_active_record(
|
||||
notification_uri,
|
||||
session_id,
|
||||
serial,
|
||||
rsync_uri,
|
||||
current_hash,
|
||||
);
|
||||
store
|
||||
.put_rrdp_uri_owner_record(&record)
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub fn put_rrdp_uri_owner_withdrawn(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
session_id: &str,
|
||||
serial: u64,
|
||||
rsync_uri: &str,
|
||||
current_hash: Option<String>,
|
||||
) -> Result<(), String> {
|
||||
let record = build_rrdp_uri_owner_withdrawn_record(
|
||||
notification_uri,
|
||||
session_id,
|
||||
serial,
|
||||
rsync_uri,
|
||||
current_hash,
|
||||
);
|
||||
store
|
||||
.put_rrdp_uri_owner_record(&record)
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub fn update_rrdp_source_record_on_success(
|
||||
store: &RocksStore,
|
||||
notification_uri: &str,
|
||||
session_id: &str,
|
||||
serial: u64,
|
||||
sync_state: RrdpSourceSyncState,
|
||||
last_snapshot_uri: Option<&str>,
|
||||
last_snapshot_hash_hex: Option<&str>,
|
||||
) -> Result<(), String> {
|
||||
let now = now_pack_time();
|
||||
let mut record = match store
|
||||
.get_rrdp_source_record(notification_uri)
|
||||
.map_err(|e| e.to_string())?
|
||||
{
|
||||
Some(existing) => existing,
|
||||
None => RrdpSourceRecord {
|
||||
notify_uri: notification_uri.to_string(),
|
||||
last_session_id: None,
|
||||
last_serial: None,
|
||||
first_seen_at: now.clone(),
|
||||
last_seen_at: now.clone(),
|
||||
last_sync_at: None,
|
||||
sync_state,
|
||||
last_snapshot_uri: None,
|
||||
last_snapshot_hash: None,
|
||||
last_error: None,
|
||||
},
|
||||
};
|
||||
|
||||
record.last_session_id = Some(session_id.to_string());
|
||||
record.last_serial = Some(serial);
|
||||
record.last_seen_at = now.clone();
|
||||
record.last_sync_at = Some(now);
|
||||
record.sync_state = sync_state;
|
||||
record.last_snapshot_uri = last_snapshot_uri.map(str::to_string);
|
||||
record.last_snapshot_hash = last_snapshot_hash_hex.map(str::to_string);
|
||||
record.last_error = None;
|
||||
|
||||
store
|
||||
.put_rrdp_source_record(&record)
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub fn compute_sha256_hex(bytes: &[u8]) -> String {
|
||||
hex::encode(sha2::Sha256::digest(bytes))
|
||||
}
|
||||
|
||||
pub fn now_pack_time() -> PackTime {
|
||||
PackTime::from_utc_offset_datetime(time::OffsetDateTime::now_utc())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{prepare_repo_bytes_batch, prepare_repo_bytes_batch_owned};
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
#[test]
|
||||
fn prepare_repo_bytes_batch_deduplicates_by_hash() {
|
||||
let objects = vec![
|
||||
(
|
||||
"rsync://example.test/repo/a.roa".to_string(),
|
||||
b"same".to_vec(),
|
||||
),
|
||||
(
|
||||
"rsync://example.test/repo/b.roa".to_string(),
|
||||
b"same".to_vec(),
|
||||
),
|
||||
(
|
||||
"rsync://example.test/repo/c.roa".to_string(),
|
||||
b"other".to_vec(),
|
||||
),
|
||||
];
|
||||
let prepared = prepare_repo_bytes_batch(&objects).expect("prepare repo bytes");
|
||||
assert_eq!(prepared.uri_to_hash.len(), 3);
|
||||
assert_eq!(prepared.blobs_to_write.len(), 2);
|
||||
let unique_hashes = prepared
|
||||
.blobs_to_write
|
||||
.iter()
|
||||
.map(|(hash, _)| hash.clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
assert_eq!(unique_hashes.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_repo_bytes_batch_owned_deduplicates_without_borrowed_input() {
|
||||
let objects = vec![
|
||||
(
|
||||
"rsync://example.test/repo/a.roa".to_string(),
|
||||
b"same".to_vec(),
|
||||
),
|
||||
(
|
||||
"rsync://example.test/repo/b.roa".to_string(),
|
||||
b"same".to_vec(),
|
||||
),
|
||||
];
|
||||
let prepared = prepare_repo_bytes_batch_owned(objects).expect("prepare repo bytes");
|
||||
assert_eq!(prepared.uri_to_hash.len(), 2);
|
||||
assert_eq!(prepared.blobs_to_write.len(), 1);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user