NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260416_2 并行优化phase1后进行snapshot fast path优化，通过四方面关键优化技术消除了验证主链路上snapshot构建是的db访问性能热点问题，性能热点转移到ROA验证处理本身，目前APNIC+ARIN全量同步从500秒压缩到212秒，离rpkclient 112秒差距变小

Browse Source
This commit is contained in:
yuyr
2026-04-17 14:58:47 +08:00
parent 38421b1ae7
commit 224ae10052
36 changed files with 2935 additions and 937 deletions
Download Patch File Download Diff File Expand all files Collapse all files

235
src/blob_store.rs
View File

@@ -6,11 +6,16 @@ use rocksdb::{DB, Options, WriteBatch};
use crate::storage::{RawByHashEntry, RocksStore, StorageError, StorageResult};
const RAW_BY_HASH_KEY_PREFIX: &str = "rawbyhash:";
const RAW_BLOB_KEY_PREFIX: &str = "rawblob:";
fn raw_by_hash_key(sha256_hex: &str) -> String {
format!("{RAW_BY_HASH_KEY_PREFIX}{sha256_hex}")
}
fn raw_blob_key(sha256_hex: &str) -> String {
format!("{RAW_BLOB_KEY_PREFIX}{sha256_hex}")
}
pub trait RawObjectStore {
fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>>;
@@ -24,10 +29,7 @@ pub trait RawObjectStore {
.map(|entry| entry.map(|entry| entry.bytes))
}
fn get_blob_bytes_batch(
&self,
sha256_hexes: &[String],
) -> StorageResult<Vec<Option<Vec<u8>>>> {
fn get_blob_bytes_batch(&self, sha256_hexes: &[String]) -> StorageResult<Vec<Option<Vec<u8>>>> {
self.get_raw_entries_batch(sha256_hexes).map(|entries| {
entries
.into_iter()
@@ -62,10 +64,19 @@ impl ExternalRawStoreDb {
pub fn put_raw_entry(&self, entry: &RawByHashEntry) -> StorageResult<()> {
entry.validate_internal()?;
let key = raw_by_hash_key(&entry.sha256_hex);
let value =
serde_cbor::to_vec(entry).map_err(|e| StorageError::Codec { entity: "raw_by_hash", detail: e.to_string() })?;
let blob_key = raw_blob_key(&entry.sha256_hex);
let value = serde_cbor::to_vec(entry).map_err(|e| StorageError::Codec {
entity: "raw_by_hash",
detail: e.to_string(),
})?;
let blob_value = entry.bytes.clone();
self.db
.put(key.as_bytes(), value)
.write({
let mut batch = WriteBatch::default();
batch.put(key.as_bytes(), value);
batch.put(blob_key.as_bytes(), blob_value);
batch
})
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(())
}
@@ -78,11 +89,13 @@ impl ExternalRawStoreDb {
for entry in entries {
entry.validate_internal()?;
let key = raw_by_hash_key(&entry.sha256_hex);
let blob_key = raw_blob_key(&entry.sha256_hex);
let value = serde_cbor::to_vec(entry).map_err(|e| StorageError::Codec {
entity: "raw_by_hash",
detail: e.to_string(),
})?;
batch.put(key.as_bytes(), value);
batch.put(blob_key.as_bytes(), entry.bytes.as_slice());
}
self.db
.write(batch)
@@ -92,8 +105,14 @@ impl ExternalRawStoreDb {
pub fn delete_raw_entry(&self, sha256_hex: &str) -> StorageResult<()> {
let key = raw_by_hash_key(sha256_hex);
let blob_key = raw_blob_key(sha256_hex);
self.db
.delete(key.as_bytes())
.write({
let mut batch = WriteBatch::default();
batch.delete(key.as_bytes());
batch.delete(blob_key.as_bytes());
batch
})
.map_err(|e| StorageError::RocksDb(e.to_string()))
}
@@ -113,6 +132,26 @@ impl RawObjectStore for RocksStore {
) -> StorageResult<Vec<Option<RawByHashEntry>>> {
self.get_raw_by_hash_entries_batch(sha256_hexes)
}
fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult<Option<Vec<u8>>> {
if let Some(raw_store) = self.external_raw_store_ref() {
return raw_store.get_blob_bytes(sha256_hex);
}
self.get_raw_entry(sha256_hex)
.map(|entry| entry.map(|entry| entry.bytes))
}
fn get_blob_bytes_batch(&self, sha256_hexes: &[String]) -> StorageResult<Vec<Option<Vec<u8>>>> {
if let Some(raw_store) = self.external_raw_store_ref() {
return raw_store.get_blob_bytes_batch(sha256_hexes);
}
self.get_raw_entries_batch(sha256_hexes).map(|entries| {
entries
.into_iter()
.map(|entry| entry.map(|entry| entry.bytes))
.collect()
})
}
}
impl RawObjectStore for ExternalRawStoreDb {
@@ -125,10 +164,11 @@ impl RawObjectStore for ExternalRawStoreDb {
else {
return Ok(None);
};
let entry = serde_cbor::from_slice::<RawByHashEntry>(&bytes).map_err(|e| StorageError::Codec {
entity: "raw_by_hash",
detail: e.to_string(),
})?;
let entry =
serde_cbor::from_slice::<RawByHashEntry>(&bytes).map_err(|e| StorageError::Codec {
entity: "raw_by_hash",
detail: e.to_string(),
})?;
entry.validate_internal()?;
Ok(Some(entry))
}
@@ -140,7 +180,10 @@ impl RawObjectStore for ExternalRawStoreDb {
if sha256_hexes.is_empty() {
return Ok(Vec::new());
}
let keys: Vec<String> = sha256_hexes.iter().map(|hash| raw_by_hash_key(hash)).collect();
let keys: Vec<String> = sha256_hexes
.iter()
.map(|hash| raw_by_hash_key(hash))
.collect();
self.db
.multi_get(keys.iter().map(|key| key.as_bytes()))
.into_iter()
@@ -148,12 +191,13 @@ impl RawObjectStore for ExternalRawStoreDb {
let maybe = res.map_err(|e| StorageError::RocksDb(e.to_string()))?;
match maybe {
Some(bytes) => {
let entry = serde_cbor::from_slice::<RawByHashEntry>(&bytes).map_err(|e| {
StorageError::Codec {
entity: "raw_by_hash",
detail: e.to_string(),
}
})?;
let entry =
serde_cbor::from_slice::<RawByHashEntry>(&bytes).map_err(|e| {
StorageError::Codec {
entity: "raw_by_hash",
detail: e.to_string(),
}
})?;
entry.validate_internal()?;
Ok(Some(entry))
}
@@ -162,18 +206,59 @@ impl RawObjectStore for ExternalRawStoreDb {
})
.collect()
}
fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult<Option<Vec<u8>>> {
let key = raw_blob_key(sha256_hex);
self.db
.get(key.as_bytes())
.map_err(|e| StorageError::RocksDb(e.to_string()))
}
fn get_blob_bytes_batch(&self, sha256_hexes: &[String]) -> StorageResult<Vec<Option<Vec<u8>>>> {
if sha256_hexes.is_empty() {
return Ok(Vec::new());
}
let keys: Vec<String> = sha256_hexes.iter().map(|hash| raw_blob_key(hash)).collect();
self.db
.multi_get(keys.iter().map(|key| key.as_bytes()))
.into_iter()
.map(|res| res.map_err(|e| StorageError::RocksDb(e.to_string())))
.collect()
}
}
#[cfg(test)]
mod tests {
use super::{ExternalRawStoreDb, RawObjectStore};
use crate::storage::{RawByHashEntry, RocksStore, StorageError};
use crate::storage::{RawByHashEntry, RocksStore, StorageError, StorageResult};
use std::collections::HashMap;
fn sha256_hex(bytes: &[u8]) -> String {
use sha2::{Digest, Sha256};
hex::encode(Sha256::digest(bytes))
}
#[derive(Default)]
struct MockRawStore {
entries: HashMap<String, RawByHashEntry>,
}
impl RawObjectStore for MockRawStore {
fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>> {
Ok(self.entries.get(sha256_hex).cloned())
}
fn get_raw_entries_batch(
&self,
sha256_hexes: &[String],
) -> StorageResult<Vec<Option<RawByHashEntry>>> {
Ok(sha256_hexes
.iter()
.map(|hash| self.entries.get(hash).cloned())
.collect())
}
}
#[test]
fn rocks_store_raw_object_store_reads_single_and_batch_entries() {
let td = tempfile::tempdir().expect("tempdir");
@@ -201,18 +286,27 @@ mod tests {
.get_raw_entries_batch(&[a_hash.clone(), "00".repeat(32), b_hash.clone()])
.expect("get batch");
assert_eq!(batch.len(), 3);
assert_eq!(batch[0].as_ref().map(|entry| entry.bytes.as_slice()), Some(a.as_slice()));
assert_eq!(
batch[0].as_ref().map(|entry| entry.bytes.as_slice()),
Some(a.as_slice())
);
assert!(batch[1].is_none());
assert_eq!(batch[2].as_ref().map(|entry| entry.bytes.as_slice()), Some(b.as_slice()));
assert_eq!(
batch[2].as_ref().map(|entry| entry.bytes.as_slice()),
Some(b.as_slice())
);
}
#[test]
fn external_raw_store_db_roundtrips_entries() {
let td = tempfile::tempdir().expect("tempdir");
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let raw_store =
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let mut entry = RawByHashEntry::from_bytes(sha256_hex(b"blob"), b"blob".to_vec());
entry.origin_uris.push("rsync://example.test/repo/a.cer".to_string());
entry
.origin_uris
.push("rsync://example.test/repo/a.cer".to_string());
entry.object_type = Some("cer".to_string());
raw_store.put_raw_entry(&entry).expect("put raw entry");
@@ -226,7 +320,8 @@ mod tests {
#[test]
fn external_raw_store_db_batch_writes_and_reads() {
let td = tempfile::tempdir().expect("tempdir");
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let raw_store =
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let a = RawByHashEntry::from_bytes(sha256_hex(b"a"), b"a".to_vec());
let b = RawByHashEntry::from_bytes(sha256_hex(b"b"), b"b".to_vec());
@@ -249,7 +344,9 @@ mod tests {
.expect("open raw store");
let mut entry = RawByHashEntry::from_bytes(sha256_hex(b"blob"), b"blob".to_vec());
entry.origin_uris.push("rsync://example.test/repo/blob.roa".to_string());
entry
.origin_uris
.push("rsync://example.test/repo/blob.roa".to_string());
raw_store.put_raw_entry(&entry).expect("put raw entry");
let single = raw_store
@@ -264,25 +361,81 @@ mod tests {
assert_eq!(batch, vec![Some(b"blob".to_vec()), None]);
}
#[test]
fn raw_object_store_default_blob_helpers_work_for_custom_store() {
let mut store = MockRawStore::default();
let a = RawByHashEntry::from_bytes(sha256_hex(b"a"), b"a".to_vec());
let b = RawByHashEntry::from_bytes(sha256_hex(b"b"), b"b".to_vec());
store.entries.insert(a.sha256_hex.clone(), a.clone());
store.entries.insert(b.sha256_hex.clone(), b.clone());
let single = store
.get_blob_bytes(&a.sha256_hex)
.expect("single blob bytes")
.expect("present");
assert_eq!(single, b"a".to_vec());
let batch = store
.get_blob_bytes_batch(&[a.sha256_hex.clone(), "00".repeat(32), b.sha256_hex.clone()])
.expect("batch blob bytes");
assert_eq!(batch, vec![Some(b"a".to_vec()), None, Some(b"b".to_vec())]);
}
#[test]
fn rocks_store_blob_helpers_use_external_raw_store_fast_path() {
let td = tempfile::tempdir().expect("tempdir");
let store = RocksStore::open_with_external_raw_store(
&td.path().join("db"),
&td.path().join("raw-store.db"),
)
.expect("open store with external raw store");
let entry = RawByHashEntry::from_bytes(sha256_hex(b"blob-fast"), b"blob-fast".to_vec());
store.put_raw_by_hash_entry(&entry).expect("put");
let single = store
.get_blob_bytes(&entry.sha256_hex)
.expect("single blob bytes")
.expect("present");
assert_eq!(single, b"blob-fast".to_vec());
let batch = store
.get_blob_bytes_batch(&[entry.sha256_hex.clone(), "00".repeat(32)])
.expect("batch blob bytes");
assert_eq!(batch, vec![Some(b"blob-fast".to_vec()), None]);
}
#[test]
fn external_raw_store_db_delete_removes_entry() {
let td = tempfile::tempdir().expect("tempdir");
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let raw_store =
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let entry = RawByHashEntry::from_bytes(sha256_hex(b"gone"), b"gone".to_vec());
raw_store.put_raw_entry(&entry).expect("put");
assert!(raw_store.get_raw_entry(&entry.sha256_hex).unwrap().is_some());
assert!(
raw_store
.get_raw_entry(&entry.sha256_hex)
.unwrap()
.is_some()
);
raw_store
.delete_raw_entry(&entry.sha256_hex)
.expect("delete entry");
assert!(raw_store.get_raw_entry(&entry.sha256_hex).unwrap().is_none());
assert!(
raw_store
.get_raw_entry(&entry.sha256_hex)
.unwrap()
.is_none()
);
}
#[test]
fn external_raw_store_db_rejects_invalid_entry_on_put() {
let td = tempfile::tempdir().expect("tempdir");
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let raw_store =
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let bad = RawByHashEntry {
sha256_hex: "11".repeat(32),
@@ -291,14 +444,17 @@ mod tests {
object_type: None,
encoding: None,
};
let err = raw_store.put_raw_entry(&bad).expect_err("invalid hash should fail");
let err = raw_store
.put_raw_entry(&bad)
.expect_err("invalid hash should fail");
assert!(matches!(err, StorageError::InvalidData { .. }));
}
#[test]
fn external_raw_store_db_reports_codec_error_for_corrupt_value() {
let td = tempfile::tempdir().expect("tempdir");
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let raw_store =
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
raw_store
.db
.put(b"rawbyhash:deadbeef", b"not-cbor")
@@ -307,17 +463,26 @@ mod tests {
let err = raw_store
.get_raw_entry("deadbeef")
.expect_err("corrupt value should fail");
assert!(matches!(err, StorageError::Codec { entity: "raw_by_hash", .. }));
assert!(matches!(
err,
StorageError::Codec {
entity: "raw_by_hash",
..
}
));
}
#[test]
fn external_raw_store_db_batch_returns_empty_for_empty_request() {
let td = tempfile::tempdir().expect("tempdir");
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let raw_store =
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
let entries = raw_store
.get_raw_entries_batch(&[])
.expect("empty batch succeeds");
assert!(entries.is_empty());
raw_store.put_raw_entries_batch(&[]).expect("empty put succeeds");
raw_store
.put_raw_entries_batch(&[])
.expect("empty put succeeds");
}
}