20260416_2 并行优化phase1后进行snapshot fast path优化,通过四方面关键优化技术消除了验证主链路上snapshot构建是的db访问性能热点问题,性能热点转移到ROA验证处理本身,目前APNIC+ARIN全量同步从500秒压缩到212秒,离rpkclient 112秒差距变小
This commit is contained in:
299
src/current_repo_index.rs
Normal file
299
src/current_repo_index.rs
Normal file
@@ -0,0 +1,299 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use crate::storage::{RepositoryViewEntry, RepositoryViewState};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct CurrentRepoEntry {
|
||||
pub current_hash: [u8; 32],
|
||||
pub current_hash_hex: String,
|
||||
pub repository_source: String,
|
||||
pub object_type: Option<String>,
|
||||
pub state: RepositoryViewState,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct CurrentRepoObject {
|
||||
pub rsync_uri: String,
|
||||
pub current_hash_hex: String,
|
||||
pub repository_source: String,
|
||||
pub object_type: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
pub struct CurrentRepoIndex {
|
||||
by_uri: HashMap<String, CurrentRepoEntry>,
|
||||
by_scope: HashMap<String, HashSet<String>>,
|
||||
}
|
||||
|
||||
pub type CurrentRepoIndexHandle = Arc<Mutex<CurrentRepoIndex>>;
|
||||
|
||||
impl CurrentRepoIndex {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn shared() -> CurrentRepoIndexHandle {
|
||||
Arc::new(Mutex::new(Self::new()))
|
||||
}
|
||||
|
||||
pub fn get_by_uri(&self, rsync_uri: &str) -> Option<&CurrentRepoEntry> {
|
||||
self.by_uri.get(rsync_uri)
|
||||
}
|
||||
|
||||
pub fn list_scope_uris(&self, repository_source: &str) -> Vec<String> {
|
||||
let mut out = self
|
||||
.by_scope
|
||||
.get(repository_source)
|
||||
.map(|set| set.iter().cloned().collect::<Vec<_>>())
|
||||
.unwrap_or_default();
|
||||
out.sort();
|
||||
out
|
||||
}
|
||||
|
||||
pub fn active_uri_count(&self) -> usize {
|
||||
self.by_uri.len()
|
||||
}
|
||||
|
||||
pub fn scope_count(&self) -> usize {
|
||||
self.by_scope.len()
|
||||
}
|
||||
|
||||
pub fn snapshot_objects(&self) -> Vec<CurrentRepoObject> {
|
||||
let mut out = self
|
||||
.by_uri
|
||||
.iter()
|
||||
.map(|(rsync_uri, entry)| CurrentRepoObject {
|
||||
rsync_uri: rsync_uri.clone(),
|
||||
current_hash_hex: entry.current_hash_hex.clone(),
|
||||
repository_source: entry.repository_source.clone(),
|
||||
object_type: entry.object_type.clone(),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
out.sort();
|
||||
out
|
||||
}
|
||||
|
||||
pub fn apply_repository_view_entries(
|
||||
&mut self,
|
||||
entries: &[RepositoryViewEntry],
|
||||
) -> Result<(), String> {
|
||||
for entry in entries {
|
||||
self.apply_repository_view_entry(entry)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn apply_repository_view_entry(&mut self, entry: &RepositoryViewEntry) -> Result<(), String> {
|
||||
entry.validate_internal().map_err(|e| e.to_string())?;
|
||||
|
||||
let old_scope = self
|
||||
.by_uri
|
||||
.get(&entry.rsync_uri)
|
||||
.map(|existing| existing.repository_source.clone());
|
||||
|
||||
match entry.state {
|
||||
RepositoryViewState::Present | RepositoryViewState::Replaced => {
|
||||
let repository_source = entry.repository_source.clone().ok_or_else(|| {
|
||||
format!(
|
||||
"repository_view entry missing repository_source for current object {}",
|
||||
entry.rsync_uri
|
||||
)
|
||||
})?;
|
||||
let current_hash_hex = entry.current_hash.clone().ok_or_else(|| {
|
||||
format!(
|
||||
"repository_view entry missing current_hash for current object {}",
|
||||
entry.rsync_uri
|
||||
)
|
||||
})?;
|
||||
let current_hash = decode_sha256_hex_32(¤t_hash_hex)?;
|
||||
|
||||
if let Some(old_scope) = old_scope.as_ref() {
|
||||
if old_scope != &repository_source {
|
||||
self.remove_uri_from_scope(old_scope, &entry.rsync_uri);
|
||||
}
|
||||
}
|
||||
|
||||
self.by_scope
|
||||
.entry(repository_source.clone())
|
||||
.or_default()
|
||||
.insert(entry.rsync_uri.clone());
|
||||
self.by_uri.insert(
|
||||
entry.rsync_uri.clone(),
|
||||
CurrentRepoEntry {
|
||||
current_hash,
|
||||
current_hash_hex: current_hash_hex.to_ascii_lowercase(),
|
||||
repository_source,
|
||||
object_type: entry.object_type.clone(),
|
||||
state: entry.state,
|
||||
},
|
||||
);
|
||||
}
|
||||
RepositoryViewState::Withdrawn => {
|
||||
if let Some(scope) = entry.repository_source.as_ref().or(old_scope.as_ref()) {
|
||||
self.remove_uri_from_scope(scope, &entry.rsync_uri);
|
||||
}
|
||||
self.by_uri.remove(&entry.rsync_uri);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn remove_uri_from_scope(&mut self, scope: &str, rsync_uri: &str) {
|
||||
let empty = if let Some(entries) = self.by_scope.get_mut(scope) {
|
||||
entries.remove(rsync_uri);
|
||||
entries.is_empty()
|
||||
} else {
|
||||
false
|
||||
};
|
||||
if empty {
|
||||
self.by_scope.remove(scope);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_sha256_hex_32(value: &str) -> Result<[u8; 32], String> {
|
||||
if value.len() != 64 || !value.as_bytes().iter().all(u8::is_ascii_hexdigit) {
|
||||
return Err(format!("invalid sha256 hex: {value}"));
|
||||
}
|
||||
let mut out = [0u8; 32];
|
||||
hex::decode_to_slice(value, &mut out).map_err(|e| format!("hex decode failed: {e}"))?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::CurrentRepoIndex;
|
||||
use crate::storage::{RepositoryViewEntry, RepositoryViewState};
|
||||
|
||||
fn present(source: &str, uri: &str, hash: &str) -> RepositoryViewEntry {
|
||||
RepositoryViewEntry {
|
||||
rsync_uri: uri.to_string(),
|
||||
current_hash: Some(hash.to_string()),
|
||||
repository_source: Some(source.to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_repo_index_tracks_present_and_withdrawn_entries() {
|
||||
let mut index = CurrentRepoIndex::new();
|
||||
let uri = "rsync://example.test/repo/a.roa";
|
||||
let source = "rsync://example.test/repo/";
|
||||
let hash = &"11".repeat(32);
|
||||
index
|
||||
.apply_repository_view_entries(&[present(source, uri, hash)])
|
||||
.expect("apply present");
|
||||
let got = index.get_by_uri(uri).expect("current entry");
|
||||
assert_eq!(got.current_hash_hex, hash.to_string());
|
||||
assert_eq!(index.list_scope_uris(source), vec![uri.to_string()]);
|
||||
|
||||
index
|
||||
.apply_repository_view_entries(&[RepositoryViewEntry {
|
||||
rsync_uri: uri.to_string(),
|
||||
current_hash: Some(hash.to_string()),
|
||||
repository_source: Some(source.to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Withdrawn,
|
||||
}])
|
||||
.expect("apply withdrawn");
|
||||
assert!(index.get_by_uri(uri).is_none());
|
||||
assert!(index.list_scope_uris(source).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_repo_index_moves_uri_between_scopes() {
|
||||
let mut index = CurrentRepoIndex::new();
|
||||
let uri = "rsync://example.test/repo/a.roa";
|
||||
let old_scope = "rsync://example.test/repo/";
|
||||
let new_scope = "https://rrdp.example.test/notification.xml";
|
||||
index
|
||||
.apply_repository_view_entries(&[present(old_scope, uri, &"22".repeat(32))])
|
||||
.expect("apply old scope");
|
||||
index
|
||||
.apply_repository_view_entries(&[present(new_scope, uri, &"33".repeat(32))])
|
||||
.expect("apply new scope");
|
||||
|
||||
assert!(index.list_scope_uris(old_scope).is_empty());
|
||||
assert_eq!(index.list_scope_uris(new_scope), vec![uri.to_string()]);
|
||||
assert_eq!(
|
||||
index.get_by_uri(uri).expect("entry").current_hash_hex,
|
||||
"33".repeat(32)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_repo_index_snapshot_objects_and_counts_are_sorted() {
|
||||
let handle = CurrentRepoIndex::shared();
|
||||
let mut index = handle.lock().expect("lock index");
|
||||
index
|
||||
.apply_repository_view_entries(&[
|
||||
present("rsync://example.test/repo-b/", "rsync://example.test/repo-b/b.roa", &"22".repeat(32)),
|
||||
present("rsync://example.test/repo-a/", "rsync://example.test/repo-a/a.roa", &"11".repeat(32)),
|
||||
])
|
||||
.expect("apply present entries");
|
||||
assert_eq!(index.active_uri_count(), 2);
|
||||
assert_eq!(index.scope_count(), 2);
|
||||
|
||||
let snapshot = index.snapshot_objects();
|
||||
assert_eq!(snapshot.len(), 2);
|
||||
assert_eq!(snapshot[0].rsync_uri, "rsync://example.test/repo-a/a.roa");
|
||||
assert_eq!(snapshot[1].rsync_uri, "rsync://example.test/repo-b/b.roa");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_repo_index_reports_missing_fields_and_invalid_hash() {
|
||||
let mut index = CurrentRepoIndex::new();
|
||||
let err = index
|
||||
.apply_repository_view_entries(&[RepositoryViewEntry {
|
||||
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
|
||||
current_hash: Some("11".repeat(32)),
|
||||
repository_source: None,
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
}])
|
||||
.expect_err("missing source should fail");
|
||||
assert!(err.contains("missing repository_source"), "{err}");
|
||||
|
||||
let err = index
|
||||
.apply_repository_view_entries(&[RepositoryViewEntry {
|
||||
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
|
||||
current_hash: Some("not-a-valid-sha256".to_string()),
|
||||
repository_source: Some("rsync://example.test/repo/".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
}])
|
||||
.expect_err("invalid hash should fail");
|
||||
assert!(err.contains("invalid"), "{err}");
|
||||
|
||||
let err = index
|
||||
.apply_repository_view_entries(&[RepositoryViewEntry {
|
||||
rsync_uri: "rsync://example.test/repo/b.roa".to_string(),
|
||||
current_hash: Some("22".repeat(32)),
|
||||
repository_source: Some("rsync://example.test/repo/".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Present,
|
||||
}])
|
||||
.expect("valid entry");
|
||||
let got = index.get_by_uri("rsync://example.test/repo/b.roa").unwrap();
|
||||
assert_eq!(got.current_hash_hex, "22".repeat(32));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_repo_index_withdraw_unknown_uri_is_noop() {
|
||||
let mut index = CurrentRepoIndex::new();
|
||||
index
|
||||
.apply_repository_view_entries(&[RepositoryViewEntry {
|
||||
rsync_uri: "rsync://example.test/repo/missing.roa".to_string(),
|
||||
current_hash: None,
|
||||
repository_source: Some("rsync://example.test/repo/".to_string()),
|
||||
object_type: Some("roa".to_string()),
|
||||
state: RepositoryViewState::Withdrawn,
|
||||
}])
|
||||
.expect("withdraw unknown should not fail");
|
||||
assert_eq!(index.active_uri_count(), 0);
|
||||
assert_eq!(index.scope_count(), 0);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user