NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260416_2 并行优化phase1后进行snapshot fast path优化，通过四方面关键优化技术消除了验证主链路上snapshot构建是的db访问性能热点问题，性能热点转移到ROA验证处理本身，目前APNIC+ARIN全量同步从500秒压缩到212秒，离rpkclient 112秒差距变小

Browse Source
This commit is contained in:
yuyr
2026-04-17 14:58:47 +08:00
parent 38421b1ae7
commit 224ae10052
36 changed files with 2935 additions and 937 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
src/parallel/repo_scheduler.rs
View File

@@ -1,11 +1,11 @@
use std::collections::HashMap;
use crate::policy::SyncPreference;
use crate::parallel::types::{
InFlightRepoEntry, RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoRuntimeState,
RepoSyncResultEnvelope, RepoSyncResultKind, RepoSyncResultRef, RepoSyncTask, RepoTaskState,
RepoTransportMode, RepoTransportResultEnvelope, RepoTransportResultKind, RepoTransportTask,
};
use crate::policy::SyncPreference;
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum RepoRequestAction {
@@ -241,7 +241,13 @@ impl TransportStateTables {
}
}
self.register_rsync_request(identity, requester, validation_time, priority, rsync_scope_uri)
self.register_rsync_request(
identity,
requester,
validation_time,
priority,
rsync_scope_uri,
)
}
fn register_rsync_request(
@@ -384,7 +390,10 @@ impl TransportStateTables {
finished_at: time::OffsetDateTime,
) -> Result<TransportCompletion, String> {
match (&result.dedup_key, &result.result) {
(RepoDedupKey::RrdpNotify { notification_uri }, RepoTransportResultKind::Success { .. }) => {
(
RepoDedupKey::RrdpNotify { notification_uri },
RepoTransportResultKind::Success { .. },
) => {
let entry = self
.rrdp_inflight
.get_mut(notification_uri)
@@ -406,7 +415,10 @@ impl TransportStateTables {
follow_up_tasks: Vec::new(),
})
}
(RepoDedupKey::RrdpNotify { notification_uri }, RepoTransportResultKind::Failed { .. }) => {
(
RepoDedupKey::RrdpNotify { notification_uri },
RepoTransportResultKind::Failed { .. },
) => {
let entry = self
.rrdp_inflight
.get_mut(notification_uri)
@@ -422,7 +434,9 @@ impl TransportStateTables {
record.state = RepoRuntimeState::RrdpFailedPendingRsync;
let rsync_scope_uri = record.rsync_scope_key.clone();
if let Some(existing) = self.rsync_inflight.get_mut(&rsync_scope_uri) {
existing.waiting_requesters.extend(record.requesters.clone());
existing
.waiting_requesters
.extend(record.requesters.clone());
record.state = RepoRuntimeState::WaitingRsync;
} else {
let task = RepoTransportTask {
@@ -458,7 +472,10 @@ impl TransportStateTables {
follow_up_tasks,
})
}
(RepoDedupKey::RsyncScope { rsync_scope_uri }, RepoTransportResultKind::Success { .. }) => {
(
RepoDedupKey::RsyncScope { rsync_scope_uri },
RepoTransportResultKind::Success { .. },
) => {
let entry = self
.rsync_inflight
.get_mut(rsync_scope_uri)
@@ -471,7 +488,8 @@ impl TransportStateTables {
if record.rsync_scope_key == *rsync_scope_uri
&& matches!(
record.state,
RepoRuntimeState::WaitingRsync | RepoRuntimeState::RrdpFailedPendingRsync
RepoRuntimeState::WaitingRsync
| RepoRuntimeState::RrdpFailedPendingRsync
)
{
record.state = RepoRuntimeState::RsyncOk;
@@ -483,7 +501,10 @@ impl TransportStateTables {
follow_up_tasks: Vec::new(),
})
}
(RepoDedupKey::RsyncScope { rsync_scope_uri }, RepoTransportResultKind::Failed { .. }) => {
(
RepoDedupKey::RsyncScope { rsync_scope_uri },
RepoTransportResultKind::Failed { .. },
) => {
let entry = self
.rsync_inflight
.get_mut(rsync_scope_uri)
@@ -496,7 +517,8 @@ impl TransportStateTables {
if record.rsync_scope_key == *rsync_scope_uri
&& matches!(
record.state,
RepoRuntimeState::WaitingRsync | RepoRuntimeState::RrdpFailedPendingRsync
RepoRuntimeState::WaitingRsync
| RepoRuntimeState::RrdpFailedPendingRsync
)
{
record.state = RepoRuntimeState::FailedTerminal;
@@ -512,7 +534,6 @@ impl TransportStateTables {
}
}
#[derive(Default)]
pub struct InFlightRepoTable {
entries: HashMap<RepoKey, InFlightRepoEntry>,
@@ -536,7 +557,9 @@ impl InFlightRepoTable {
}
pub fn last_result(&self, key: &RepoKey) -> Option<&RepoSyncResultEnvelope> {
self.entries.get(key).and_then(|entry| entry.last_result.as_ref())
self.entries
.get(key)
.and_then(|entry| entry.last_result.as_ref())
}
pub fn register_request(
@@ -626,7 +649,9 @@ impl InFlightRepoTable {
RepoSyncResultKind::Success(result_ref)
| RepoSyncResultKind::Reused(result_ref) => result_ref.clone(),
RepoSyncResultKind::Failed { detail } => {
return Err(format!("success completion called with failure result: {detail}"));
return Err(format!(
"success completion called with failure result: {detail}"
));
}
};
entry.state = RepoTaskState::Succeeded;
@@ -912,12 +937,14 @@ mod tests {
#[cfg(test)]
mod transport_tests {
use crate::policy::SyncPreference;
use crate::parallel::repo_scheduler::{TransportRequestAction, TransportStateTables, TransportTaskState};
use crate::parallel::repo_scheduler::{
TransportRequestAction, TransportStateTables, TransportTaskState,
};
use crate::parallel::types::{
RepoDedupKey, RepoIdentity, RepoRequester, RepoRuntimeState, RepoTransportMode,
RepoTransportResultEnvelope, RepoTransportResultKind,
};
use crate::policy::SyncPreference;
fn requester(id: &str) -> RepoRequester {
RepoRequester::with_tal_rir(
@@ -956,7 +983,10 @@ mod transport_tests {
}
);
let key = "https://example.test/notify.xml".to_string();
assert_eq!(tables.rrdp_inflight.get(&key).unwrap().state, TransportTaskState::Pending);
assert_eq!(
tables.rrdp_inflight.get(&key).unwrap().state,
TransportTaskState::Pending
);
assert_eq!(
tables.runtime_records.get(&identity).unwrap().state,
RepoRuntimeState::WaitingRrdp
@@ -1171,7 +1201,10 @@ mod transport_tests {
"rsync://example.test/module/".to_string(),
SyncPreference::RsyncOnly,
);
assert!(matches!(action, TransportRequestAction::ReusedTerminalFailure(_)));
assert!(matches!(
action,
TransportRequestAction::ReusedTerminalFailure(_)
));
}
#[test]