NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260620 优化delta长尾控制面和PP cache快路径

Browse Source
This commit is contained in:
yuyr
2026-06-20 15:31:20 +08:00
parent bd266ef2a5
commit 184d3cb95b
33 changed files with 1951 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
src/parallel/repo_runtime.rs
View File

@@ -1,3 +1,4 @@
use std::collections::HashSet;
use std::sync::{Arc, Mutex};
use std::time::Duration;
@@ -67,6 +68,27 @@ pub trait RepoSyncRuntime: Send + Sync {
timeout: Duration,
) -> Result<Option<RepoSyncRuntimeEvent>, String>;
fn drain_repo_results_timeout(
&self,
timeout: Duration,
max_events: usize,
) -> Result<Vec<RepoSyncRuntimeEvent>, String> {
let max_events = max_events.max(1);
let mut events = Vec::new();
for index in 0..max_events {
let poll_timeout = if index == 0 {
timeout
} else {
Duration::from_millis(0)
};
let Some(event) = self.recv_repo_result_timeout(poll_timeout)? else {
break;
};
events.push(event);
}
Ok(events)
}
fn reset_run_state(&self) -> Result<(), String>;
fn prefetch_discovered_children(
@@ -87,6 +109,7 @@ pub struct Phase1RepoSyncRuntime<E: RepoTransportExecutor> {
coordinator: Mutex<GlobalRunCoordinator>,
worker_pool: Mutex<RepoTransportWorkerPool<E>>,
transport_prefetch_recorder: Option<Mutex<TransportPrefetchRecorder>>,
retry_short_rsync_scopes: Mutex<HashSet<String>>,
rsync_scope_resolver: Arc<dyn Fn(&str) -> String + Send + Sync>,
rsync_failure_scope_resolver: Arc<dyn Fn(&str) -> Option<String> + Send + Sync>,
sync_preference: SyncPreference,
@@ -119,6 +142,7 @@ impl<E: RepoTransportExecutor> Phase1RepoSyncRuntime<E> {
coordinator: Mutex::new(coordinator),
worker_pool: Mutex::new(worker_pool),
transport_prefetch_recorder: None,
retry_short_rsync_scopes: Mutex::new(HashSet::new()),
rsync_scope_resolver,
rsync_failure_scope_resolver,
sync_preference,
@@ -138,6 +162,7 @@ impl<E: RepoTransportExecutor> Phase1RepoSyncRuntime<E> {
worker_pool: Mutex::new(worker_pool),
transport_prefetch_recorder: record_transport_prefetch_requests
.then(|| Mutex::new(TransportPrefetchRecorder::default())),
retry_short_rsync_scopes: Mutex::new(HashSet::new()),
rsync_scope_resolver,
rsync_failure_scope_resolver,
sync_preference,
@@ -191,6 +216,7 @@ impl<E: RepoTransportExecutor> Phase1RepoSyncRuntime<E> {
rsync_scope_uri,
rsync_failure_scope_uri,
self.sync_preference,
false,
)
};
@@ -310,6 +336,12 @@ impl<E: RepoTransportExecutor> Phase1RepoSyncRuntime<E> {
};
let transport_identity = envelope.repo_identity.clone();
let completed_envelope = envelope.clone();
if let Some(recorder) = self.transport_prefetch_recorder.as_ref() {
recorder
.lock()
.expect("transport prefetch recorder lock poisoned")
.record_result(&envelope);
}
crate::progress_log::emit(
"phase1_repo_task_result",
serde_json::json!({
@@ -334,7 +366,19 @@ impl<E: RepoTransportExecutor> Phase1RepoSyncRuntime<E> {
};
if !completion.follow_up_tasks.is_empty() {
let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
for task in completion.follow_up_tasks {
for mut task in completion.follow_up_tasks {
if let crate::parallel::types::RepoDedupKey::RsyncScope { rsync_scope_uri } =
&task.dedup_key
{
if self
.retry_short_rsync_scopes
.lock()
.expect("retry short rsync scopes lock poisoned")
.contains(rsync_scope_uri)
{
task.retry_short_timeout = true;
}
}
crate::progress_log::emit(
"phase1_repo_task_enqueued",
serde_json::json!({
@@ -384,6 +428,27 @@ impl<E: RepoTransportExecutor> Phase1RepoSyncRuntime<E> {
}))
}
fn pump_transport_results(
&self,
timeout: Duration,
max_events: usize,
) -> Result<Vec<RepoSyncRuntimeEvent>, String> {
let max_events = max_events.max(1);
let mut events = Vec::new();
for index in 0..max_events {
let poll_timeout = if index == 0 {
timeout
} else {
Duration::from_millis(0)
};
let Some(event) = self.pump_one_transport_result(poll_timeout)? else {
break;
};
events.push(event);
}
Ok(events)
}
fn runtime_state_for_identity(&self, identity: &RepoIdentity) -> Option<RepoRuntimeState> {
let coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
coordinator
@@ -445,6 +510,14 @@ impl<E: RepoTransportExecutor> RepoSyncRuntime for Phase1RepoSyncRuntime<E> {
self.pump_one_transport_result(timeout)
}
fn drain_repo_results_timeout(
&self,
timeout: Duration,
max_events: usize,
) -> Result<Vec<RepoSyncRuntimeEvent>, String> {
self.pump_transport_results(timeout, max_events)
}
fn reset_run_state(&self) -> Result<(), String> {
{
let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
@@ -499,6 +572,12 @@ impl<E: RepoTransportExecutor> RepoSyncRuntime for Phase1RepoSyncRuntime<E> {
stats.skipped_incompatible += 1;
continue;
}
if request.retry_short_rsync_timeout() {
self.retry_short_rsync_scopes
.lock()
.expect("retry short rsync scopes lock poisoned")
.insert(current_rsync_scope_uri.clone());
}
let action = {
let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
coordinator.register_transport_request(
@@ -509,6 +588,7 @@ impl<E: RepoTransportExecutor> RepoSyncRuntime for Phase1RepoSyncRuntime<E> {
current_rsync_scope_uri,
current_rsync_failure_scope_uri,
self.sync_preference,
request.retry_short_timeout(),
)
};
@@ -957,6 +1037,64 @@ mod tests {
);
}
#[test]
fn phase1_runtime_drains_multiple_ready_transport_events() {
let count = Arc::new(AtomicUsize::new(0));
let coordinator = GlobalRunCoordinator::new(
ParallelPhase1Config::default(),
vec![TalInputSpec::from_url("https://example.test/arin.tal")],
);
let pool = RepoTransportWorkerPool::new(
RepoWorkerPoolConfig { max_workers: 2 },
CountingSuccessTransportExecutor {
count: Arc::clone(&count),
},
)
.expect("pool");
let runtime = Phase1RepoSyncRuntime::new(
coordinator,
pool,
Arc::new(|base: &str| base.to_string()),
SyncPreference::RrdpThenRsync,
);
let ca1 = sample_ca("rsync://example.test/repo/root.mft");
let mut ca2 = sample_ca("rsync://example.net/repo/root.mft");
ca2.rsync_base_uri = "rsync://example.net/repo/".to_string();
ca2.publication_point_rsync_uri = "rsync://example.net/repo/".to_string();
ca2.rrdp_notification_uri = Some("https://example.net/notify.xml".to_string());
assert!(matches!(
runtime
.request_publication_point_repo(&ca1, 0)
.expect("request ca1"),
super::RepoSyncRequestStatus::Pending { .. }
));
assert!(matches!(
runtime
.request_publication_point_repo(&ca2, 0)
.expect("request ca2"),
super::RepoSyncRequestStatus::Pending { .. }
));
let started = Instant::now();
while count.load(Ordering::SeqCst) < 2 && started.elapsed() < Duration::from_secs(1) {
std::thread::sleep(Duration::from_millis(5));
}
assert_eq!(count.load(Ordering::SeqCst), 2);
let events = runtime
.drain_repo_results_timeout(Duration::from_millis(0), 8)
.expect("drain events");
assert_eq!(events.len(), 2);
assert_eq!(
events
.iter()
.map(|event| event.completions.len())
.sum::<usize>(),
2
);
}
#[test]
fn phase1_runtime_reset_run_state_clears_completed_transport_reuse() {
let count = Arc::new(AtomicUsize::new(0));

3
src/parallel/repo_scheduler.rs
View File

@@ -284,6 +284,7 @@ impl TransportStateTables {
rsync_failure_scope_uri: None,
repo_identity: identity.clone(),
mode: RepoTransportMode::Rrdp,
retry_short_timeout: false,
tal_id: requester.tal_id.clone(),
rir_id: requester.rir_id.clone(),
validation_time,
@@ -460,6 +461,7 @@ impl TransportStateTables {
rsync_failure_scope_uri: rsync_failure_scope_uri.clone(),
repo_identity: identity.clone(),
mode: RepoTransportMode::Rsync,
retry_short_timeout: false,
tal_id: requester.tal_id.clone(),
rir_id: requester.rir_id.clone(),
validation_time,
@@ -557,6 +559,7 @@ impl TransportStateTables {
rsync_failure_scope_uri: record.rsync_failure_scope_key.clone(),
repo_identity: record.identity.clone(),
mode: RepoTransportMode::Rsync,
retry_short_timeout: false,
tal_id: first_requester.tal_id.clone(),
rir_id: first_requester.rir_id.clone(),
validation_time: record.validation_time,

76
src/parallel/repo_worker.rs
View File

@@ -12,6 +12,9 @@ use crate::analysis::timing::TimingHandle;
use crate::audit_downloads::DownloadLogHandle;
use crate::current_repo_index::CurrentRepoIndexHandle;
use crate::fetch::rsync::RsyncFetcher;
use crate::fetch::rsync_system::{
RsyncFailFastProfile, with_scoped_rsync_fail_fast_profile, with_scoped_rsync_timeout_override,
};
use crate::policy::Policy;
use crate::storage::RocksStore;
use crate::sync::repo::{
@@ -19,6 +22,8 @@ use crate::sync::repo::{
};
use crate::sync::rrdp::Fetcher;
const RETRY_SHORT_TIMEOUT: Duration = Duration::from_secs(1);
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RepoWorkerPoolConfig {
pub max_workers: usize,
@@ -75,14 +80,28 @@ impl<H: Fetcher + 'static> RepoTransportExecutor for LiveRrdpTransportExecutor<H
.notification_uri
.as_deref()
.expect("rrdp transport requires notification uri");
match run_rrdp_transport(
self.store.as_ref(),
notification_uri,
Some(&self.current_repo_index),
self.http_fetcher.as_ref(),
self.timing.as_ref(),
self.download_log.as_ref(),
) {
let sync_result = if task.retry_short_timeout {
crate::fetch::http::with_scoped_http_timeout_override(RETRY_SHORT_TIMEOUT, || {
run_rrdp_transport(
self.store.as_ref(),
notification_uri,
Some(&self.current_repo_index),
self.http_fetcher.as_ref(),
self.timing.as_ref(),
self.download_log.as_ref(),
)
})
} else {
run_rrdp_transport(
self.store.as_ref(),
notification_uri,
Some(&self.current_repo_index),
self.http_fetcher.as_ref(),
self.timing.as_ref(),
self.download_log.as_ref(),
)
};
match sync_result {
Ok(_) => RepoTransportResultEnvelope {
dedup_key: task.dedup_key,
rsync_failure_scope_uri: task.rsync_failure_scope_uri,
@@ -143,14 +162,37 @@ impl<R: RsyncFetcher + 'static> RepoTransportExecutor for LiveRsyncTransportExec
fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope {
let started = std::time::Instant::now();
debug_assert_eq!(task.mode, RepoTransportMode::Rsync);
match run_rsync_transport(
self.store.as_ref(),
&task.repo_identity.rsync_base_uri,
Some(&self.current_repo_index),
self.rsync_fetcher.as_ref(),
self.timing.as_ref(),
self.download_log.as_ref(),
) {
let sync_result = if task.retry_short_timeout {
with_scoped_rsync_timeout_override(RETRY_SHORT_TIMEOUT, || {
with_scoped_rsync_fail_fast_profile(
RsyncFailFastProfile {
initial_wall_clock_timeout: RETRY_SHORT_TIMEOUT,
max_wall_clock_timeout: RETRY_SHORT_TIMEOUT,
max_attempts: 1,
},
|| {
run_rsync_transport(
self.store.as_ref(),
&task.repo_identity.rsync_base_uri,
Some(&self.current_repo_index),
self.rsync_fetcher.as_ref(),
self.timing.as_ref(),
self.download_log.as_ref(),
)
},
)
})
} else {
run_rsync_transport(
self.store.as_ref(),
&task.repo_identity.rsync_base_uri,
Some(&self.current_repo_index),
self.rsync_fetcher.as_ref(),
self.timing.as_ref(),
self.download_log.as_ref(),
)
};
match sync_result {
Ok(_) => RepoTransportResultEnvelope {
dedup_key: task.dedup_key,
rsync_failure_scope_uri: task.rsync_failure_scope_uri,
@@ -667,6 +709,7 @@ mod tests {
rsync_failure_scope_uri: None,
repo_identity: RepoIdentity::new(Some(notification_uri.to_string()), rsync_base_uri),
mode: RepoTransportMode::Rrdp,
retry_short_timeout: false,
tal_id: "arin".to_string(),
rir_id: "arin".to_string(),
validation_time: time::OffsetDateTime::UNIX_EPOCH,
@@ -692,6 +735,7 @@ mod tests {
rsync_failure_scope_uri: None,
repo_identity: RepoIdentity::new(None, rsync_base_uri),
mode: RepoTransportMode::Rsync,
retry_short_timeout: false,
tal_id: "arin".to_string(),
rir_id: "arin".to_string(),
validation_time: time::OffsetDateTime::UNIX_EPOCH,

9
src/parallel/run_coordinator.rs
View File

@@ -125,8 +125,9 @@ impl GlobalRunCoordinator {
rsync_scope_uri: String,
rsync_failure_scope_uri: Option<String>,
sync_preference: SyncPreference,
retry_short_timeout: bool,
) -> TransportRequestAction {
let action = self.transport_tables.register_transport_request(
let mut action = self.transport_tables.register_transport_request(
identity,
requester,
validation_time,
@@ -135,8 +136,11 @@ impl GlobalRunCoordinator {
rsync_failure_scope_uri,
sync_preference,
);
match &action {
match &mut action {
TransportRequestAction::Enqueue(task) => {
if retry_short_timeout {
task.retry_short_timeout = true;
}
self.stats.repo_tasks_total += 1;
self.pending_transport_tasks.push_back(task.clone());
self.stats.repo_queue_depth = self.pending_transport_tasks.len();
@@ -308,6 +312,7 @@ mod tests {
"rsync://example.test/repo/".to_string(),
None,
SyncPreference::RrdpThenRsync,
false,
);
assert!(matches!(
action,

110
src/parallel/transport_prefetch.rs
View File

@@ -40,6 +40,10 @@ pub struct TransportPrefetchRequest {
pub rsync_failure_scope_uri: Option<String>,
pub repo_identity: TransportPrefetchRepoIdentity,
pub mode: TransportPrefetchMode,
#[serde(default)]
pub last_result: Option<TransportPrefetchLastResult>,
#[serde(default)]
pub last_rsync_result: Option<TransportPrefetchLastResult>,
pub tal_id: String,
pub rir_id: String,
pub priority: u8,
@@ -84,6 +88,8 @@ impl TransportPrefetchRequest {
rsync_failure_scope_uri,
repo_identity: TransportPrefetchRepoIdentity::from_identity(identity),
mode,
last_result: None,
last_rsync_result: None,
tal_id: requester.tal_id.clone(),
rir_id: requester.rir_id.clone(),
priority,
@@ -98,6 +104,8 @@ impl TransportPrefetchRequest {
rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(),
repo_identity: TransportPrefetchRepoIdentity::from_identity(&task.repo_identity),
mode: TransportPrefetchMode::from_mode(task.mode),
last_result: None,
last_rsync_result: None,
tal_id: task.tal_id.clone(),
rir_id: task.rir_id.clone(),
priority: task.priority,
@@ -133,6 +141,20 @@ impl TransportPrefetchRequest {
fn recorder_key(&self) -> String {
self.dedup_key.stable_key()
}
pub fn retry_short_timeout(&self) -> bool {
matches!(
self.last_result,
Some(TransportPrefetchLastResult { ok: false })
)
}
pub fn retry_short_rsync_timeout(&self) -> bool {
matches!(
self.last_rsync_result,
Some(TransportPrefetchLastResult { ok: false })
) || (self.mode == TransportPrefetchMode::Rsync && self.retry_short_timeout())
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -197,6 +219,11 @@ impl TransportPrefetchMode {
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TransportPrefetchLastResult {
pub ok: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TransportPrefetchRequester {
pub tal_id: String,
@@ -234,6 +261,8 @@ impl TransportPrefetchRequester {
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct TransportPrefetchRecorder {
requests_by_key: BTreeMap<String, TransportPrefetchRequest>,
results_by_key: BTreeMap<String, TransportPrefetchLastResult>,
rsync_results_by_scope: BTreeMap<String, TransportPrefetchLastResult>,
request_order: Vec<String>,
}
@@ -263,6 +292,31 @@ impl TransportPrefetchRecorder {
self.record_request(request);
}
pub fn record_result(&mut self, result: &crate::parallel::types::RepoTransportResultEnvelope) {
let key = TransportPrefetchDedupKey::from_repo_key(&result.dedup_key).stable_key();
let last_result = TransportPrefetchLastResult {
ok: matches!(
result.result,
crate::parallel::types::RepoTransportResultKind::Success { .. }
),
};
self.results_by_key.insert(key.clone(), last_result);
if let Some(request) = self.requests_by_key.get_mut(&key) {
request.last_result = Some(last_result);
}
if let crate::parallel::types::RepoDedupKey::RsyncScope { rsync_scope_uri } =
&result.dedup_key
{
self.rsync_results_by_scope
.insert(rsync_scope_uri.clone(), last_result);
for request in self.requests_by_key.values_mut() {
if request.rsync_scope_uri == *rsync_scope_uri {
request.last_rsync_result = Some(last_result);
}
}
}
}
pub fn snapshot(&self, sync_preference: SyncPreference) -> TransportPrefetchSnapshot {
TransportPrefetchSnapshot::new(
sync_preference,
@@ -282,6 +336,16 @@ impl TransportPrefetchRecorder {
let key = request.recorder_key();
if !self.requests_by_key.contains_key(&key) {
self.request_order.push(key.clone());
let mut request = request;
if request.last_result.is_none() {
request.last_result = self.results_by_key.get(&key).copied();
}
if request.last_rsync_result.is_none() {
request.last_rsync_result = self
.rsync_results_by_scope
.get(&request.rsync_scope_uri)
.copied();
}
self.requests_by_key.insert(key, request);
}
}
@@ -322,6 +386,7 @@ mod tests {
"rsync://example.test/repo/",
),
mode: RepoTransportMode::Rrdp,
retry_short_timeout: false,
tal_id: "apnic".to_string(),
rir_id: "apnic".to_string(),
validation_time: time::OffsetDateTime::UNIX_EPOCH,
@@ -397,4 +462,49 @@ mod tests {
assert!(snapshot.is_compatible_with(SyncPreference::RrdpThenRsync));
assert!(!snapshot.is_compatible_with(SyncPreference::RsyncOnly));
}
#[test]
fn recorder_marks_failed_rrdp_and_rsync_for_next_run_short_timeouts() {
let mut recorder = TransportPrefetchRecorder::default();
let rrdp_task = task(
"https://example.test/notification.xml",
"rsync://example.test/repo/a.mft",
);
recorder.record_task(&rrdp_task, "rsync://example.test/repo/".to_string());
recorder.record_result(&crate::parallel::types::RepoTransportResultEnvelope {
dedup_key: RepoDedupKey::RrdpNotify {
notification_uri: "https://example.test/notification.xml".to_string(),
},
rsync_failure_scope_uri: None,
repo_identity: rrdp_task.repo_identity.clone(),
mode: RepoTransportMode::Rrdp,
tal_id: "apnic".to_string(),
rir_id: "apnic".to_string(),
timing_ms: 1,
result: crate::parallel::types::RepoTransportResultKind::Failed {
detail: "timeout".to_string(),
warnings: Vec::new(),
},
});
recorder.record_result(&crate::parallel::types::RepoTransportResultEnvelope {
dedup_key: RepoDedupKey::RsyncScope {
rsync_scope_uri: "rsync://example.test/repo/".to_string(),
},
rsync_failure_scope_uri: None,
repo_identity: rrdp_task.repo_identity.clone(),
mode: RepoTransportMode::Rsync,
tal_id: "apnic".to_string(),
rir_id: "apnic".to_string(),
timing_ms: 1,
result: crate::parallel::types::RepoTransportResultKind::Failed {
detail: "timeout".to_string(),
warnings: Vec::new(),
},
});
let snapshot = recorder.snapshot(SyncPreference::RrdpThenRsync);
assert_eq!(snapshot.requests.len(), 1);
assert!(snapshot.requests[0].retry_short_timeout());
assert!(snapshot.requests[0].retry_short_rsync_timeout());
}
}

2
src/parallel/types.rs
View File

@@ -108,6 +108,7 @@ pub struct RepoTransportTask {
pub rsync_failure_scope_uri: Option<String>,
pub repo_identity: RepoIdentity,
pub mode: RepoTransportMode,
pub retry_short_timeout: bool,
pub tal_id: String,
pub rir_id: String,
pub validation_time: time::OffsetDateTime,
@@ -223,6 +224,7 @@ impl RepoSyncTask {
rsync_failure_scope_uri: None,
repo_identity: self.repo_key.as_identity(),
mode,
retry_short_timeout: false,
tal_id: self.tal_id.clone(),
rir_id: self.rir_id.clone(),
validation_time: self.validation_time,