NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260418_2 phase2 并行优化 mix quick 耗时105/48秒

Browse Source
This commit is contained in:
yuyr
2026-04-19 00:08:29 +08:00
parent 417c82bef6
commit f6a601e16c
21 changed files with 4127 additions and 594 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
src/parallel/config.rs
View File

@@ -5,6 +5,23 @@ pub struct ParallelPhase1Config {
pub max_pending_repo_results: usize,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ParallelPhase2Config {
pub object_workers: usize,
pub worker_queue_capacity: usize,
}
impl Default for ParallelPhase2Config {
fn default() -> Self {
Self {
object_workers: std::thread::available_parallelism()
.map(|n| n.get().max(1))
.unwrap_or(4),
worker_queue_capacity: 256,
}
}
}
impl Default for ParallelPhase1Config {
fn default() -> Self {
Self {
@@ -17,7 +34,7 @@ impl Default for ParallelPhase1Config {
#[cfg(test)]
mod tests {
use super::ParallelPhase1Config;
use super::{ParallelPhase1Config, ParallelPhase2Config};
#[test]
fn default_parallel_phase1_config_is_bounded() {
@@ -26,4 +43,11 @@ mod tests {
assert!(cfg.max_inflight_snapshot_bytes_global > 0);
assert!(cfg.max_pending_repo_results > 0);
}
#[test]
fn default_parallel_phase2_config_is_bounded() {
let cfg = ParallelPhase2Config::default();
assert!(cfg.object_workers > 0);
assert!(cfg.worker_queue_capacity > 0);
}
}

2
src/parallel/mod.rs
View File

@@ -1,4 +1,6 @@
pub mod config;
pub mod object_worker;
pub mod phase2_scheduler;
pub mod repo_runtime;
pub mod repo_scheduler;
pub mod repo_worker;

287
src/parallel/object_worker.rs Normal file
View File

@@ -0,0 +1,287 @@
use std::sync::mpsc::{self, Receiver, RecvTimeoutError, SyncSender, TrySendError};
use std::sync::Arc;
use std::thread::{self, JoinHandle};
use std::time::Duration;
pub trait ObjectTaskExecutor<T, R>: Send + Sync + 'static {
fn execute(&self, worker_index: usize, task: T) -> R;
}
enum ObjectWorkerMessage<T> {
Task(T),
Shutdown,
}
#[derive(Debug)]
pub enum ObjectWorkerSubmitError<T> {
QueueFull { worker_index: usize, task: T },
Disconnected { worker_index: usize, task: T },
}
pub struct ObjectWorkerPool<T, R, E>
where
T: Send + 'static,
R: Send + 'static,
E: ObjectTaskExecutor<T, R>,
{
task_txs: Vec<SyncSender<ObjectWorkerMessage<T>>>,
result_rx: Receiver<R>,
workers: Vec<JoinHandle<()>>,
next_worker_idx: usize,
_executor: Arc<E>,
}
impl<T, R, E> ObjectWorkerPool<T, R, E>
where
T: Send + 'static,
R: Send + 'static,
E: ObjectTaskExecutor<T, R>,
{
pub fn new(worker_count: usize, queue_capacity: usize, executor: E) -> Result<Self, String> {
if worker_count == 0 {
return Err("ObjectWorkerPool requires at least one worker".to_string());
}
if queue_capacity == 0 {
return Err("ObjectWorkerPool requires queue_capacity > 0".to_string());
}
let executor = Arc::new(executor);
let (result_tx, result_rx) = mpsc::channel::<R>();
let mut task_txs = Vec::with_capacity(worker_count);
let mut workers = Vec::with_capacity(worker_count);
for worker_index in 0..worker_count {
let (task_tx, task_rx) = mpsc::sync_channel::<ObjectWorkerMessage<T>>(queue_capacity);
let result_tx = result_tx.clone();
let executor = Arc::clone(&executor);
let handle = thread::Builder::new()
.name(format!("object-validation-worker-{worker_index}"))
.spawn(move || object_worker_loop(worker_index, task_rx, result_tx, executor))
.map_err(|e| format!("spawn object worker failed: {e}"))?;
task_txs.push(task_tx);
workers.push(handle);
}
Ok(Self {
task_txs,
result_rx,
workers,
next_worker_idx: 0,
_executor: executor,
})
}
pub fn worker_count(&self) -> usize {
self.task_txs.len()
}
pub fn next_worker_index(&self) -> usize {
self.next_worker_idx
}
pub fn try_submit_round_robin(&mut self, task: T) -> Result<usize, ObjectWorkerSubmitError<T>> {
let worker_index = self.next_worker_idx % self.task_txs.len();
match self.task_txs[worker_index].try_send(ObjectWorkerMessage::Task(task)) {
Ok(()) => {
self.next_worker_idx = (worker_index + 1) % self.task_txs.len();
Ok(worker_index)
}
Err(TrySendError::Full(ObjectWorkerMessage::Task(task))) => {
Err(ObjectWorkerSubmitError::QueueFull { worker_index, task })
}
Err(TrySendError::Disconnected(ObjectWorkerMessage::Task(task))) => {
Err(ObjectWorkerSubmitError::Disconnected { worker_index, task })
}
Err(TrySendError::Full(ObjectWorkerMessage::Shutdown))
| Err(TrySendError::Disconnected(ObjectWorkerMessage::Shutdown)) => {
unreachable!("shutdown is never submitted via try_submit_round_robin")
}
}
}
pub fn recv_result_timeout(&self, timeout: Duration) -> Result<Option<R>, String> {
match self.result_rx.recv_timeout(timeout) {
Ok(result) => Ok(Some(result)),
Err(RecvTimeoutError::Timeout) => Ok(None),
Err(RecvTimeoutError::Disconnected) => {
Err("object worker result channel disconnected".to_string())
}
}
}
pub fn shutdown(mut self) -> Result<(), String> {
self.shutdown_inner()
}
fn shutdown_inner(&mut self) -> Result<(), String> {
if self.workers.is_empty() {
return Ok(());
}
for tx in &self.task_txs {
tx.send(ObjectWorkerMessage::Shutdown)
.map_err(|e| format!("send shutdown to object worker failed: {e}"))?;
}
let mut first_err = None;
for handle in self.workers.drain(..) {
if let Err(e) = handle.join() {
if first_err.is_none() {
first_err = Some(format!("join object worker failed: {e:?}"));
}
}
}
if let Some(err) = first_err {
return Err(err);
}
Ok(())
}
}
impl<T, R, E> Drop for ObjectWorkerPool<T, R, E>
where
T: Send + 'static,
R: Send + 'static,
E: ObjectTaskExecutor<T, R>,
{
fn drop(&mut self) {
let _ = self.shutdown_inner();
}
}
fn object_worker_loop<T, R, E>(
worker_index: usize,
task_rx: Receiver<ObjectWorkerMessage<T>>,
result_tx: mpsc::Sender<R>,
executor: Arc<E>,
) where
T: Send + 'static,
R: Send + 'static,
E: ObjectTaskExecutor<T, R>,
{
loop {
match task_rx.recv() {
Ok(ObjectWorkerMessage::Task(task)) => {
let result = executor.execute(worker_index, task);
if result_tx.send(result).is_err() {
break;
}
}
Ok(ObjectWorkerMessage::Shutdown) | Err(_) => break,
}
}
}
#[cfg(test)]
mod tests {
use super::{ObjectTaskExecutor, ObjectWorkerPool, ObjectWorkerSubmitError};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Barrier};
use std::time::Duration;
#[derive(Clone)]
struct EchoExecutor;
impl ObjectTaskExecutor<u32, (usize, u32)> for EchoExecutor {
fn execute(&self, worker_index: usize, task: u32) -> (usize, u32) {
(worker_index, task)
}
}
#[test]
fn object_worker_pool_rejects_invalid_config_and_shutdowns_explicitly() {
let err = match ObjectWorkerPool::new(0, 1, EchoExecutor) {
Ok(_) => panic!("zero workers should be rejected"),
Err(err) => err,
};
assert!(err.contains("at least one worker"));
let err = match ObjectWorkerPool::new(1, 0, EchoExecutor) {
Ok(_) => panic!("zero queue should be rejected"),
Err(err) => err,
};
assert!(err.contains("queue_capacity > 0"));
let pool = ObjectWorkerPool::new(2, 1, EchoExecutor).expect("pool");
assert_eq!(pool.worker_count(), 2);
assert_eq!(pool.next_worker_index(), 0);
pool.shutdown().expect("shutdown");
}
#[test]
fn object_worker_pool_round_robin_submits_to_worker_queues() {
let mut pool = ObjectWorkerPool::new(3, 4, EchoExecutor).expect("pool");
assert_eq!(pool.try_submit_round_robin(10).expect("submit 10"), 0);
assert_eq!(pool.try_submit_round_robin(11).expect("submit 11"), 1);
assert_eq!(pool.try_submit_round_robin(12).expect("submit 12"), 2);
assert_eq!(pool.try_submit_round_robin(13).expect("submit 13"), 0);
let mut results = Vec::new();
for _ in 0..4 {
results.push(
pool.recv_result_timeout(Duration::from_secs(1))
.expect("result channel")
.expect("result"),
);
}
results.sort_by_key(|(_, task)| *task);
assert_eq!(results, vec![(0, 10), (1, 11), (2, 12), (0, 13)]);
}
struct BlockingExecutor {
barrier: Arc<Barrier>,
started: Arc<AtomicBool>,
}
impl ObjectTaskExecutor<u32, u32> for BlockingExecutor {
fn execute(&self, _worker_index: usize, task: u32) -> u32 {
self.started.store(true, Ordering::SeqCst);
self.barrier.wait();
task
}
}
#[test]
fn object_worker_pool_reports_full_worker_queue_without_advancing_round_robin() {
let barrier = Arc::new(Barrier::new(2));
let started = Arc::new(AtomicBool::new(false));
let mut pool = ObjectWorkerPool::new(
1,
1,
BlockingExecutor {
barrier: Arc::clone(&barrier),
started: Arc::clone(&started),
},
)
.expect("pool");
assert_eq!(pool.try_submit_round_robin(1).expect("first task"), 0);
let deadline = std::time::Instant::now() + Duration::from_secs(1);
while !started.load(Ordering::SeqCst) {
assert!(
std::time::Instant::now() < deadline,
"worker did not start first task"
);
std::thread::sleep(Duration::from_millis(1));
}
assert_eq!(pool.try_submit_round_robin(2).expect("queued task"), 0);
match pool.try_submit_round_robin(3) {
Err(ObjectWorkerSubmitError::QueueFull { worker_index, task }) => {
assert_eq!(worker_index, 0);
assert_eq!(task, 3);
}
other => panic!("expected queue full, got {other:?}"),
}
assert_eq!(pool.next_worker_index(), 0);
barrier.wait();
assert_eq!(
pool.recv_result_timeout(Duration::from_secs(1))
.expect("result channel"),
Some(1)
);
barrier.wait();
assert_eq!(
pool.recv_result_timeout(Duration::from_secs(1))
.expect("result channel"),
Some(2)
);
}
}

324
src/parallel/phase2_scheduler.rs Normal file
View File

@@ -0,0 +1,324 @@
use std::collections::{HashMap, VecDeque};
use crate::parallel::types::RepoIdentity;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct CaInstanceId(pub u64);
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct PublicationPointId(pub u64);
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PublicationPointState {
pub ca_instance_id: CaInstanceId,
pub pending_roa_tasks: usize,
pub child_discovery_released: bool,
pub finalized: bool,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Phase2CompletionSnapshot {
pub ca_ready_queue_empty: bool,
pub ca_waiting_repo_empty: bool,
pub repo_tasks_idle: bool,
pub pending_roa_dispatch_empty: bool,
pub worker_queues_empty: bool,
pub object_result_queue_empty: bool,
pub object_workers_idle: bool,
pub inflight_publication_points_empty: bool,
}
impl Phase2CompletionSnapshot {
pub fn is_complete(&self) -> bool {
self.ca_ready_queue_empty
&& self.ca_waiting_repo_empty
&& self.repo_tasks_idle
&& self.pending_roa_dispatch_empty
&& self.worker_queues_empty
&& self.object_result_queue_empty
&& self.object_workers_idle
&& self.inflight_publication_points_empty
}
}
#[derive(Default)]
pub struct Phase2SchedulerState<T> {
ca_waiting_repo_by_identity: HashMap<RepoIdentity, Vec<CaInstanceId>>,
ca_ready_queue: VecDeque<CaInstanceId>,
inflight_publication_points: HashMap<PublicationPointId, PublicationPointState>,
pending_roa_dispatch: VecDeque<T>,
}
impl<T> Phase2SchedulerState<T> {
pub fn new() -> Self {
Self {
ca_waiting_repo_by_identity: HashMap::new(),
ca_ready_queue: VecDeque::new(),
inflight_publication_points: HashMap::new(),
pending_roa_dispatch: VecDeque::new(),
}
}
pub fn wait_for_repo(&mut self, identity: RepoIdentity, ca_id: CaInstanceId) {
self.ca_waiting_repo_by_identity
.entry(identity)
.or_default()
.push(ca_id);
}
pub fn release_repo_waiters(&mut self, identity: &RepoIdentity) -> Vec<CaInstanceId> {
let released = self
.ca_waiting_repo_by_identity
.remove(identity)
.unwrap_or_default();
for ca_id in &released {
self.ca_ready_queue.push_back(*ca_id);
}
released
}
pub fn push_ready_ca(&mut self, ca_id: CaInstanceId) {
self.ca_ready_queue.push_back(ca_id);
}
pub fn pop_ready_ca(&mut self) -> Option<CaInstanceId> {
self.ca_ready_queue.pop_front()
}
pub fn start_publication_point(
&mut self,
pp_id: PublicationPointId,
ca_id: CaInstanceId,
pending_roa_tasks: usize,
) {
self.inflight_publication_points.insert(
pp_id,
PublicationPointState {
ca_instance_id: ca_id,
pending_roa_tasks,
child_discovery_released: false,
finalized: false,
},
);
}
pub fn mark_child_discovery_released(&mut self, pp_id: PublicationPointId) {
if let Some(state) = self.inflight_publication_points.get_mut(&pp_id) {
state.child_discovery_released = true;
}
}
pub fn enqueue_roa_task(&mut self, task: T) {
self.pending_roa_dispatch.push_back(task);
}
pub fn pop_pending_roa_dispatch(&mut self) -> Option<T> {
self.pending_roa_dispatch.pop_front()
}
pub fn push_front_pending_roa_dispatch(&mut self, task: T) {
self.pending_roa_dispatch.push_front(task);
}
pub fn record_roa_result(&mut self, pp_id: PublicationPointId) -> Option<PublicationPointId> {
let state = self.inflight_publication_points.get_mut(&pp_id)?;
state.pending_roa_tasks = state.pending_roa_tasks.saturating_sub(1);
if state.pending_roa_tasks == 0 {
state.finalized = true;
self.inflight_publication_points.remove(&pp_id);
Some(pp_id)
} else {
None
}
}
pub fn waiting_repo_len(&self) -> usize {
self.ca_waiting_repo_by_identity
.values()
.map(Vec::len)
.sum()
}
pub fn ready_queue_len(&self) -> usize {
self.ca_ready_queue.len()
}
pub fn inflight_len(&self) -> usize {
self.inflight_publication_points.len()
}
pub fn pending_roa_dispatch_len(&self) -> usize {
self.pending_roa_dispatch.len()
}
pub fn publication_point_state(
&self,
pp_id: PublicationPointId,
) -> Option<&PublicationPointState> {
self.inflight_publication_points.get(&pp_id)
}
pub fn completion_snapshot(
&self,
repo_tasks_idle: bool,
worker_queues_empty: bool,
object_result_queue_empty: bool,
object_workers_idle: bool,
) -> Phase2CompletionSnapshot {
Phase2CompletionSnapshot {
ca_ready_queue_empty: self.ca_ready_queue.is_empty(),
ca_waiting_repo_empty: self.ca_waiting_repo_by_identity.is_empty(),
repo_tasks_idle,
pending_roa_dispatch_empty: self.pending_roa_dispatch.is_empty(),
worker_queues_empty,
object_result_queue_empty,
object_workers_idle,
inflight_publication_points_empty: self.inflight_publication_points.is_empty(),
}
}
}
#[cfg(test)]
mod tests {
use super::{CaInstanceId, Phase2SchedulerState, PublicationPointId};
use crate::parallel::object_worker::{ObjectTaskExecutor, ObjectWorkerPool};
use crate::parallel::types::RepoIdentity;
use std::time::Duration;
fn identity(name: &str) -> RepoIdentity {
RepoIdentity::new(
Some(format!("https://example.test/{name}/notification.xml")),
format!("rsync://example.test/{name}/"),
)
}
#[test]
fn scheduler_repo_ready_moves_waiting_ca_to_ready_queue() {
let mut state = Phase2SchedulerState::<u64>::new();
let repo = identity("arin");
state.wait_for_repo(repo.clone(), CaInstanceId(1));
state.wait_for_repo(repo.clone(), CaInstanceId(2));
assert_eq!(state.waiting_repo_len(), 2);
let released = state.release_repo_waiters(&repo);
assert_eq!(released, vec![CaInstanceId(1), CaInstanceId(2)]);
assert_eq!(state.waiting_repo_len(), 0);
assert_eq!(state.ready_queue_len(), 2);
assert_eq!(state.pop_ready_ca(), Some(CaInstanceId(1)));
assert_eq!(state.pop_ready_ca(), Some(CaInstanceId(2)));
}
#[test]
fn scheduler_releases_child_before_roa_results_finalize_parent() {
let mut state = Phase2SchedulerState::<u64>::new();
let pp = PublicationPointId(10);
state.start_publication_point(pp, CaInstanceId(1), 2);
state.mark_child_discovery_released(pp);
state.push_ready_ca(CaInstanceId(2));
let pp_state = state.publication_point_state(pp).expect("inflight pp");
assert!(pp_state.child_discovery_released);
assert_eq!(pp_state.pending_roa_tasks, 2);
assert_eq!(state.pop_ready_ca(), Some(CaInstanceId(2)));
assert_eq!(state.record_roa_result(pp), None);
assert_eq!(state.inflight_len(), 1);
assert_eq!(state.record_roa_result(pp), Some(pp));
assert_eq!(state.inflight_len(), 0);
}
#[test]
fn scheduler_completion_requires_all_queues_and_inflight_to_be_empty() {
let mut state = Phase2SchedulerState::new();
assert!(state
.completion_snapshot(true, true, true, true)
.is_complete());
state.enqueue_roa_task(1u64);
assert!(!state
.completion_snapshot(true, true, true, true)
.is_complete());
assert_eq!(state.pop_pending_roa_dispatch(), Some(1));
assert!(state
.completion_snapshot(true, true, true, true)
.is_complete());
}
#[test]
fn scheduler_can_retry_pending_roa_task_at_front() {
let mut state = Phase2SchedulerState::new();
state.enqueue_roa_task(1u64);
state.enqueue_roa_task(2u64);
assert_eq!(state.pop_pending_roa_dispatch(), Some(1));
state.push_front_pending_roa_dispatch(3);
assert_eq!(state.pop_pending_roa_dispatch(), Some(3));
assert_eq!(state.pop_pending_roa_dispatch(), Some(2));
assert_eq!(state.pop_pending_roa_dispatch(), None);
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct TestRoaTask {
pp_id: PublicationPointId,
value: u64,
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct TestRoaResult {
worker_index: usize,
pp_id: PublicationPointId,
value: u64,
}
#[derive(Clone)]
struct TestRoaExecutor;
impl ObjectTaskExecutor<TestRoaTask, TestRoaResult> for TestRoaExecutor {
fn execute(&self, worker_index: usize, task: TestRoaTask) -> TestRoaResult {
TestRoaResult {
worker_index,
pp_id: task.pp_id,
value: task.value,
}
}
}
#[test]
fn scheduler_dispatches_pending_roa_tasks_to_workers_and_finalizes_on_results() {
let pp = PublicationPointId(42);
let mut state = Phase2SchedulerState::new();
state.start_publication_point(pp, CaInstanceId(7), 3);
state.mark_child_discovery_released(pp);
for value in 0..3 {
state.enqueue_roa_task(TestRoaTask { pp_id: pp, value });
}
let mut pool = ObjectWorkerPool::new(2, 4, TestRoaExecutor).expect("object pool");
while let Some(task) = state.pop_pending_roa_dispatch() {
pool.try_submit_round_robin(task).expect("submit task");
}
assert_eq!(state.pending_roa_dispatch_len(), 0);
let mut results = Vec::new();
for _ in 0..3 {
let result = pool
.recv_result_timeout(Duration::from_secs(1))
.expect("result channel")
.expect("result");
let finalized = state.record_roa_result(result.pp_id);
results.push(result);
if results.len() < 3 {
assert_eq!(finalized, None);
} else {
assert_eq!(finalized, Some(pp));
}
}
results.sort_by_key(|result| result.value);
assert_eq!(results[0].worker_index, 0);
assert_eq!(results[1].worker_index, 1);
assert_eq!(results[2].worker_index, 0);
assert_eq!(state.inflight_len(), 0);
assert!(state
.completion_snapshot(true, true, true, true)
.is_complete());
}
}

386
src/parallel/repo_runtime.rs
View File

@@ -22,12 +22,50 @@ pub struct RepoSyncRuntimeOutcome {
pub warnings: Vec<Warning>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum RepoSyncRequestStatus {
Ready {
identity: RepoIdentity,
outcome: RepoSyncRuntimeOutcome,
},
Pending {
identity: RepoIdentity,
state: RepoRuntimeState,
},
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RepoSyncRuntimeCompletion {
pub identity: RepoIdentity,
pub state: RepoRuntimeState,
pub outcome: RepoSyncRuntimeOutcome,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RepoSyncRuntimeEvent {
pub transport_identity: RepoIdentity,
pub completions: Vec<RepoSyncRuntimeCompletion>,
}
pub trait RepoSyncRuntime: Send + Sync {
fn sync_publication_point_repo(
&self,
ca: &CaInstanceHandle,
) -> Result<RepoSyncRuntimeOutcome, String>;
fn request_publication_point_repo(
&self,
ca: &CaInstanceHandle,
priority: u8,
) -> Result<RepoSyncRequestStatus, String>;
fn recv_repo_result_timeout(
&self,
timeout: Duration,
) -> Result<Option<RepoSyncRuntimeEvent>, String>;
fn reset_run_state(&self) -> Result<(), String>;
fn prefetch_discovered_children(
&self,
children: &[DiscoveredChildCaInstance],
@@ -71,11 +109,11 @@ impl<E: RepoTransportExecutor> Phase1RepoSyncRuntime<E> {
RepoIdentity::new(ca.rrdp_notification_uri.clone(), ca.rsync_base_uri.clone())
}
fn ensure_transport_requested(
fn request_transport_for_ca(
&self,
ca: &CaInstanceHandle,
priority: u8,
) -> Result<Option<RepoSyncRuntimeOutcome>, String> {
) -> Result<RepoSyncRequestStatus, String> {
let identity = Self::build_identity(ca);
let requester = Self::build_requester(ca);
let rsync_scope_uri = (self.rsync_scope_resolver)(&identity.rsync_base_uri);
@@ -108,7 +146,12 @@ impl<E: RepoTransportExecutor> Phase1RepoSyncRuntime<E> {
}),
);
self.drain_pending_transport_tasks()?;
Ok(None)
Ok(RepoSyncRequestStatus::Pending {
identity,
state: self
.runtime_state_for_identity(&task.repo_identity)
.unwrap_or(RepoRuntimeState::WaitingRrdp),
})
}
TransportRequestAction::Waiting { state } => {
crate::progress_log::emit(
@@ -122,7 +165,7 @@ impl<E: RepoTransportExecutor> Phase1RepoSyncRuntime<E> {
"runtime_state": format!("{state:?}"),
}),
);
Ok(None)
Ok(RepoSyncRequestStatus::Pending { identity, state })
}
TransportRequestAction::ReusedSuccess(result)
| TransportRequestAction::ReusedTerminalFailure(result) => {
@@ -140,11 +183,14 @@ impl<E: RepoTransportExecutor> Phase1RepoSyncRuntime<E> {
},
}),
);
Ok(Some(outcome_from_transport_result(
&result,
self.runtime_state_for_identity(&identity)
.unwrap_or(RepoRuntimeState::Init),
)))
Ok(RepoSyncRequestStatus::Ready {
outcome: outcome_from_transport_result(
&result,
self.runtime_state_for_identity(&identity)
.unwrap_or(RepoRuntimeState::Init),
),
identity,
})
}
}
}
@@ -182,14 +228,19 @@ impl<E: RepoTransportExecutor> Phase1RepoSyncRuntime<E> {
Ok(())
}
fn pump_one_transport_result(&self) -> Result<(), String> {
fn pump_one_transport_result(
&self,
timeout: Duration,
) -> Result<Option<RepoSyncRuntimeEvent>, String> {
let envelope = {
let pool = self.worker_pool.lock().expect("worker pool lock poisoned");
pool.recv_result_timeout(Duration::from_millis(50))?
pool.recv_result_timeout(timeout)?
};
let Some(envelope) = envelope else {
return Ok(());
return Ok(None);
};
let transport_identity = envelope.repo_identity.clone();
let completed_dedup_key = envelope.dedup_key.clone();
crate::progress_log::emit(
"phase1_repo_task_result",
serde_json::json!({
@@ -229,7 +280,38 @@ impl<E: RepoTransportExecutor> Phase1RepoSyncRuntime<E> {
}
}
self.drain_pending_transport_tasks()?;
Ok(())
let completions = {
let coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
coordinator
.finalized_runtime_records_for_transport(&completed_dedup_key)
.into_iter()
.filter_map(|record| {
let outcome = match record.state {
RepoRuntimeState::RrdpOk | RepoRuntimeState::RsyncOk => record
.last_success
.as_ref()
.map(|result| outcome_from_transport_result(result, record.state)),
RepoRuntimeState::FailedTerminal => record
.terminal_failure
.as_ref()
.map(|result| outcome_from_transport_result(result, record.state)),
_ => None,
}?;
Some(RepoSyncRuntimeCompletion {
identity: record.identity,
state: record.state,
outcome,
})
})
.collect::<Vec<_>>()
};
if completions.is_empty() {
return Ok(None);
}
Ok(Some(RepoSyncRuntimeEvent {
transport_identity,
completions,
}))
}
fn runtime_state_for_identity(&self, identity: &RepoIdentity) -> Option<RepoRuntimeState> {
@@ -264,24 +346,64 @@ impl<E: RepoTransportExecutor> RepoSyncRuntime for Phase1RepoSyncRuntime<E> {
&self,
ca: &CaInstanceHandle,
) -> Result<RepoSyncRuntimeOutcome, String> {
if let Some(done) = self.ensure_transport_requested(ca, 0)? {
return Ok(done);
if let RepoSyncRequestStatus::Ready { outcome, .. } =
self.request_publication_point_repo(ca, 0)?
{
return Ok(outcome);
}
let identity = Self::build_identity(ca);
loop {
if let Some(done) = self.resolved_outcome_for_identity(&identity) {
return Ok(done);
}
self.pump_one_transport_result()?;
let _ = self.recv_repo_result_timeout(Duration::from_millis(50))?;
}
}
fn request_publication_point_repo(
&self,
ca: &CaInstanceHandle,
priority: u8,
) -> Result<RepoSyncRequestStatus, String> {
self.request_transport_for_ca(ca, priority)
}
fn recv_repo_result_timeout(
&self,
timeout: Duration,
) -> Result<Option<RepoSyncRuntimeEvent>, String> {
self.pump_one_transport_result(timeout)
}
fn reset_run_state(&self) -> Result<(), String> {
{
let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
if coordinator.stats.repo_tasks_running != 0 {
return Err(format!(
"cannot reset repo runtime with {} repo task(s) still running",
coordinator.stats.repo_tasks_running
));
}
coordinator.reset_run_state();
}
loop {
let maybe_result = {
let pool = self.worker_pool.lock().expect("worker pool lock poisoned");
pool.recv_result_timeout(Duration::from_millis(0))?
};
if maybe_result.is_none() {
break;
}
}
Ok(())
}
fn prefetch_discovered_children(
&self,
children: &[DiscoveredChildCaInstance],
) -> Result<(), String> {
for child in children {
let _ = self.ensure_transport_requested(&child.handle, 1)?;
let _ = self.request_publication_point_repo(&child.handle, 1)?;
}
Ok(())
}
@@ -355,8 +477,8 @@ mod tests {
};
use crate::parallel::run_coordinator::GlobalRunCoordinator;
use crate::parallel::types::{
RepoTransportMode, RepoTransportResultEnvelope, RepoTransportResultKind, RepoTransportTask,
TalInputSpec,
RepoRuntimeState, RepoTransportMode, RepoTransportResultEnvelope, RepoTransportResultKind,
RepoTransportTask, TalInputSpec,
};
use crate::policy::SyncPreference;
use crate::report::Warning;
@@ -400,6 +522,31 @@ mod tests {
}
}
struct CountingSuccessTransportExecutor {
count: Arc<AtomicUsize>,
}
impl RepoTransportExecutor for CountingSuccessTransportExecutor {
fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope {
self.count.fetch_add(1, Ordering::SeqCst);
RepoTransportResultEnvelope {
dedup_key: task.dedup_key,
repo_identity: task.repo_identity,
mode: task.mode,
tal_id: task.tal_id,
rir_id: task.rir_id,
timing_ms: 7,
result: RepoTransportResultKind::Success {
source: match task.mode {
RepoTransportMode::Rrdp => "rrdp".to_string(),
RepoTransportMode::Rsync => "rsync".to_string(),
},
warnings: vec![Warning::new("transport ok")],
},
}
}
}
struct FailRrdpThenSucceedRsyncExecutor {
rrdp_count: Arc<AtomicUsize>,
rsync_count: Arc<AtomicUsize>,
@@ -468,6 +615,207 @@ mod tests {
assert_eq!(outcome.repo_sync_phase.as_deref(), Some("rrdp_ok"));
}
#[test]
fn phase1_runtime_request_repo_returns_pending_then_repo_ready_event() {
let coordinator = GlobalRunCoordinator::new(
ParallelPhase1Config::default(),
vec![TalInputSpec::from_url("https://example.test/arin.tal")],
);
let pool = RepoTransportWorkerPool::new(
RepoWorkerPoolConfig { max_workers: 1 },
SuccessTransportExecutor,
)
.expect("pool");
let runtime = Phase1RepoSyncRuntime::new(
coordinator,
pool,
Arc::new(|base: &str| base.to_string()),
SyncPreference::RrdpThenRsync,
);
let ca = sample_ca("rsync://example.test/repo/root.mft");
let status = runtime
.request_publication_point_repo(&ca, 0)
.expect("request repo");
let identity = match status {
super::RepoSyncRequestStatus::Pending { identity, state } => {
assert_eq!(state, RepoRuntimeState::WaitingRrdp);
identity
}
other => panic!("expected pending, got {other:?}"),
};
let event = runtime
.recv_repo_result_timeout(Duration::from_secs(1))
.expect("repo event")
.expect("event");
assert_eq!(event.transport_identity, identity);
assert_eq!(event.completions.len(), 1);
assert_eq!(event.completions[0].identity, identity);
assert_eq!(event.completions[0].state, RepoRuntimeState::RrdpOk);
assert!(event.completions[0].outcome.repo_sync_ok);
assert_eq!(
event.completions[0].outcome.repo_sync_source.as_deref(),
Some("rrdp")
);
}
#[test]
fn phase1_runtime_request_repo_reuses_ready_event_result() {
let coordinator = GlobalRunCoordinator::new(
ParallelPhase1Config::default(),
vec![TalInputSpec::from_url("https://example.test/arin.tal")],
);
let pool = RepoTransportWorkerPool::new(
RepoWorkerPoolConfig { max_workers: 1 },
SuccessTransportExecutor,
)
.expect("pool");
let runtime = Phase1RepoSyncRuntime::new(
coordinator,
pool,
Arc::new(|base: &str| base.to_string()),
SyncPreference::RrdpThenRsync,
);
let ca = sample_ca("rsync://example.test/repo/root.mft");
let first = runtime
.request_publication_point_repo(&ca, 0)
.expect("request repo");
assert!(matches!(
first,
super::RepoSyncRequestStatus::Pending { .. }
));
let _ = runtime
.recv_repo_result_timeout(Duration::from_secs(1))
.expect("repo event")
.expect("event");
let second = runtime
.request_publication_point_repo(&ca, 0)
.expect("request repo reused");
match second {
super::RepoSyncRequestStatus::Ready { outcome, .. } => {
assert!(outcome.repo_sync_ok);
assert_eq!(outcome.repo_sync_phase.as_deref(), Some("rrdp_ok"));
}
other => panic!("expected ready reuse, got {other:?}"),
}
}
#[test]
fn phase1_runtime_repo_event_reports_all_finalized_identities_for_shared_rrdp() {
let coordinator = GlobalRunCoordinator::new(
ParallelPhase1Config::default(),
vec![TalInputSpec::from_url("https://example.test/arin.tal")],
);
let pool = RepoTransportWorkerPool::new(
RepoWorkerPoolConfig { max_workers: 1 },
SuccessTransportExecutor,
)
.expect("pool");
let runtime = Phase1RepoSyncRuntime::new(
coordinator,
pool,
Arc::new(|base: &str| base.to_string()),
SyncPreference::RrdpThenRsync,
);
let ca1 = sample_ca("rsync://example.test/repo/root.mft");
let mut ca2 = sample_ca("rsync://example.test/other/root.mft");
ca2.rsync_base_uri = "rsync://example.test/other/".to_string();
ca2.publication_point_rsync_uri = "rsync://example.test/other/".to_string();
let id1 = match runtime
.request_publication_point_repo(&ca1, 0)
.expect("request first")
{
super::RepoSyncRequestStatus::Pending { identity, .. } => identity,
other => panic!("expected first pending, got {other:?}"),
};
let id2 = match runtime
.request_publication_point_repo(&ca2, 0)
.expect("request second")
{
super::RepoSyncRequestStatus::Pending { identity, .. } => identity,
other => panic!("expected second pending, got {other:?}"),
};
assert_ne!(id1, id2);
let event = runtime
.recv_repo_result_timeout(Duration::from_secs(1))
.expect("repo event")
.expect("event");
let mut identities = event
.completions
.iter()
.map(|completion| completion.identity.clone())
.collect::<Vec<_>>();
identities.sort_by(|a, b| a.rsync_base_uri.cmp(&b.rsync_base_uri));
let mut expected = vec![id1, id2];
expected.sort_by(|a, b| a.rsync_base_uri.cmp(&b.rsync_base_uri));
assert_eq!(identities, expected);
assert!(
event
.completions
.iter()
.all(|completion| completion.state == RepoRuntimeState::RrdpOk)
);
}
#[test]
fn phase1_runtime_reset_run_state_clears_completed_transport_reuse() {
let count = Arc::new(AtomicUsize::new(0));
let coordinator = GlobalRunCoordinator::new(
ParallelPhase1Config::default(),
vec![TalInputSpec::from_url("https://example.test/arin.tal")],
);
let pool = RepoTransportWorkerPool::new(
RepoWorkerPoolConfig { max_workers: 1 },
CountingSuccessTransportExecutor {
count: Arc::clone(&count),
},
)
.expect("pool");
let runtime = Phase1RepoSyncRuntime::new(
coordinator,
pool,
Arc::new(|base: &str| base.to_string()),
SyncPreference::RrdpThenRsync,
);
let ca = sample_ca("rsync://example.test/repo/root.mft");
assert!(matches!(
runtime
.request_publication_point_repo(&ca, 0)
.expect("first request"),
super::RepoSyncRequestStatus::Pending { .. }
));
let _ = runtime
.recv_repo_result_timeout(Duration::from_secs(1))
.expect("first event")
.expect("event");
assert_eq!(count.load(Ordering::SeqCst), 1);
assert!(matches!(
runtime
.request_publication_point_repo(&ca, 0)
.expect("ready reuse before reset"),
super::RepoSyncRequestStatus::Ready { .. }
));
runtime.reset_run_state().expect("reset");
assert!(matches!(
runtime
.request_publication_point_repo(&ca, 0)
.expect("second request after reset"),
super::RepoSyncRequestStatus::Pending { .. }
));
let _ = runtime
.recv_repo_result_timeout(Duration::from_secs(1))
.expect("second event")
.expect("event");
assert_eq!(count.load(Ordering::SeqCst), 2);
}
#[test]
fn phase1_runtime_transitions_rrdp_failure_to_rsync_success() {
let rrdp_count = Arc::new(AtomicUsize::new(0));

48
src/parallel/repo_scheduler.rs
View File

@@ -81,6 +81,38 @@ impl TransportStateTables {
self.runtime_records.get(identity)
}
pub fn finalized_runtime_records_for_transport(
&self,
dedup_key: &RepoDedupKey,
) -> Vec<RepoRuntimeRecord> {
self.runtime_records
.values()
.filter(|record| match dedup_key {
RepoDedupKey::RrdpNotify { notification_uri } => {
record.rrdp_notification_key.as_deref() == Some(notification_uri.as_str())
}
RepoDedupKey::RsyncScope { rsync_scope_uri } => {
record.rsync_scope_key == *rsync_scope_uri
}
})
.filter(|record| {
matches!(
record.state,
RepoRuntimeState::RrdpOk
| RepoRuntimeState::RsyncOk
| RepoRuntimeState::FailedTerminal
)
})
.cloned()
.collect()
}
pub fn reset_run_state(&mut self) {
self.rrdp_inflight.clear();
self.rsync_inflight.clear();
self.runtime_records.clear();
}
pub fn register_transport_request(
&mut self,
identity: RepoIdentity,
@@ -182,9 +214,9 @@ impl TransportStateTables {
entry.waiting_requesters.push(requester.clone());
self.runtime_records.insert(
identity,
identity.clone(),
RepoRuntimeRecord {
identity: entry.task.repo_identity.clone(),
identity,
state: RepoRuntimeState::WaitingRrdp,
rrdp_notification_key: Some(notification_uri),
rsync_scope_key: rsync_scope_uri,
@@ -263,9 +295,9 @@ impl TransportStateTables {
return match result.result {
RepoTransportResultKind::Success { .. } => {
self.runtime_records.insert(
identity,
identity.clone(),
RepoRuntimeRecord {
identity: entry.task.repo_identity.clone(),
identity,
state: RepoRuntimeState::RsyncOk,
rrdp_notification_key: None,
rsync_scope_key: rsync_scope_uri,
@@ -280,9 +312,9 @@ impl TransportStateTables {
}
RepoTransportResultKind::Failed { .. } => {
self.runtime_records.insert(
identity,
identity.clone(),
RepoRuntimeRecord {
identity: entry.task.repo_identity.clone(),
identity,
state: RepoRuntimeState::FailedTerminal,
rrdp_notification_key: None,
rsync_scope_key: rsync_scope_uri,
@@ -552,6 +584,10 @@ impl InFlightRepoTable {
self.entries.is_empty()
}
pub fn reset_run_state(&mut self) {
self.entries.clear();
}
pub fn get(&self, key: &RepoKey) -> Option<&InFlightRepoEntry> {
self.entries.get(key)
}

82
src/parallel/run_coordinator.rs
View File

@@ -8,7 +8,7 @@ use crate::parallel::repo_scheduler::{
};
use crate::parallel::stats::ParallelRunStats;
use crate::parallel::types::{
RepoIdentity, RepoKey, RepoRequester, RepoSyncResultEnvelope, RepoSyncTask,
RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoSyncResultEnvelope, RepoSyncTask,
RepoTransportResultEnvelope, RepoTransportTask, TalInputSpec,
};
use crate::policy::SyncPreference;
@@ -196,6 +196,25 @@ impl GlobalRunCoordinator {
) -> Option<&crate::parallel::repo_scheduler::RepoRuntimeRecord> {
self.transport_tables.runtime_record(identity)
}
pub fn finalized_runtime_records_for_transport(
&self,
dedup_key: &RepoDedupKey,
) -> Vec<crate::parallel::repo_scheduler::RepoRuntimeRecord> {
self.transport_tables
.finalized_runtime_records_for_transport(dedup_key)
}
pub fn reset_run_state(&mut self) {
self.in_flight_repos.reset_run_state();
self.transport_tables.reset_run_state();
self.pending_repo_tasks.clear();
self.pending_transport_tasks.clear();
self.stats = ParallelRunStats::default();
if let Ok(mut index) = self.current_repo_index.lock() {
index.clear();
}
}
}
#[cfg(test)]
@@ -204,10 +223,11 @@ mod tests {
use crate::parallel::repo_scheduler::RepoRequestAction;
use crate::parallel::run_coordinator::GlobalRunCoordinator;
use crate::parallel::types::{
RepoKey, RepoRequester, RepoSyncResultEnvelope, RepoSyncResultKind, RepoSyncResultRef,
TalInputSpec,
RepoIdentity, RepoKey, RepoRequester, RepoSyncResultEnvelope, RepoSyncResultKind,
RepoSyncResultRef, TalInputSpec,
};
use crate::policy::SyncPreference;
use crate::storage::{RepositoryViewEntry, RepositoryViewState};
fn requester(tal_id: &str, rir_id: &str, manifest: &str) -> RepoRequester {
RepoRequester {
@@ -259,6 +279,62 @@ mod tests {
assert_eq!(coordinator.stats.repo_queue_depth, 0);
}
#[test]
fn coordinator_reset_run_state_clears_runtime_only_state() {
let mut coordinator = GlobalRunCoordinator::new(
ParallelPhase1Config::default(),
vec![TalInputSpec::from_url("https://example.test/arin.tal")],
);
let identity = RepoIdentity::new(
Some("https://example.test/notify.xml".to_string()),
"rsync://example.test/repo/",
);
let requester = requester("arin", "arin", "rsync://example.test/repo/root.mft");
let action = coordinator.register_transport_request(
identity.clone(),
requester,
time::OffsetDateTime::UNIX_EPOCH,
0,
"rsync://example.test/repo/".to_string(),
SyncPreference::RrdpThenRsync,
);
assert!(matches!(
action,
crate::parallel::repo_scheduler::TransportRequestAction::Enqueue(_)
));
assert!(coordinator.runtime_record(&identity).is_some());
assert_eq!(coordinator.pending_transport_tasks.len(), 1);
{
let mut index = coordinator.current_repo_index.lock().expect("index lock");
index
.apply_repository_view_entries(&[RepositoryViewEntry {
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
current_hash: Some("aa".repeat(32)),
repository_source: Some("rsync://example.test/repo/".to_string()),
object_type: Some("roa".to_string()),
state: RepositoryViewState::Present,
}])
.expect("apply current object");
assert_eq!(index.active_uri_count(), 1);
}
coordinator.reset_run_state();
assert!(coordinator.runtime_record(&identity).is_none());
assert!(coordinator.pending_transport_tasks.is_empty());
assert!(coordinator.pending_repo_tasks.is_empty());
assert_eq!(
coordinator
.current_repo_index
.lock()
.expect("index lock")
.active_uri_count(),
0
);
assert_eq!(coordinator.stats.repo_tasks_total, 0);
}
#[test]
fn coordinator_merges_waiting_requesters_and_reuses_success() {
let mut coordinator = GlobalRunCoordinator::new(