20260801 并行控制面：stage worker池+CurrentRepoIndex RwLock化，--control-plane-stage-workers开关默认关闭(#138)

src/cli.rs

@@ -276,6 +276,9 @@ Options:
                              Legacy Phase 2 scheduler finalize time budget; dedicated finalize worker ignores it (default: 100)
   --parallel-phase2-finalize-queue-capacity <n>
                              Phase 2 dedicated finalize worker queue capacity (default: 32768)
+  --control-plane-stage-workers <n>
+                             Experimental: Phase 2 ready publication point stage worker count;
+                             0 disables the stage pool and keeps inline staging (default: 0)
 
   --rsync-local-dir <path>   Use LocalDirRsyncFetcher rooted at this directory (offline tests)
   --disable-rrdp            Disable RRDP and synchronize only via rsync
@@ -519,6 +522,15 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
                         format!("invalid --parallel-phase2-finalize-queue-capacity: {v}")
                     })?;
             }
+            "--control-plane-stage-workers" => {
+                i += 1;
+                let v = argv
+                    .get(i)
+                    .ok_or("--control-plane-stage-workers requires a value")?;
+                parallel_phase2_cfg.stage_workers = v
+                    .parse::<usize>()
+                    .map_err(|_| format!("invalid --control-plane-stage-workers: {v}"))?;
+            }
             "--db" => {
                 i += 1;
                 let v = argv.get(i).ok_or("--db requires a value")?;

src/current_repo_index.rs

@@ -1,5 +1,5 @@
 use std::collections::{HashMap, HashSet};
-use std::sync::{Arc, Mutex};
+use std::sync::{Arc, RwLock};
 
 use crate::storage::{RepositoryViewEntry, RepositoryViewState};
 
@@ -25,7 +25,12 @@ pub struct CurrentRepoIndex {
     by_uri: HashMap<String, CurrentRepoEntry>,
 }
 
-pub type CurrentRepoIndexHandle = Arc<Mutex<CurrentRepoIndex>>;
+/// Shared handle to the run-wide current repository index. Readers (phase 2
+/// publication point staging, cache lookups, output snapshots) take
+/// `.read()` and no longer exclude each other; writers (repo sync transports
+/// applying repository view entries, run-state reset) take `.write()` and
+/// stay exclusive.
+pub type CurrentRepoIndexHandle = Arc<RwLock<CurrentRepoIndex>>;
 
 impl CurrentRepoIndex {
     pub fn new() -> Self {
@@ -33,7 +38,7 @@ impl CurrentRepoIndex {
     }
 
     pub fn shared() -> CurrentRepoIndexHandle {
-        Arc::new(Mutex::new(Self::new()))
+        Arc::new(RwLock::new(Self::new()))
     }
 
     pub fn get_by_uri(&self, rsync_uri: &str) -> Option<&CurrentRepoEntry> {
@@ -205,7 +210,7 @@ mod tests {
     #[test]
     fn current_repo_index_snapshot_objects_and_counts_are_sorted() {
         let handle = CurrentRepoIndex::shared();
-        let mut index = handle.lock().expect("lock index");
+        let mut index = handle.write().expect("write-lock index");
         index
             .apply_repository_view_entries(&[
                 present(

src/parallel/config.rs

@@ -15,6 +15,9 @@ pub struct ParallelPhase2Config {
     pub publication_point_finalize_batch_size: usize,
     pub publication_point_finalize_wall_time_budget_ms: u64,
     pub publication_point_finalize_queue_capacity: usize,
+    /// Experimental: workers of the ready publication point stage pool. `0`
+    /// disables the pool and keeps the inline compute+apply staging path.
+    pub stage_workers: usize,
 }
 
 impl Default for ParallelPhase2Config {
@@ -28,6 +31,7 @@ impl Default for ParallelPhase2Config {
             publication_point_finalize_batch_size: 256,
             publication_point_finalize_wall_time_budget_ms: 100,
             publication_point_finalize_queue_capacity: 32768,
+            stage_workers: 0,
         }
     }
 }

src/parallel/object_worker.rs

@@ -1,9 +1,13 @@
+use std::marker::PhantomData;
 use std::sync::Arc;
 use std::sync::mpsc::{self, Receiver, RecvTimeoutError, SyncSender, TrySendError};
-use std::thread::{self, JoinHandle};
+use std::thread::{self, JoinHandle, Scope};
 use std::time::Duration;
 
-pub trait ObjectTaskExecutor<T, R>: Send + Sync + 'static {
+/// Executor shared by every worker of a pool. The `'static` bound required to
+/// move the executor into detached threads is expressed on the pool types
+/// instead of this trait so scoped pools can borrow their environment.
+pub trait ObjectTaskExecutor<T, R>: Send + Sync {
     fn execute(&self, worker_index: usize, task: T) -> R;
 }
 
@@ -22,7 +26,7 @@ pub struct ObjectWorkerPool<T, R, E>
 where
     T: Send + 'static,
     R: Send + 'static,
-    E: ObjectTaskExecutor<T, R>,
+    E: ObjectTaskExecutor<T, R> + 'static,
 {
     task_txs: Vec<SyncSender<ObjectWorkerMessage<T>>>,
     result_rx: Receiver<R>,
@@ -35,7 +39,7 @@ impl<T, R, E> ObjectWorkerPool<T, R, E>
 where
     T: Send + 'static,
     R: Send + 'static,
-    E: ObjectTaskExecutor<T, R>,
+    E: ObjectTaskExecutor<T, R> + 'static,
 {
     pub fn new(worker_count: usize, queue_capacity: usize, executor: E) -> Result<Self, String> {
         if worker_count == 0 {
@@ -140,7 +144,7 @@ impl<T, R, E> Drop for ObjectWorkerPool<T, R, E>
 where
     T: Send + 'static,
     R: Send + 'static,
-    E: ObjectTaskExecutor<T, R>,
+    E: ObjectTaskExecutor<T, R> + 'static,
 {
     fn drop(&mut self) {
         let _ = self.shutdown_inner();
@@ -153,8 +157,8 @@ fn object_worker_loop<T, R, E>(
     result_tx: mpsc::Sender<R>,
     executor: Arc<E>,
 ) where
-    T: Send + 'static,
-    R: Send + 'static,
+    T: Send,
+    R: Send,
     E: ObjectTaskExecutor<T, R>,
 {
     loop {
@@ -170,6 +174,120 @@ fn object_worker_loop<T, R, E>(
     }
 }
 
+/// Scoped variant of `ObjectWorkerPool`: workers are spawned on a
+/// `std::thread::Scope`, so tasks, results and the executor may borrow their
+/// environment (`'env`) instead of being `'static`. The pool never sends
+/// `Shutdown`; workers exit when every task sender is dropped, which happens
+/// when the pool itself is dropped ahead of the scope join.
+pub struct ScopedObjectWorkerPool<'scope, 'env, T, R, E>
+where
+    T: Send + 'env,
+    R: Send + 'env,
+    E: ObjectTaskExecutor<T, R> + 'env,
+{
+    task_txs: Vec<SyncSender<ObjectWorkerMessage<T>>>,
+    result_rx: Receiver<R>,
+    next_worker_idx: usize,
+    _executor: Arc<E>,
+    // Join handles are intentionally not stored: dropping a `ScopedJoinHandle`
+    // detaches the worker and the enclosing scope joins it on exit, after the
+    // dropped task senders have made every worker return.
+    _marker: PhantomData<(&'scope (), &'env ())>,
+}
+
+impl<'scope, 'env, T, R, E> ScopedObjectWorkerPool<'scope, 'env, T, R, E>
+where
+    T: Send + 'env,
+    R: Send + 'env,
+    E: ObjectTaskExecutor<T, R> + 'env,
+{
+    pub fn new(
+        scope: &'scope Scope<'scope, 'env>,
+        worker_count: usize,
+        queue_capacity: usize,
+        executor: E,
+    ) -> Result<Self, String> {
+        if worker_count == 0 {
+            return Err("ScopedObjectWorkerPool requires at least one worker".to_string());
+        }
+        if queue_capacity == 0 {
+            return Err("ScopedObjectWorkerPool requires queue_capacity > 0".to_string());
+        }
+
+        let executor = Arc::new(executor);
+        let (result_tx, result_rx) = mpsc::channel::<R>();
+        let mut task_txs = Vec::with_capacity(worker_count);
+
+        for worker_index in 0..worker_count {
+            let (task_tx, task_rx) = mpsc::sync_channel::<ObjectWorkerMessage<T>>(queue_capacity);
+            let result_tx = result_tx.clone();
+            let executor = Arc::clone(&executor);
+            thread::Builder::new()
+                .name(format!("object-validation-worker-{worker_index}"))
+                .spawn_scoped(scope, move || {
+                    object_worker_loop(worker_index, task_rx, result_tx, executor)
+                })
+                .map_err(|e| format!("spawn scoped object worker failed: {e}"))?;
+            task_txs.push(task_tx);
+        }
+
+        Ok(Self {
+            task_txs,
+            result_rx,
+            next_worker_idx: 0,
+            _executor: executor,
+            _marker: PhantomData,
+        })
+    }
+
+    pub fn worker_count(&self) -> usize {
+        self.task_txs.len()
+    }
+
+    pub fn try_submit_round_robin(&mut self, task: T) -> Result<usize, ObjectWorkerSubmitError<T>> {
+        let worker_index = self.next_worker_idx % self.task_txs.len();
+        match self.task_txs[worker_index].try_send(ObjectWorkerMessage::Task(task)) {
+            Ok(()) => {
+                self.next_worker_idx = (worker_index + 1) % self.task_txs.len();
+                Ok(worker_index)
+            }
+            Err(TrySendError::Full(ObjectWorkerMessage::Task(task))) => {
+                Err(ObjectWorkerSubmitError::QueueFull { worker_index, task })
+            }
+            Err(TrySendError::Disconnected(ObjectWorkerMessage::Task(task))) => {
+                Err(ObjectWorkerSubmitError::Disconnected { worker_index, task })
+            }
+            Err(TrySendError::Full(ObjectWorkerMessage::Shutdown))
+            | Err(TrySendError::Disconnected(ObjectWorkerMessage::Shutdown)) => {
+                unreachable!("shutdown is never submitted via try_submit_round_robin")
+            }
+        }
+    }
+
+    pub fn recv_result_timeout(&self, timeout: Duration) -> Result<Option<R>, String> {
+        match self.result_rx.recv_timeout(timeout) {
+            Ok(result) => Ok(Some(result)),
+            Err(RecvTimeoutError::Timeout) => Ok(None),
+            Err(RecvTimeoutError::Disconnected) => {
+                Err("scoped object worker result channel disconnected".to_string())
+            }
+        }
+    }
+}
+
+impl<'scope, 'env, T, R, E> Drop for ScopedObjectWorkerPool<'scope, 'env, T, R, E>
+where
+    T: Send + 'env,
+    R: Send + 'env,
+    E: ObjectTaskExecutor<T, R> + 'env,
+{
+    fn drop(&mut self) {
+        // Close every worker input queue so blocked `recv` calls return and
+        // the scoped workers exit before the enclosing scope joins them.
+        self.task_txs.clear();
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::{ObjectTaskExecutor, ObjectWorkerPool, ObjectWorkerSubmitError};
@@ -284,4 +402,84 @@ mod tests {
             Some(2)
         );
     }
+
+    struct BorrowingEchoExecutor<'a> {
+        base: &'a u32,
+    }
+
+    impl<'a> ObjectTaskExecutor<u32, u32> for BorrowingEchoExecutor<'a> {
+        fn execute(&self, _worker_index: usize, task: u32) -> u32 {
+            task + *self.base
+        }
+    }
+
+    #[test]
+    fn scoped_object_worker_pool_borrows_environment_and_processes_tasks() {
+        let base = 100u32;
+        std::thread::scope(|scope| {
+            let mut pool = super::ScopedObjectWorkerPool::new(
+                scope,
+                2,
+                2,
+                BorrowingEchoExecutor { base: &base },
+            )
+            .expect("scoped pool");
+            assert_eq!(pool.worker_count(), 2);
+            pool.try_submit_round_robin(1).expect("submit 1");
+            pool.try_submit_round_robin(2).expect("submit 2");
+            let mut results = Vec::new();
+            for _ in 0..2 {
+                results.push(
+                    pool.recv_result_timeout(Duration::from_secs(1))
+                        .expect("result channel")
+                        .expect("result"),
+                );
+            }
+            results.sort();
+            assert_eq!(results, vec![101, 102]);
+            // Dropping the pool inside the scope closes the task queues; the
+            // workers exit on their own and the scope join below must not hang.
+            drop(pool);
+        });
+    }
+
+    #[test]
+    fn scoped_object_worker_pool_reports_full_queue() {
+        std::thread::scope(|scope| {
+            let barrier = Arc::new(Barrier::new(2));
+            let started = Arc::new(AtomicBool::new(false));
+            let mut pool = super::ScopedObjectWorkerPool::new(
+                scope,
+                1,
+                1,
+                BlockingExecutor {
+                    barrier: Arc::clone(&barrier),
+                    started: Arc::clone(&started),
+                },
+            )
+            .expect("scoped pool");
+            pool.try_submit_round_robin(1).expect("first task");
+            let deadline = std::time::Instant::now() + Duration::from_secs(1);
+            while !started.load(Ordering::SeqCst) {
+                assert!(
+                    std::time::Instant::now() < deadline,
+                    "scoped worker did not start first task"
+                );
+                std::thread::sleep(Duration::from_millis(1));
+            }
+            pool.try_submit_round_robin(2).expect("queued task");
+            match pool.try_submit_round_robin(3) {
+                Err(ObjectWorkerSubmitError::QueueFull { worker_index, task }) => {
+                    assert_eq!(worker_index, 0);
+                    assert_eq!(task, 3);
+                }
+                other => panic!("expected queue full, got {other:?}"),
+            }
+            // Dropping the pool closes the task queues; releasing the barrier
+            // afterwards lets the blocked worker finish task 1, observe the
+            // closed result channel, and exit before the scope join.
+            drop(pool);
+            barrier.wait();
+        });
+    }
 }

src/parallel/repo_worker.rs

@@ -1160,7 +1160,7 @@ mod tests {
             matches!(result.result, RepoTransportResultKind::Success { .. }),
             "{result:?}"
         );
-        let index = current_repo_index.lock().expect("index lock");
+        let index = current_repo_index.read().expect("index read lock");
         assert!(
             index
                 .get_by_uri("rsync://example.test/repo/a.roa")
@@ -1217,7 +1217,7 @@ mod tests {
             result.result,
             RepoTransportResultKind::Success { .. }
         ));
-        let index = current_repo_index.lock().expect("index lock");
+        let index = current_repo_index.read().expect("index read lock");
         assert!(
             index
                 .get_by_uri("rsync://example.test/repo/a.roa")

src/parallel/run_coordinator.rs

@@ -225,7 +225,7 @@ impl GlobalRunCoordinator {
         self.pending_repo_tasks.clear();
         self.pending_transport_tasks.clear();
         self.stats = ParallelRunStats::default();
-        if let Ok(mut index) = self.current_repo_index.lock() {
+        if let Ok(mut index) = self.current_repo_index.write() {
             index.clear();
         }
     }
@@ -322,7 +322,7 @@ mod tests {
         assert_eq!(coordinator.pending_transport_tasks.len(), 1);
 
         {
-            let mut index = coordinator.current_repo_index.lock().expect("index lock");
+            let mut index = coordinator.current_repo_index.write().expect("index write lock");
             index
                 .apply_repository_view_entries(&[RepositoryViewEntry {
                     rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
@@ -343,8 +343,8 @@ mod tests {
         assert_eq!(
             coordinator
                 .current_repo_index
-                .lock()
-                .expect("index lock")
+                .read()
+                .expect("index read lock")
                 .active_uri_count(),
             0
         );

src/sync/repo.rs

@@ -672,7 +672,7 @@ fn rsync_sync_into_current_store(
         .map_err(|e| RepoSyncError::Storage(e.to_string()))?;
     if let Some(index) = current_repo_index {
         index
-            .lock()
+            .write()
             .map_err(|_| RepoSyncError::Storage("current repo index lock poisoned".to_string()))?
             .apply_repository_view_entries(&repository_view_entries)
             .map_err(RepoSyncError::Storage)?;

src/sync/rrdp.rs

@@ -1029,7 +1029,7 @@ fn hydrate_current_repo_index_from_rrdp_members(
     }
 
     index
-        .lock()
+        .write()
         .map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))?
         .apply_repository_view_entries(&entries)
         .map_err(RrdpSyncError::Storage)?;
@@ -1184,7 +1184,7 @@ fn apply_delta(
                         &current_hash,
                     );
                     index
-                        .lock()
+                        .write()
                         .map_err(|_| {
                             RrdpSyncError::Storage("current repo index lock poisoned".to_string())
                         })?
@@ -1240,7 +1240,7 @@ fn apply_delta(
                             Some(previous_hash.clone()),
                         );
                         index
-                            .lock()
+                            .write()
                             .map_err(|_| {
                                 RrdpSyncError::Storage(
                                     "current repo index lock poisoned".to_string(),

src/sync/rrdp/snapshot_apply.rs

@@ -306,7 +306,7 @@ pub(super) fn apply_snapshot_from_bufread<R: BufRead>(
         .map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
     if let Some(index) = current_repo_index {
         index
-            .lock()
+            .write()
             .map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))?
             .apply_repository_view_entries(&repository_view_entries)
             .map_err(RrdpSyncError::Storage)?;
@@ -365,7 +365,7 @@ fn flush_snapshot_publish_batch(
         .map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
     if let Some(index) = current_repo_index {
         index
-            .lock()
+            .write()
             .map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))?
             .apply_repository_view_entries(&repository_view_entries)
             .map_err(RrdpSyncError::Storage)?;

src/sync/rrdp/tests.rs

@@ -1025,7 +1025,7 @@ fn sync_from_notification_same_serial_hydrates_current_repo_index() {
     .expect("same serial no-op");
     assert_eq!(applied, 0);
 
-    let index = index.lock().expect("lock index");
+    let index = index.read().expect("read-lock index");
     assert_eq!(index.active_uri_count(), 2);
     assert!(index.get_by_uri(uri_a).is_some());
     assert!(index.get_by_uri(uri_b).is_some());
@@ -1089,7 +1089,7 @@ fn sync_from_notification_delta_hydrates_unchanged_current_repo_entries() {
     .expect("delta sync");
     assert_eq!(applied, 1);
 
-    let index = index.lock().expect("lock index");
+    let index = index.read().expect("read-lock index");
     assert_eq!(index.active_uri_count(), 3);
     assert!(
         index.get_by_uri(uri_a).is_some(),

src/validation/manifest.rs

@@ -639,7 +639,7 @@ pub(crate) fn try_build_fresh_publication_point_with_timing(
 > {
     let mut timing = FreshPublicationPointTimingBreakdown::default();
     let current_index_lock_started = std::time::Instant::now();
-    let current_index_guard = current_repo_index.and_then(|handle| handle.lock().ok());
+    let current_index_guard = current_repo_index.and_then(|handle| handle.read().ok());
     timing.current_index_lock_ms = current_index_lock_started.elapsed().as_millis() as u64;
 
     if !rsync_uri_is_under_publication_point(manifest_rsync_uri, publication_point_rsync_uri) {
@@ -1445,8 +1445,8 @@ mod tests {
         }
 
         current_index
-            .lock()
-            .expect("index lock")
+            .write()
+            .expect("index write lock")
             .apply_repository_view_entries(&entries)
             .expect("apply current index");
 

src/validation/run_tree_from_tal.rs

@@ -119,7 +119,7 @@ fn snapshot_current_repo_objects(
         return Vec::new();
     }
     current_repo_index
-        .and_then(|handle| handle.lock().ok().map(|idx| idx.snapshot_objects()))
+        .and_then(|handle| handle.read().ok().map(|idx| idx.snapshot_objects()))
         .unwrap_or_default()
 }
 
@@ -2392,8 +2392,8 @@ mod multi_tal_tests {
     fn snapshot_current_repo_objects_is_on_demand() {
         let handle = CurrentRepoIndex::shared();
         handle
-            .lock()
-            .expect("lock index")
+            .write()
+            .expect("write-lock index")
             .apply_repository_view_entries(&[RepositoryViewEntry {
                 rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
                 current_hash: Some("11".repeat(32)),

src/validation/tree_runner.rs

@@ -511,7 +511,7 @@ impl<'a> Rpkiv1PublicationPointRunner<'a> {
 
     fn current_hash_for_uri(&self, uri: &str) -> Option<[u8; 32]> {
         if let Some(index) = self.current_repo_index.as_ref() {
-            if let Ok(index) = index.lock() {
+            if let Ok(index) = index.read() {
                 if let Some(entry) = index.get_by_uri(uri) {
                     return Some(entry.current_hash);
                 }
@@ -760,7 +760,7 @@ impl<'a> Rpkiv1PublicationPointRunner<'a> {
 
     fn current_manifest_hash_hex_for_audit(&self, ca: &CaInstanceHandle) -> Option<String> {
         if let Some(index_handle) = self.current_repo_index.as_ref()
-            && let Ok(index) = index_handle.lock()
+            && let Ok(index) = index_handle.read()
             && let Some(entry) = index.get_by_uri(&ca.manifest_rsync_uri)
         {
             return Some(entry.current_hash_hex.clone());