20260413_2 并行化架构优化第一阶段,apnic 串行98s->并行74s,传输层任务并行,同repo内发布点串行
This commit is contained in:
29
src/parallel/config.rs
Normal file
29
src/parallel/config.rs
Normal file
@@ -0,0 +1,29 @@
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ParallelPhase1Config {
|
||||
pub max_repo_sync_workers_global: usize,
|
||||
pub max_inflight_snapshot_bytes_global: usize,
|
||||
pub max_pending_repo_results: usize,
|
||||
}
|
||||
|
||||
impl Default for ParallelPhase1Config {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_repo_sync_workers_global: 4,
|
||||
max_inflight_snapshot_bytes_global: 512 * 1024 * 1024,
|
||||
max_pending_repo_results: 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::ParallelPhase1Config;
|
||||
|
||||
#[test]
|
||||
fn default_parallel_phase1_config_is_bounded() {
|
||||
let cfg = ParallelPhase1Config::default();
|
||||
assert!(cfg.max_repo_sync_workers_global > 0);
|
||||
assert!(cfg.max_inflight_snapshot_bytes_global > 0);
|
||||
assert!(cfg.max_pending_repo_results > 0);
|
||||
}
|
||||
}
|
||||
7
src/parallel/mod.rs
Normal file
7
src/parallel/mod.rs
Normal file
@@ -0,0 +1,7 @@
|
||||
pub mod config;
|
||||
pub mod repo_runtime;
|
||||
pub mod repo_scheduler;
|
||||
pub mod repo_worker;
|
||||
pub mod run_coordinator;
|
||||
pub mod stats;
|
||||
pub mod types;
|
||||
548
src/parallel/repo_runtime.rs
Normal file
548
src/parallel/repo_runtime.rs
Normal file
@@ -0,0 +1,548 @@
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::parallel::repo_scheduler::TransportRequestAction;
|
||||
use crate::parallel::repo_worker::{RepoTransportExecutor, RepoTransportWorkerPool};
|
||||
use crate::parallel::run_coordinator::GlobalRunCoordinator;
|
||||
use crate::parallel::types::{
|
||||
RepoIdentity, RepoRuntimeState, RepoTransportMode, RepoTransportResultEnvelope,
|
||||
RepoTransportResultKind, RepoRequester,
|
||||
};
|
||||
use crate::policy::SyncPreference;
|
||||
use crate::report::Warning;
|
||||
use crate::validation::tree::{CaInstanceHandle, DiscoveredChildCaInstance};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RepoSyncRuntimeOutcome {
|
||||
pub repo_sync_ok: bool,
|
||||
pub repo_sync_err: Option<String>,
|
||||
pub repo_sync_source: Option<String>,
|
||||
pub repo_sync_phase: Option<String>,
|
||||
pub repo_sync_duration_ms: u64,
|
||||
pub warnings: Vec<Warning>,
|
||||
}
|
||||
|
||||
pub trait RepoSyncRuntime: Send + Sync {
|
||||
fn sync_publication_point_repo(
|
||||
&self,
|
||||
ca: &CaInstanceHandle,
|
||||
) -> Result<RepoSyncRuntimeOutcome, String>;
|
||||
|
||||
fn prefetch_discovered_children(
|
||||
&self,
|
||||
children: &[DiscoveredChildCaInstance],
|
||||
) -> Result<(), String>;
|
||||
}
|
||||
|
||||
pub struct Phase1RepoSyncRuntime<E: RepoTransportExecutor> {
|
||||
coordinator: Mutex<GlobalRunCoordinator>,
|
||||
worker_pool: Mutex<RepoTransportWorkerPool<E>>,
|
||||
rsync_scope_resolver: Arc<dyn Fn(&str) -> String + Send + Sync>,
|
||||
sync_preference: SyncPreference,
|
||||
}
|
||||
|
||||
impl<E: RepoTransportExecutor> Phase1RepoSyncRuntime<E> {
|
||||
pub fn new(
|
||||
coordinator: GlobalRunCoordinator,
|
||||
worker_pool: RepoTransportWorkerPool<E>,
|
||||
rsync_scope_resolver: Arc<dyn Fn(&str) -> String + Send + Sync>,
|
||||
sync_preference: SyncPreference,
|
||||
) -> Self {
|
||||
Self {
|
||||
coordinator: Mutex::new(coordinator),
|
||||
worker_pool: Mutex::new(worker_pool),
|
||||
rsync_scope_resolver,
|
||||
sync_preference,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_requester(ca: &CaInstanceHandle) -> RepoRequester {
|
||||
RepoRequester {
|
||||
tal_id: ca.tal_id.clone(),
|
||||
rir_id: ca.tal_id.clone(),
|
||||
parent_node_id: None,
|
||||
ca_instance_handle_id: format!("{}:{}", ca.tal_id, ca.manifest_rsync_uri),
|
||||
publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(),
|
||||
manifest_rsync_uri: ca.manifest_rsync_uri.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn build_identity(ca: &CaInstanceHandle) -> RepoIdentity {
|
||||
RepoIdentity::new(ca.rrdp_notification_uri.clone(), ca.rsync_base_uri.clone())
|
||||
}
|
||||
|
||||
fn ensure_transport_requested(
|
||||
&self,
|
||||
ca: &CaInstanceHandle,
|
||||
priority: u8,
|
||||
) -> Result<Option<RepoSyncRuntimeOutcome>, String> {
|
||||
let identity = Self::build_identity(ca);
|
||||
let requester = Self::build_requester(ca);
|
||||
let rsync_scope_uri = (self.rsync_scope_resolver)(&identity.rsync_base_uri);
|
||||
let action = {
|
||||
let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
|
||||
coordinator.register_transport_request(
|
||||
identity.clone(),
|
||||
requester,
|
||||
time::OffsetDateTime::now_utc(),
|
||||
priority,
|
||||
rsync_scope_uri,
|
||||
self.sync_preference,
|
||||
)
|
||||
};
|
||||
|
||||
match action {
|
||||
TransportRequestAction::Enqueue(task) => {
|
||||
crate::progress_log::emit(
|
||||
"phase1_repo_task_enqueued",
|
||||
serde_json::json!({
|
||||
"manifest_rsync_uri": ca.manifest_rsync_uri,
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri,
|
||||
"repo_key_notification_uri": task.repo_identity.notification_uri,
|
||||
"priority": priority,
|
||||
"transport_mode": match task.mode {
|
||||
RepoTransportMode::Rrdp => "rrdp",
|
||||
RepoTransportMode::Rsync => "rsync",
|
||||
},
|
||||
}),
|
||||
);
|
||||
self.drain_pending_transport_tasks()?;
|
||||
Ok(None)
|
||||
}
|
||||
TransportRequestAction::Waiting { state } => {
|
||||
crate::progress_log::emit(
|
||||
"phase1_repo_task_waiting",
|
||||
serde_json::json!({
|
||||
"manifest_rsync_uri": ca.manifest_rsync_uri,
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"repo_key_rsync_base_uri": identity.rsync_base_uri,
|
||||
"repo_key_notification_uri": identity.notification_uri,
|
||||
"priority": priority,
|
||||
"runtime_state": format!("{state:?}"),
|
||||
}),
|
||||
);
|
||||
Ok(None)
|
||||
}
|
||||
TransportRequestAction::ReusedSuccess(result)
|
||||
| TransportRequestAction::ReusedTerminalFailure(result) => {
|
||||
crate::progress_log::emit(
|
||||
"phase1_repo_task_reused",
|
||||
serde_json::json!({
|
||||
"manifest_rsync_uri": ca.manifest_rsync_uri,
|
||||
"publication_point_rsync_uri": ca.publication_point_rsync_uri,
|
||||
"repo_key_rsync_base_uri": identity.rsync_base_uri,
|
||||
"repo_key_notification_uri": identity.notification_uri,
|
||||
"priority": priority,
|
||||
"transport_mode": match result.mode {
|
||||
RepoTransportMode::Rrdp => "rrdp",
|
||||
RepoTransportMode::Rsync => "rsync",
|
||||
},
|
||||
}),
|
||||
);
|
||||
Ok(Some(outcome_from_transport_result(
|
||||
&result,
|
||||
self.runtime_state_for_identity(&identity)
|
||||
.unwrap_or(RepoRuntimeState::Init),
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn drain_pending_transport_tasks(&self) -> Result<(), String> {
|
||||
loop {
|
||||
let maybe_task = {
|
||||
let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
|
||||
coordinator.pop_next_transport_task()
|
||||
};
|
||||
let Some(task) = maybe_task else {
|
||||
break;
|
||||
};
|
||||
{
|
||||
let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
|
||||
coordinator.mark_transport_running(&task.dedup_key, time::OffsetDateTime::now_utc())?;
|
||||
}
|
||||
crate::progress_log::emit(
|
||||
"phase1_repo_task_dispatched",
|
||||
serde_json::json!({
|
||||
"repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri,
|
||||
"repo_key_notification_uri": task.repo_identity.notification_uri,
|
||||
"requester_count": task.requesters.len(),
|
||||
"priority": task.priority,
|
||||
"transport_mode": match task.mode {
|
||||
RepoTransportMode::Rrdp => "rrdp",
|
||||
RepoTransportMode::Rsync => "rsync",
|
||||
},
|
||||
}),
|
||||
);
|
||||
let pool = self.worker_pool.lock().expect("worker pool lock poisoned");
|
||||
pool.submit(task)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn pump_one_transport_result(&self) -> Result<(), String> {
|
||||
let envelope = {
|
||||
let pool = self.worker_pool.lock().expect("worker pool lock poisoned");
|
||||
pool.recv_result_timeout(Duration::from_millis(50))?
|
||||
};
|
||||
let Some(envelope) = envelope else {
|
||||
return Ok(());
|
||||
};
|
||||
crate::progress_log::emit(
|
||||
"phase1_repo_task_result",
|
||||
serde_json::json!({
|
||||
"repo_key_rsync_base_uri": envelope.repo_identity.rsync_base_uri,
|
||||
"repo_key_notification_uri": envelope.repo_identity.notification_uri,
|
||||
"timing_ms": envelope.timing_ms,
|
||||
"transport_mode": match envelope.mode {
|
||||
RepoTransportMode::Rrdp => "rrdp",
|
||||
RepoTransportMode::Rsync => "rsync",
|
||||
},
|
||||
"result": match &envelope.result {
|
||||
RepoTransportResultKind::Success { .. } => "success",
|
||||
RepoTransportResultKind::Failed { .. } => "failed",
|
||||
},
|
||||
}),
|
||||
);
|
||||
let finished_at = time::OffsetDateTime::now_utc();
|
||||
let completion = {
|
||||
let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
|
||||
coordinator.complete_transport_result(envelope, finished_at)?
|
||||
};
|
||||
if !completion.follow_up_tasks.is_empty() {
|
||||
let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
|
||||
for task in completion.follow_up_tasks {
|
||||
crate::progress_log::emit(
|
||||
"phase1_repo_task_enqueued",
|
||||
serde_json::json!({
|
||||
"manifest_rsync_uri": serde_json::Value::Null,
|
||||
"publication_point_rsync_uri": task.requesters.first().map(|r| r.publication_point_rsync_uri.clone()),
|
||||
"repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri,
|
||||
"repo_key_notification_uri": task.repo_identity.notification_uri,
|
||||
"priority": task.priority,
|
||||
"transport_mode": "rsync",
|
||||
}),
|
||||
);
|
||||
coordinator.push_transport_task(task);
|
||||
}
|
||||
}
|
||||
self.drain_pending_transport_tasks()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn runtime_state_for_identity(&self, identity: &RepoIdentity) -> Option<RepoRuntimeState> {
|
||||
let coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
|
||||
coordinator.runtime_record(identity).map(|record| record.state)
|
||||
}
|
||||
|
||||
fn resolved_outcome_for_identity(
|
||||
&self,
|
||||
identity: &RepoIdentity,
|
||||
) -> Option<RepoSyncRuntimeOutcome> {
|
||||
let coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
|
||||
let record = coordinator.runtime_record(identity)?;
|
||||
match record.state {
|
||||
RepoRuntimeState::RrdpOk | RepoRuntimeState::RsyncOk => record
|
||||
.last_success
|
||||
.as_ref()
|
||||
.map(|result| outcome_from_transport_result(result, record.state)),
|
||||
RepoRuntimeState::FailedTerminal => record
|
||||
.terminal_failure
|
||||
.as_ref()
|
||||
.map(|result| outcome_from_transport_result(result, record.state)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<E: RepoTransportExecutor> RepoSyncRuntime for Phase1RepoSyncRuntime<E> {
|
||||
fn sync_publication_point_repo(
|
||||
&self,
|
||||
ca: &CaInstanceHandle,
|
||||
) -> Result<RepoSyncRuntimeOutcome, String> {
|
||||
if let Some(done) = self.ensure_transport_requested(ca, 0)? {
|
||||
return Ok(done);
|
||||
}
|
||||
let identity = Self::build_identity(ca);
|
||||
loop {
|
||||
if let Some(done) = self.resolved_outcome_for_identity(&identity) {
|
||||
return Ok(done);
|
||||
}
|
||||
self.pump_one_transport_result()?;
|
||||
}
|
||||
}
|
||||
|
||||
fn prefetch_discovered_children(
|
||||
&self,
|
||||
children: &[DiscoveredChildCaInstance],
|
||||
) -> Result<(), String> {
|
||||
for child in children {
|
||||
let _ = self.ensure_transport_requested(&child.handle, 1)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn outcome_from_transport_result(
|
||||
envelope: &RepoTransportResultEnvelope,
|
||||
state: RepoRuntimeState,
|
||||
) -> RepoSyncRuntimeOutcome {
|
||||
match (&envelope.result, state) {
|
||||
(RepoTransportResultKind::Success { source, warnings }, RepoRuntimeState::RrdpOk) => {
|
||||
RepoSyncRuntimeOutcome {
|
||||
repo_sync_ok: true,
|
||||
repo_sync_err: None,
|
||||
repo_sync_source: Some(source.clone()),
|
||||
repo_sync_phase: Some("rrdp_ok".to_string()),
|
||||
repo_sync_duration_ms: envelope.timing_ms,
|
||||
warnings: warnings.clone(),
|
||||
}
|
||||
}
|
||||
(RepoTransportResultKind::Success { source, warnings }, RepoRuntimeState::RsyncOk) => {
|
||||
RepoSyncRuntimeOutcome {
|
||||
repo_sync_ok: true,
|
||||
repo_sync_err: None,
|
||||
repo_sync_source: Some(source.clone()),
|
||||
repo_sync_phase: Some(if envelope.repo_identity.notification_uri.is_some() {
|
||||
"rrdp_failed_rsync_ok".to_string()
|
||||
} else {
|
||||
"rsync_only_ok".to_string()
|
||||
}),
|
||||
repo_sync_duration_ms: envelope.timing_ms,
|
||||
warnings: warnings.clone(),
|
||||
}
|
||||
}
|
||||
(RepoTransportResultKind::Failed { detail, warnings }, RepoRuntimeState::FailedTerminal) => {
|
||||
RepoSyncRuntimeOutcome {
|
||||
repo_sync_ok: false,
|
||||
repo_sync_err: Some(detail.clone()),
|
||||
repo_sync_source: None,
|
||||
repo_sync_phase: Some(if envelope.repo_identity.notification_uri.is_some() {
|
||||
"rrdp_failed_rsync_failed".to_string()
|
||||
} else {
|
||||
"rsync_failed".to_string()
|
||||
}),
|
||||
repo_sync_duration_ms: envelope.timing_ms,
|
||||
warnings: warnings.clone(),
|
||||
}
|
||||
}
|
||||
_ => RepoSyncRuntimeOutcome {
|
||||
repo_sync_ok: false,
|
||||
repo_sync_err: Some("repo runtime state unresolved".to_string()),
|
||||
repo_sync_source: None,
|
||||
repo_sync_phase: Some("repo_runtime_unresolved".to_string()),
|
||||
repo_sync_duration_ms: envelope.timing_ms,
|
||||
warnings: Vec::new(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::policy::SyncPreference;
|
||||
use crate::parallel::config::ParallelPhase1Config;
|
||||
use crate::parallel::repo_runtime::{Phase1RepoSyncRuntime, RepoSyncRuntime};
|
||||
use crate::parallel::repo_worker::{
|
||||
RepoTransportExecutor, RepoTransportWorkerPool, RepoWorkerPoolConfig,
|
||||
};
|
||||
use crate::parallel::run_coordinator::GlobalRunCoordinator;
|
||||
use crate::parallel::types::{
|
||||
RepoTransportMode, RepoTransportResultEnvelope, RepoTransportResultKind,
|
||||
RepoTransportTask, TalInputSpec,
|
||||
};
|
||||
use crate::report::Warning;
|
||||
use crate::validation::tree::{CaInstanceHandle, DiscoveredChildCaInstance};
|
||||
|
||||
fn sample_ca(manifest: &str) -> CaInstanceHandle {
|
||||
CaInstanceHandle {
|
||||
depth: 0,
|
||||
tal_id: "arin".to_string(),
|
||||
parent_manifest_rsync_uri: None,
|
||||
ca_certificate_der: vec![1, 2, 3],
|
||||
ca_certificate_rsync_uri: None,
|
||||
effective_ip_resources: None,
|
||||
effective_as_resources: None,
|
||||
rsync_base_uri: "rsync://example.test/repo/".to_string(),
|
||||
manifest_rsync_uri: manifest.to_string(),
|
||||
publication_point_rsync_uri: "rsync://example.test/repo/".to_string(),
|
||||
rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
struct SuccessTransportExecutor;
|
||||
|
||||
impl RepoTransportExecutor for SuccessTransportExecutor {
|
||||
fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope {
|
||||
RepoTransportResultEnvelope {
|
||||
dedup_key: task.dedup_key,
|
||||
repo_identity: task.repo_identity,
|
||||
mode: task.mode,
|
||||
tal_id: task.tal_id,
|
||||
rir_id: task.rir_id,
|
||||
timing_ms: 7,
|
||||
result: RepoTransportResultKind::Success {
|
||||
source: match task.mode {
|
||||
RepoTransportMode::Rrdp => "rrdp".to_string(),
|
||||
RepoTransportMode::Rsync => "rsync".to_string(),
|
||||
},
|
||||
warnings: vec![Warning::new("transport ok")],
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct FailRrdpThenSucceedRsyncExecutor {
|
||||
rrdp_count: Arc<AtomicUsize>,
|
||||
rsync_count: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl RepoTransportExecutor for FailRrdpThenSucceedRsyncExecutor {
|
||||
fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope {
|
||||
match task.mode {
|
||||
RepoTransportMode::Rrdp => {
|
||||
self.rrdp_count.fetch_add(1, Ordering::SeqCst);
|
||||
RepoTransportResultEnvelope {
|
||||
dedup_key: task.dedup_key,
|
||||
repo_identity: task.repo_identity,
|
||||
mode: RepoTransportMode::Rrdp,
|
||||
tal_id: task.tal_id,
|
||||
rir_id: task.rir_id,
|
||||
timing_ms: 10,
|
||||
result: RepoTransportResultKind::Failed {
|
||||
detail: "rrdp failed".to_string(),
|
||||
warnings: vec![Warning::new("rrdp failed")],
|
||||
},
|
||||
}
|
||||
}
|
||||
RepoTransportMode::Rsync => {
|
||||
self.rsync_count.fetch_add(1, Ordering::SeqCst);
|
||||
RepoTransportResultEnvelope {
|
||||
dedup_key: task.dedup_key,
|
||||
repo_identity: task.repo_identity,
|
||||
mode: RepoTransportMode::Rsync,
|
||||
tal_id: task.tal_id,
|
||||
rir_id: task.rir_id,
|
||||
timing_ms: 12,
|
||||
result: RepoTransportResultKind::Success {
|
||||
source: "rsync".to_string(),
|
||||
warnings: vec![Warning::new("rsync ok")],
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn phase1_runtime_waits_for_rrdp_transport_and_returns_rrdp_outcome() {
|
||||
let coordinator = GlobalRunCoordinator::new(
|
||||
ParallelPhase1Config::default(),
|
||||
vec![TalInputSpec::from_url("https://example.test/arin.tal")],
|
||||
);
|
||||
let pool = RepoTransportWorkerPool::new(
|
||||
RepoWorkerPoolConfig { max_workers: 1 },
|
||||
SuccessTransportExecutor,
|
||||
)
|
||||
.expect("pool");
|
||||
let runtime = Phase1RepoSyncRuntime::new(
|
||||
coordinator,
|
||||
pool,
|
||||
Arc::new(|base: &str| base.to_string()),
|
||||
SyncPreference::RrdpThenRsync,
|
||||
);
|
||||
|
||||
let outcome = runtime
|
||||
.sync_publication_point_repo(&sample_ca("rsync://example.test/repo/root.mft"))
|
||||
.expect("sync repo");
|
||||
assert!(outcome.repo_sync_ok);
|
||||
assert_eq!(outcome.repo_sync_source.as_deref(), Some("rrdp"));
|
||||
assert_eq!(outcome.repo_sync_phase.as_deref(), Some("rrdp_ok"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn phase1_runtime_transitions_rrdp_failure_to_rsync_success() {
|
||||
let rrdp_count = Arc::new(AtomicUsize::new(0));
|
||||
let rsync_count = Arc::new(AtomicUsize::new(0));
|
||||
let coordinator = GlobalRunCoordinator::new(
|
||||
ParallelPhase1Config::default(),
|
||||
vec![TalInputSpec::from_url("https://example.test/arin.tal")],
|
||||
);
|
||||
let pool = RepoTransportWorkerPool::new(
|
||||
RepoWorkerPoolConfig { max_workers: 1 },
|
||||
FailRrdpThenSucceedRsyncExecutor {
|
||||
rrdp_count: Arc::clone(&rrdp_count),
|
||||
rsync_count: Arc::clone(&rsync_count),
|
||||
},
|
||||
)
|
||||
.expect("pool");
|
||||
let runtime = Phase1RepoSyncRuntime::new(
|
||||
coordinator,
|
||||
pool,
|
||||
Arc::new(|_base: &str| "rsync://example.test/module/".to_string()),
|
||||
SyncPreference::RrdpThenRsync,
|
||||
);
|
||||
|
||||
let outcome = runtime
|
||||
.sync_publication_point_repo(&sample_ca("rsync://example.test/repo/root.mft"))
|
||||
.expect("sync repo");
|
||||
assert!(outcome.repo_sync_ok);
|
||||
assert_eq!(outcome.repo_sync_source.as_deref(), Some("rsync"));
|
||||
assert_eq!(outcome.repo_sync_phase.as_deref(), Some("rrdp_failed_rsync_ok"));
|
||||
assert_eq!(rrdp_count.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(rsync_count.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn phase1_runtime_prefetch_submits_transport_task_before_consumption() {
|
||||
let rrdp_count = Arc::new(AtomicUsize::new(0));
|
||||
let rsync_count = Arc::new(AtomicUsize::new(0));
|
||||
let coordinator = GlobalRunCoordinator::new(
|
||||
ParallelPhase1Config::default(),
|
||||
vec![TalInputSpec::from_url("https://example.test/arin.tal")],
|
||||
);
|
||||
let pool = RepoTransportWorkerPool::new(
|
||||
RepoWorkerPoolConfig { max_workers: 1 },
|
||||
FailRrdpThenSucceedRsyncExecutor {
|
||||
rrdp_count: Arc::clone(&rrdp_count),
|
||||
rsync_count: Arc::clone(&rsync_count),
|
||||
},
|
||||
)
|
||||
.expect("pool");
|
||||
let runtime = Arc::new(Phase1RepoSyncRuntime::new(
|
||||
coordinator,
|
||||
pool,
|
||||
Arc::new(|_base: &str| "rsync://example.test/module/".to_string()),
|
||||
SyncPreference::RrdpThenRsync,
|
||||
));
|
||||
|
||||
let child = DiscoveredChildCaInstance {
|
||||
handle: sample_ca("rsync://example.test/repo/child.mft"),
|
||||
discovered_from: crate::audit::DiscoveredFrom {
|
||||
parent_manifest_rsync_uri: "rsync://example.test/repo/root.mft".to_string(),
|
||||
child_ca_certificate_rsync_uri: "rsync://example.test/repo/child.cer".to_string(),
|
||||
child_ca_certificate_sha256_hex: "00".repeat(32),
|
||||
},
|
||||
};
|
||||
|
||||
runtime
|
||||
.prefetch_discovered_children(std::slice::from_ref(&child))
|
||||
.expect("prefetch");
|
||||
|
||||
let started = Instant::now();
|
||||
while rrdp_count.load(Ordering::SeqCst) == 0 && started.elapsed() < Duration::from_secs(1) {
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
assert_eq!(rrdp_count.load(Ordering::SeqCst), 1);
|
||||
|
||||
let outcome = runtime
|
||||
.sync_publication_point_repo(&child.handle)
|
||||
.expect("sync child repo");
|
||||
assert!(outcome.repo_sync_ok);
|
||||
assert_eq!(rsync_count.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
1209
src/parallel/repo_scheduler.rs
Normal file
1209
src/parallel/repo_scheduler.rs
Normal file
File diff suppressed because it is too large
Load Diff
1121
src/parallel/repo_worker.rs
Normal file
1121
src/parallel/repo_worker.rs
Normal file
File diff suppressed because it is too large
Load Diff
364
src/parallel/run_coordinator.rs
Normal file
364
src/parallel/run_coordinator.rs
Normal file
@@ -0,0 +1,364 @@
|
||||
use std::collections::VecDeque;
|
||||
|
||||
use crate::parallel::config::ParallelPhase1Config;
|
||||
use crate::parallel::repo_scheduler::{
|
||||
InFlightRepoTable, RepoCompletion, RepoRequestAction, TransportCompletion, TransportRequestAction,
|
||||
TransportStateTables,
|
||||
};
|
||||
use crate::parallel::stats::ParallelRunStats;
|
||||
use crate::parallel::types::{
|
||||
RepoIdentity, RepoKey, RepoRequester, RepoSyncResultEnvelope, RepoSyncTask, RepoTransportResultEnvelope,
|
||||
RepoTransportTask, TalInputSpec,
|
||||
};
|
||||
use crate::policy::SyncPreference;
|
||||
|
||||
pub struct GlobalRunCoordinator {
|
||||
pub config: ParallelPhase1Config,
|
||||
pub tal_inputs: Vec<TalInputSpec>,
|
||||
pub in_flight_repos: InFlightRepoTable,
|
||||
pub transport_tables: TransportStateTables,
|
||||
pub pending_repo_tasks: VecDeque<RepoSyncTask>,
|
||||
pub pending_transport_tasks: VecDeque<RepoTransportTask>,
|
||||
pub stats: ParallelRunStats,
|
||||
}
|
||||
|
||||
impl GlobalRunCoordinator {
|
||||
pub fn new(config: ParallelPhase1Config, tal_inputs: Vec<TalInputSpec>) -> Self {
|
||||
Self {
|
||||
config,
|
||||
tal_inputs,
|
||||
in_flight_repos: InFlightRepoTable::new(),
|
||||
transport_tables: TransportStateTables::new(),
|
||||
pending_repo_tasks: VecDeque::new(),
|
||||
pending_transport_tasks: VecDeque::new(),
|
||||
stats: ParallelRunStats::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn register_repo_request(
|
||||
&mut self,
|
||||
repo_key: RepoKey,
|
||||
requester: RepoRequester,
|
||||
validation_time: time::OffsetDateTime,
|
||||
sync_preference: SyncPreference,
|
||||
priority: u8,
|
||||
) -> RepoRequestAction {
|
||||
let action = self.in_flight_repos.register_request(
|
||||
repo_key,
|
||||
requester,
|
||||
validation_time,
|
||||
sync_preference,
|
||||
priority,
|
||||
);
|
||||
|
||||
match &action {
|
||||
RepoRequestAction::Enqueued(task) => {
|
||||
self.stats.repo_tasks_total += 1;
|
||||
self.pending_repo_tasks.push_back(task.clone());
|
||||
self.stats.repo_queue_depth = self.pending_repo_tasks.len();
|
||||
}
|
||||
RepoRequestAction::Reused(_) | RepoRequestAction::FailedReuse { .. } => {
|
||||
self.stats.repo_tasks_reused += 1;
|
||||
}
|
||||
RepoRequestAction::Waiting => {}
|
||||
}
|
||||
|
||||
action
|
||||
}
|
||||
|
||||
pub fn pop_next_repo_task(&mut self) -> Option<RepoSyncTask> {
|
||||
let next = self.pending_repo_tasks.pop_front();
|
||||
self.stats.repo_queue_depth = self.pending_repo_tasks.len();
|
||||
next
|
||||
}
|
||||
|
||||
pub fn mark_repo_running(
|
||||
&mut self,
|
||||
repo_key: &RepoKey,
|
||||
started_at: time::OffsetDateTime,
|
||||
) -> Result<(), String> {
|
||||
self.in_flight_repos.mark_running(repo_key, started_at)?;
|
||||
self.stats.repo_tasks_running += 1;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn complete_repo_success(
|
||||
&mut self,
|
||||
result: RepoSyncResultEnvelope,
|
||||
finished_at: time::OffsetDateTime,
|
||||
) -> Result<RepoCompletion, String> {
|
||||
let repo_key = result.repo_key.clone();
|
||||
let completion = self
|
||||
.in_flight_repos
|
||||
.complete_success(&repo_key, result, finished_at)?;
|
||||
self.stats.repo_tasks_running = self.stats.repo_tasks_running.saturating_sub(1);
|
||||
Ok(completion)
|
||||
}
|
||||
|
||||
pub fn complete_repo_failure(
|
||||
&mut self,
|
||||
result: RepoSyncResultEnvelope,
|
||||
finished_at: time::OffsetDateTime,
|
||||
) -> Result<RepoCompletion, String> {
|
||||
let repo_key = result.repo_key.clone();
|
||||
let completion = self
|
||||
.in_flight_repos
|
||||
.complete_failure(&repo_key, result, finished_at)?;
|
||||
self.stats.repo_tasks_running = self.stats.repo_tasks_running.saturating_sub(1);
|
||||
self.stats.repo_tasks_failed += 1;
|
||||
Ok(completion)
|
||||
}
|
||||
|
||||
pub fn register_transport_request(
|
||||
&mut self,
|
||||
identity: RepoIdentity,
|
||||
requester: RepoRequester,
|
||||
validation_time: time::OffsetDateTime,
|
||||
priority: u8,
|
||||
rsync_scope_uri: String,
|
||||
sync_preference: SyncPreference,
|
||||
) -> TransportRequestAction {
|
||||
let action = self.transport_tables.register_transport_request(
|
||||
identity,
|
||||
requester,
|
||||
validation_time,
|
||||
priority,
|
||||
rsync_scope_uri,
|
||||
sync_preference,
|
||||
);
|
||||
match &action {
|
||||
TransportRequestAction::Enqueue(task) => {
|
||||
self.stats.repo_tasks_total += 1;
|
||||
self.pending_transport_tasks.push_back(task.clone());
|
||||
self.stats.repo_queue_depth = self.pending_transport_tasks.len();
|
||||
}
|
||||
TransportRequestAction::ReusedSuccess(_)
|
||||
| TransportRequestAction::ReusedTerminalFailure(_) => {
|
||||
self.stats.repo_tasks_reused += 1;
|
||||
}
|
||||
TransportRequestAction::Waiting { .. } => {}
|
||||
}
|
||||
action
|
||||
}
|
||||
|
||||
pub fn push_transport_task(&mut self, task: RepoTransportTask) {
|
||||
self.stats.repo_tasks_total += 1;
|
||||
self.pending_transport_tasks.push_back(task);
|
||||
self.stats.repo_queue_depth = self.pending_transport_tasks.len();
|
||||
}
|
||||
|
||||
pub fn pop_next_transport_task(&mut self) -> Option<RepoTransportTask> {
|
||||
let next = self.pending_transport_tasks.pop_front();
|
||||
self.stats.repo_queue_depth = self.pending_transport_tasks.len();
|
||||
next
|
||||
}
|
||||
|
||||
pub fn mark_transport_running(
|
||||
&mut self,
|
||||
dedup_key: &crate::parallel::types::RepoDedupKey,
|
||||
started_at: time::OffsetDateTime,
|
||||
) -> Result<(), String> {
|
||||
self.transport_tables.mark_transport_running(dedup_key, started_at)?;
|
||||
self.stats.repo_tasks_running += 1;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn complete_transport_result(
|
||||
&mut self,
|
||||
result: RepoTransportResultEnvelope,
|
||||
finished_at: time::OffsetDateTime,
|
||||
) -> Result<TransportCompletion, String> {
|
||||
let completion = self.transport_tables.complete_transport_result(result.clone(), finished_at)?;
|
||||
self.stats.repo_tasks_running = self.stats.repo_tasks_running.saturating_sub(1);
|
||||
if matches!(result.result, crate::parallel::types::RepoTransportResultKind::Failed { .. })
|
||||
&& result.mode == crate::parallel::types::RepoTransportMode::Rsync
|
||||
{
|
||||
self.stats.repo_tasks_failed += 1;
|
||||
}
|
||||
Ok(completion)
|
||||
}
|
||||
|
||||
pub fn runtime_record(
|
||||
&self,
|
||||
identity: &RepoIdentity,
|
||||
) -> Option<&crate::parallel::repo_scheduler::RepoRuntimeRecord> {
|
||||
self.transport_tables.runtime_record(identity)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::parallel::config::ParallelPhase1Config;
|
||||
use crate::parallel::repo_scheduler::RepoRequestAction;
|
||||
use crate::parallel::run_coordinator::GlobalRunCoordinator;
|
||||
use crate::parallel::types::{
|
||||
RepoKey, RepoRequester, RepoSyncResultEnvelope, RepoSyncResultKind, RepoSyncResultRef,
|
||||
TalInputSpec,
|
||||
};
|
||||
use crate::policy::SyncPreference;
|
||||
|
||||
fn requester(tal_id: &str, rir_id: &str, manifest: &str) -> RepoRequester {
|
||||
RepoRequester {
|
||||
tal_id: tal_id.to_string(),
|
||||
rir_id: rir_id.to_string(),
|
||||
parent_node_id: None,
|
||||
ca_instance_handle_id: format!("{tal_id}:{manifest}"),
|
||||
publication_point_rsync_uri: "rsync://example.test/repo/".to_string(),
|
||||
manifest_rsync_uri: manifest.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coordinator_holds_tal_inputs_and_default_stats() {
|
||||
let coordinator = GlobalRunCoordinator::new(
|
||||
ParallelPhase1Config::default(),
|
||||
vec![TalInputSpec::from_url("https://example.test/arin.tal")],
|
||||
);
|
||||
assert_eq!(coordinator.tal_inputs.len(), 1);
|
||||
assert_eq!(coordinator.tal_inputs[0].tal_id, "arin");
|
||||
assert_eq!(coordinator.stats.repo_tasks_total, 0);
|
||||
assert!(coordinator.in_flight_repos.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coordinator_enqueues_new_repo_task_once() {
|
||||
let mut coordinator = GlobalRunCoordinator::new(
|
||||
ParallelPhase1Config::default(),
|
||||
vec![TalInputSpec::from_url("https://example.test/arin.tal")],
|
||||
);
|
||||
let key = RepoKey::new("rsync://example.test/repo/", None);
|
||||
let action = coordinator.register_repo_request(
|
||||
key.clone(),
|
||||
requester("arin", "arin", "rsync://example.test/repo/root.mft"),
|
||||
time::OffsetDateTime::UNIX_EPOCH,
|
||||
SyncPreference::RrdpThenRsync,
|
||||
7,
|
||||
);
|
||||
let task = match action {
|
||||
RepoRequestAction::Enqueued(task) => task,
|
||||
other => panic!("expected enqueue, got {other:?}"),
|
||||
};
|
||||
assert_eq!(task.priority, 7);
|
||||
assert_eq!(coordinator.stats.repo_tasks_total, 1);
|
||||
assert_eq!(coordinator.stats.repo_queue_depth, 1);
|
||||
|
||||
let popped = coordinator.pop_next_repo_task().expect("task queued");
|
||||
assert_eq!(popped.repo_key, key);
|
||||
assert_eq!(coordinator.stats.repo_queue_depth, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coordinator_merges_waiting_requesters_and_reuses_success() {
|
||||
let mut coordinator = GlobalRunCoordinator::new(
|
||||
ParallelPhase1Config::default(),
|
||||
vec![
|
||||
TalInputSpec::from_url("https://example.test/arin.tal"),
|
||||
TalInputSpec::from_url("https://example.test/apnic.tal"),
|
||||
],
|
||||
);
|
||||
let key = RepoKey::new("rsync://shared.example/repo/", None);
|
||||
let _ = coordinator.register_repo_request(
|
||||
key.clone(),
|
||||
requester("arin", "arin", "rsync://shared.example/repo/root.mft"),
|
||||
time::OffsetDateTime::UNIX_EPOCH,
|
||||
SyncPreference::RrdpThenRsync,
|
||||
0,
|
||||
);
|
||||
let _ = coordinator.pop_next_repo_task();
|
||||
coordinator
|
||||
.mark_repo_running(&key, time::OffsetDateTime::UNIX_EPOCH)
|
||||
.expect("running");
|
||||
|
||||
let wait_action = coordinator.register_repo_request(
|
||||
key.clone(),
|
||||
requester("apnic", "apnic", "rsync://shared.example/repo/child.mft"),
|
||||
time::OffsetDateTime::UNIX_EPOCH,
|
||||
SyncPreference::RrdpThenRsync,
|
||||
0,
|
||||
);
|
||||
assert_eq!(wait_action, RepoRequestAction::Waiting);
|
||||
|
||||
let completion = coordinator
|
||||
.complete_repo_success(
|
||||
RepoSyncResultEnvelope {
|
||||
repo_key: key.clone(),
|
||||
tal_id: "arin".to_string(),
|
||||
rir_id: "arin".to_string(),
|
||||
result: RepoSyncResultKind::Success(RepoSyncResultRef {
|
||||
repo_key: key.clone(),
|
||||
source: "rrdp".to_string(),
|
||||
}),
|
||||
phase: Some("rrdp_ok".to_string()),
|
||||
timing_ms: 12,
|
||||
warnings: Vec::new(),
|
||||
},
|
||||
time::OffsetDateTime::UNIX_EPOCH,
|
||||
)
|
||||
.expect("success");
|
||||
assert_eq!(completion.released_requesters.len(), 1);
|
||||
assert_eq!(coordinator.stats.repo_tasks_running, 0);
|
||||
|
||||
let reuse_action = coordinator.register_repo_request(
|
||||
key,
|
||||
requester("ripe", "ripe", "rsync://shared.example/repo/again.mft"),
|
||||
time::OffsetDateTime::UNIX_EPOCH,
|
||||
SyncPreference::RrdpThenRsync,
|
||||
0,
|
||||
);
|
||||
assert!(matches!(reuse_action, RepoRequestAction::Reused(_)));
|
||||
assert_eq!(coordinator.stats.repo_tasks_reused, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coordinator_reuses_failures_without_enqueuing_again() {
|
||||
let mut coordinator = GlobalRunCoordinator::new(
|
||||
ParallelPhase1Config::default(),
|
||||
vec![TalInputSpec::from_url("https://example.test/arin.tal")],
|
||||
);
|
||||
let key = RepoKey::new("rsync://shared.example/repo/", None);
|
||||
let _ = coordinator.register_repo_request(
|
||||
key.clone(),
|
||||
requester("arin", "arin", "rsync://shared.example/repo/root.mft"),
|
||||
time::OffsetDateTime::UNIX_EPOCH,
|
||||
SyncPreference::RrdpThenRsync,
|
||||
0,
|
||||
);
|
||||
let _ = coordinator.pop_next_repo_task();
|
||||
coordinator
|
||||
.mark_repo_running(&key, time::OffsetDateTime::UNIX_EPOCH)
|
||||
.expect("running");
|
||||
coordinator
|
||||
.complete_repo_failure(
|
||||
RepoSyncResultEnvelope {
|
||||
repo_key: key.clone(),
|
||||
tal_id: "arin".to_string(),
|
||||
rir_id: "arin".to_string(),
|
||||
result: RepoSyncResultKind::Failed {
|
||||
detail: "timeout".to_string(),
|
||||
},
|
||||
phase: Some("rrdp_failed_rsync_failed".to_string()),
|
||||
timing_ms: 12,
|
||||
warnings: Vec::new(),
|
||||
},
|
||||
time::OffsetDateTime::UNIX_EPOCH,
|
||||
)
|
||||
.expect("failure");
|
||||
assert_eq!(coordinator.stats.repo_tasks_failed, 1);
|
||||
|
||||
let action = coordinator.register_repo_request(
|
||||
key,
|
||||
requester("ripe", "ripe", "rsync://shared.example/repo/child.mft"),
|
||||
time::OffsetDateTime::UNIX_EPOCH,
|
||||
SyncPreference::RrdpThenRsync,
|
||||
0,
|
||||
);
|
||||
assert_eq!(
|
||||
action,
|
||||
RepoRequestAction::FailedReuse {
|
||||
detail: "timeout".to_string()
|
||||
}
|
||||
);
|
||||
assert_eq!(coordinator.stats.repo_tasks_total, 1);
|
||||
assert_eq!(coordinator.stats.repo_tasks_reused, 1);
|
||||
}
|
||||
}
|
||||
25
src/parallel/stats.rs
Normal file
25
src/parallel/stats.rs
Normal file
@@ -0,0 +1,25 @@
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct ParallelRunStats {
|
||||
pub repo_tasks_total: usize,
|
||||
pub repo_tasks_reused: usize,
|
||||
pub repo_tasks_running: usize,
|
||||
pub repo_tasks_failed: usize,
|
||||
pub inflight_snapshot_bytes: usize,
|
||||
pub repo_queue_depth: usize,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::ParallelRunStats;
|
||||
|
||||
#[test]
|
||||
fn parallel_run_stats_default_to_zero() {
|
||||
let stats = ParallelRunStats::default();
|
||||
assert_eq!(stats.repo_tasks_total, 0);
|
||||
assert_eq!(stats.repo_tasks_reused, 0);
|
||||
assert_eq!(stats.repo_tasks_running, 0);
|
||||
assert_eq!(stats.repo_tasks_failed, 0);
|
||||
assert_eq!(stats.inflight_snapshot_bytes, 0);
|
||||
assert_eq!(stats.repo_queue_depth, 0);
|
||||
}
|
||||
}
|
||||
472
src/parallel/types.rs
Normal file
472
src/parallel/types.rs
Normal file
@@ -0,0 +1,472 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::policy::SyncPreference;
|
||||
use crate::report::Warning;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum TalSource {
|
||||
Url(String),
|
||||
DerBytes { tal_url: String, ta_der: Vec<u8> },
|
||||
FilePath(PathBuf),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TalInputSpec {
|
||||
pub tal_id: String,
|
||||
pub rir_id: String,
|
||||
pub source: TalSource,
|
||||
}
|
||||
|
||||
impl TalInputSpec {
|
||||
pub fn from_url(url: impl Into<String>) -> Self {
|
||||
let url = url.into();
|
||||
let tal_id = derive_tal_id_from_url_like(&url);
|
||||
Self {
|
||||
rir_id: tal_id.clone(),
|
||||
tal_id,
|
||||
source: TalSource::Url(url),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_file_path(path: impl Into<PathBuf>) -> Self {
|
||||
let path = path.into();
|
||||
let tal_id = derive_tal_id_from_path(&path);
|
||||
Self {
|
||||
rir_id: tal_id.clone(),
|
||||
tal_id,
|
||||
source: TalSource::FilePath(path),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_ta_der(tal_url: impl Into<String>, ta_der: Vec<u8>) -> Self {
|
||||
let tal_url = tal_url.into();
|
||||
let tal_id = derive_tal_id_from_url_like(&tal_url);
|
||||
Self {
|
||||
rir_id: tal_id.clone(),
|
||||
tal_id,
|
||||
source: TalSource::DerBytes { tal_url, ta_der },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct RepoIdentity {
|
||||
pub notification_uri: Option<String>,
|
||||
pub rsync_base_uri: String,
|
||||
}
|
||||
|
||||
impl RepoIdentity {
|
||||
pub fn new(notification_uri: Option<String>, rsync_base_uri: impl Into<String>) -> Self {
|
||||
Self {
|
||||
notification_uri,
|
||||
rsync_base_uri: rsync_base_uri.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||
pub enum RepoDedupKey {
|
||||
RrdpNotify { notification_uri: String },
|
||||
RsyncScope { rsync_scope_uri: String },
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum RepoTransportMode {
|
||||
Rrdp,
|
||||
Rsync,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RepoTransportTask {
|
||||
pub dedup_key: RepoDedupKey,
|
||||
pub repo_identity: RepoIdentity,
|
||||
pub mode: RepoTransportMode,
|
||||
pub tal_id: String,
|
||||
pub rir_id: String,
|
||||
pub validation_time: time::OffsetDateTime,
|
||||
pub priority: u8,
|
||||
pub requesters: Vec<RepoRequester>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum RepoTransportResultKind {
|
||||
Success {
|
||||
source: String,
|
||||
warnings: Vec<Warning>,
|
||||
},
|
||||
Failed {
|
||||
detail: String,
|
||||
warnings: Vec<Warning>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RepoTransportResultEnvelope {
|
||||
pub dedup_key: RepoDedupKey,
|
||||
pub repo_identity: RepoIdentity,
|
||||
pub mode: RepoTransportMode,
|
||||
pub tal_id: String,
|
||||
pub rir_id: String,
|
||||
pub timing_ms: u64,
|
||||
pub result: RepoTransportResultKind,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum RepoRuntimeState {
|
||||
Init,
|
||||
WaitingRrdp,
|
||||
RrdpOk,
|
||||
RrdpFailedPendingRsync,
|
||||
WaitingRsync,
|
||||
RsyncOk,
|
||||
FailedTerminal,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct RepoKey {
|
||||
pub rsync_base_uri: String,
|
||||
pub notification_uri: Option<String>,
|
||||
}
|
||||
|
||||
impl RepoKey {
|
||||
pub fn new(rsync_base_uri: impl Into<String>, notification_uri: Option<String>) -> Self {
|
||||
Self {
|
||||
rsync_base_uri: rsync_base_uri.into(),
|
||||
notification_uri,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_identity(&self) -> RepoIdentity {
|
||||
RepoIdentity {
|
||||
notification_uri: self.notification_uri.clone(),
|
||||
rsync_base_uri: self.rsync_base_uri.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RepoRequester {
|
||||
pub tal_id: String,
|
||||
pub rir_id: String,
|
||||
pub parent_node_id: Option<u64>,
|
||||
pub ca_instance_handle_id: String,
|
||||
pub publication_point_rsync_uri: String,
|
||||
pub manifest_rsync_uri: String,
|
||||
}
|
||||
|
||||
impl RepoRequester {
|
||||
pub fn with_tal_rir(
|
||||
tal_id: impl Into<String>,
|
||||
rir_id: impl Into<String>,
|
||||
manifest_rsync_uri: impl Into<String>,
|
||||
publication_point_rsync_uri: impl Into<String>,
|
||||
ca_instance_handle_id: impl Into<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
tal_id: tal_id.into(),
|
||||
rir_id: rir_id.into(),
|
||||
parent_node_id: None,
|
||||
ca_instance_handle_id: ca_instance_handle_id.into(),
|
||||
publication_point_rsync_uri: publication_point_rsync_uri.into(),
|
||||
manifest_rsync_uri: manifest_rsync_uri.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RepoSyncTask {
|
||||
pub repo_key: RepoKey,
|
||||
pub validation_time: time::OffsetDateTime,
|
||||
pub sync_preference: SyncPreference,
|
||||
pub tal_id: String,
|
||||
pub rir_id: String,
|
||||
pub priority: u8,
|
||||
pub requesters: Vec<RepoRequester>,
|
||||
}
|
||||
|
||||
impl RepoSyncTask {
|
||||
pub fn as_transport_task(
|
||||
&self,
|
||||
dedup_key: RepoDedupKey,
|
||||
mode: RepoTransportMode,
|
||||
) -> RepoTransportTask {
|
||||
RepoTransportTask {
|
||||
dedup_key,
|
||||
repo_identity: self.repo_key.as_identity(),
|
||||
mode,
|
||||
tal_id: self.tal_id.clone(),
|
||||
rir_id: self.rir_id.clone(),
|
||||
validation_time: self.validation_time,
|
||||
priority: self.priority,
|
||||
requesters: self.requesters.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum RepoTaskState {
|
||||
Pending,
|
||||
Running,
|
||||
Succeeded,
|
||||
Failed,
|
||||
Reused,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RepoSyncResultRef {
|
||||
pub repo_key: RepoKey,
|
||||
pub source: String,
|
||||
}
|
||||
|
||||
impl RepoSyncResultRef {
|
||||
pub fn as_identity(&self) -> RepoIdentity {
|
||||
self.repo_key.as_identity()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct InFlightRepoEntry {
|
||||
pub state: RepoTaskState,
|
||||
pub task_ref: Option<RepoSyncTask>,
|
||||
pub waiting_requesters: Vec<RepoRequester>,
|
||||
pub result_ref: Option<RepoSyncResultRef>,
|
||||
pub last_result: Option<RepoSyncResultEnvelope>,
|
||||
pub last_error: Option<String>,
|
||||
pub started_at: Option<time::OffsetDateTime>,
|
||||
pub finished_at: Option<time::OffsetDateTime>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RepoSyncResultEnvelope {
|
||||
pub repo_key: RepoKey,
|
||||
pub tal_id: String,
|
||||
pub rir_id: String,
|
||||
pub result: RepoSyncResultKind,
|
||||
pub phase: Option<String>,
|
||||
pub timing_ms: u64,
|
||||
pub warnings: Vec<Warning>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum RepoSyncResultKind {
|
||||
Success(RepoSyncResultRef),
|
||||
Failed { detail: String },
|
||||
Reused(RepoSyncResultRef),
|
||||
}
|
||||
|
||||
fn derive_tal_id_from_url_like(s: &str) -> String {
|
||||
if let Ok(url) = url::Url::parse(s) {
|
||||
if let Some(last) = url
|
||||
.path_segments()
|
||||
.and_then(|segments| segments.filter(|seg| !seg.is_empty()).next_back())
|
||||
{
|
||||
let stem = last.rsplit_once('.').map(|(stem, _)| stem).unwrap_or(last);
|
||||
let trimmed = stem.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return trimmed.to_string();
|
||||
}
|
||||
}
|
||||
if let Some(host) = url.host_str() {
|
||||
return host.to_string();
|
||||
}
|
||||
}
|
||||
"unknown-tal".to_string()
|
||||
}
|
||||
|
||||
fn derive_tal_id_from_path(path: &Path) -> String {
|
||||
path.file_stem()
|
||||
.and_then(|stem| stem.to_str())
|
||||
.map(|s| s.trim())
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or("unknown-tal")
|
||||
.to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::path::Path;
|
||||
|
||||
use crate::policy::SyncPreference;
|
||||
use crate::report::Warning;
|
||||
|
||||
use super::{
|
||||
derive_tal_id_from_path, derive_tal_id_from_url_like, RepoDedupKey, RepoIdentity,
|
||||
RepoKey, RepoRequester, RepoRuntimeState, RepoSyncTask, RepoTaskState,
|
||||
RepoTransportMode, RepoTransportResultEnvelope, RepoTransportResultKind, TalInputSpec,
|
||||
TalSource,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn tal_input_spec_from_url_derives_tal_and_rir_ids() {
|
||||
let spec = TalInputSpec::from_url("https://example.test/tals/apnic.tal");
|
||||
assert_eq!(spec.tal_id, "apnic");
|
||||
assert_eq!(spec.rir_id, "apnic");
|
||||
assert_eq!(
|
||||
spec.source,
|
||||
TalSource::Url("https://example.test/tals/apnic.tal".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tal_input_spec_from_file_path_derives_file_stem() {
|
||||
let spec = TalInputSpec::from_file_path("local/arin.tal");
|
||||
assert_eq!(spec.tal_id, "arin");
|
||||
assert_eq!(spec.rir_id, "arin");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tal_input_spec_from_ta_der_preserves_payload() {
|
||||
let spec = TalInputSpec::from_ta_der("https://example.test/ripe.tal", vec![1, 2, 3]);
|
||||
assert_eq!(spec.tal_id, "ripe");
|
||||
assert_eq!(spec.rir_id, "ripe");
|
||||
assert_eq!(
|
||||
spec.source,
|
||||
TalSource::DerBytes {
|
||||
tal_url: "https://example.test/ripe.tal".to_string(),
|
||||
ta_der: vec![1, 2, 3],
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repo_key_equality_uses_rsync_base_and_notification() {
|
||||
let a = RepoKey::new(
|
||||
"rsync://example.test/repo/",
|
||||
Some("https://example.test/notify.xml".to_string()),
|
||||
);
|
||||
let b = RepoKey::new(
|
||||
"rsync://example.test/repo/",
|
||||
Some("https://example.test/notify.xml".to_string()),
|
||||
);
|
||||
let c = RepoKey::new("rsync://example.test/repo/", None);
|
||||
assert_eq!(a, b);
|
||||
assert_ne!(a, c);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repo_task_state_variants_are_distinct() {
|
||||
assert_ne!(RepoTaskState::Pending, RepoTaskState::Running);
|
||||
assert_ne!(RepoTaskState::Succeeded, RepoTaskState::Failed);
|
||||
assert_ne!(RepoTaskState::Failed, RepoTaskState::Reused);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repo_identity_preserves_raw_inputs() {
|
||||
let ident = RepoIdentity::new(
|
||||
Some("https://example.test/notify.xml".to_string()),
|
||||
"rsync://example.test/repo/",
|
||||
);
|
||||
assert_eq!(
|
||||
ident.notification_uri.as_deref(),
|
||||
Some("https://example.test/notify.xml")
|
||||
);
|
||||
assert_eq!(ident.rsync_base_uri, "rsync://example.test/repo/");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repo_key_can_be_viewed_as_repo_identity() {
|
||||
let key = RepoKey::new(
|
||||
"rsync://example.test/repo/",
|
||||
Some("https://example.test/notify.xml".to_string()),
|
||||
);
|
||||
let ident = key.as_identity();
|
||||
assert_eq!(ident.rsync_base_uri, "rsync://example.test/repo/");
|
||||
assert_eq!(
|
||||
ident.notification_uri.as_deref(),
|
||||
Some("https://example.test/notify.xml")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repo_sync_task_maps_to_rrdp_transport_task() {
|
||||
let task = RepoSyncTask {
|
||||
repo_key: RepoKey::new(
|
||||
"rsync://example.test/repo/",
|
||||
Some("https://example.test/notify.xml".to_string()),
|
||||
),
|
||||
validation_time: time::OffsetDateTime::UNIX_EPOCH,
|
||||
sync_preference: SyncPreference::RrdpThenRsync,
|
||||
tal_id: "apnic".to_string(),
|
||||
rir_id: "apnic".to_string(),
|
||||
priority: 1,
|
||||
requesters: vec![RepoRequester::with_tal_rir(
|
||||
"apnic",
|
||||
"apnic",
|
||||
"rsync://example.test/repo/root.mft",
|
||||
"rsync://example.test/repo/",
|
||||
"node:1",
|
||||
)],
|
||||
};
|
||||
let transport = task.as_transport_task(
|
||||
RepoDedupKey::RrdpNotify {
|
||||
notification_uri: "https://example.test/notify.xml".to_string(),
|
||||
},
|
||||
RepoTransportMode::Rrdp,
|
||||
);
|
||||
assert_eq!(transport.mode, RepoTransportMode::Rrdp);
|
||||
assert_eq!(transport.tal_id, "apnic");
|
||||
assert_eq!(transport.rir_id, "apnic");
|
||||
assert_eq!(transport.requesters.len(), 1);
|
||||
assert_eq!(
|
||||
transport.repo_identity.notification_uri.as_deref(),
|
||||
Some("https://example.test/notify.xml")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repo_transport_result_envelope_supports_success_and_failure_shapes() {
|
||||
let identity = RepoIdentity::new(None, "rsync://example.test/repo/");
|
||||
let ok = RepoTransportResultEnvelope {
|
||||
dedup_key: RepoDedupKey::RsyncScope {
|
||||
rsync_scope_uri: "rsync://example.test/module/".to_string(),
|
||||
},
|
||||
repo_identity: identity.clone(),
|
||||
mode: RepoTransportMode::Rsync,
|
||||
tal_id: "arin".to_string(),
|
||||
rir_id: "arin".to_string(),
|
||||
timing_ms: 12,
|
||||
result: RepoTransportResultKind::Success {
|
||||
source: "rsync".to_string(),
|
||||
warnings: vec![Warning::new("ok")],
|
||||
},
|
||||
};
|
||||
let fail = RepoTransportResultEnvelope {
|
||||
dedup_key: RepoDedupKey::RsyncScope {
|
||||
rsync_scope_uri: "rsync://example.test/module/".to_string(),
|
||||
},
|
||||
repo_identity: identity,
|
||||
mode: RepoTransportMode::Rsync,
|
||||
tal_id: "arin".to_string(),
|
||||
rir_id: "arin".to_string(),
|
||||
timing_ms: 30,
|
||||
result: RepoTransportResultKind::Failed {
|
||||
detail: "timeout".to_string(),
|
||||
warnings: vec![Warning::new("timeout")],
|
||||
},
|
||||
};
|
||||
assert!(matches!(ok.result, RepoTransportResultKind::Success { .. }));
|
||||
assert!(matches!(fail.result, RepoTransportResultKind::Failed { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repo_runtime_state_variants_are_distinct() {
|
||||
assert_ne!(RepoRuntimeState::Init, RepoRuntimeState::WaitingRrdp);
|
||||
assert_ne!(RepoRuntimeState::RrdpOk, RepoRuntimeState::RsyncOk);
|
||||
assert_ne!(
|
||||
RepoRuntimeState::RrdpFailedPendingRsync,
|
||||
RepoRuntimeState::FailedTerminal
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derive_tal_id_helpers_fall_back_safely() {
|
||||
assert_eq!(
|
||||
derive_tal_id_from_url_like("https://example.test/path/afrinic.tal"),
|
||||
"afrinic"
|
||||
);
|
||||
assert_eq!(
|
||||
derive_tal_id_from_path(Path::new("foo/lacnic.tal")),
|
||||
"lacnic"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user