549 lines
21 KiB
Rust
549 lines
21 KiB
Rust
// Top-level CLI execution pipeline.
|
|
|
|
pub(crate) fn run_config(args: RunConfig) -> Result<(), String> {
|
|
let mut policy = read_policy(args.policy_path.as_deref())?;
|
|
if let Some(strict_policy) = args.strict_policy {
|
|
policy.strict = strict_policy;
|
|
}
|
|
if let Some(resource_validation_mode) = args.resource_validation_mode {
|
|
policy.resource_validation_mode = resource_validation_mode;
|
|
}
|
|
if args.disable_rrdp {
|
|
policy.sync_preference = crate::validation::policy::SyncPreference::RsyncOnly;
|
|
}
|
|
policy.ta_constraints = args.ta_constraints.clone();
|
|
for warning in policy.ta_constraints.configuration_warnings() {
|
|
crate::logging::emit(crate::logging::Level::Warn, "constraint_warning", || serde_json::json!({"reason": warning}));
|
|
}
|
|
let validation_time = args
|
|
.validation_time
|
|
.unwrap_or_else(time::OffsetDateTime::now_utc);
|
|
let validation_time =
|
|
time::OffsetDateTime::from_unix_timestamp(validation_time.unix_timestamp())
|
|
.map_err(|error| format!("normalize validation time failed: {error}"))?;
|
|
let http_root_certificates_pem = args
|
|
.http_root_cert_paths
|
|
.iter()
|
|
.map(|path| {
|
|
std::fs::read(path)
|
|
.map_err(|e| format!("read HTTP root certificate failed: {}: {e}", path.display()))
|
|
})
|
|
.collect::<Result<Vec<_>, _>>()?;
|
|
|
|
let store = if args.raw_store_db.is_some() || args.repo_bytes_db.is_some() {
|
|
Arc::new(
|
|
RocksStore::open_with_external_stores(
|
|
&args.db_path,
|
|
args.raw_store_db.as_deref(),
|
|
args.repo_bytes_db.as_deref(),
|
|
)
|
|
.map_err(|e| e.to_string())?,
|
|
)
|
|
} else {
|
|
Arc::new(RocksStore::open(&args.db_path).map_err(|e| e.to_string())?)
|
|
};
|
|
let config = TreeRunConfig {
|
|
max_depth: Some(args.max_ca_depth),
|
|
max_instances: args.max_instances,
|
|
compact_audit: args.skip_report_build
|
|
&& args.report_json_path.is_none(),
|
|
build_ccr_accumulator: args.ccr_out_path.is_some(),
|
|
};
|
|
|
|
use time::format_description::well_known::Rfc3339;
|
|
let mut timing: Option<(std::path::PathBuf, TimingHandle)> = None;
|
|
if args.analyze {
|
|
let recorded_at_utc_rfc3339 = time::OffsetDateTime::now_utc()
|
|
.to_offset(time::UtcOffset::UTC)
|
|
.format(&Rfc3339)
|
|
.map_err(|e| format!("format recorded_at_utc failed: {e}"))?;
|
|
let validation_time_utc_rfc3339 = validation_time
|
|
.to_offset(time::UtcOffset::UTC)
|
|
.format(&Rfc3339)
|
|
.map_err(|e| format!("format validation_time failed: {e}"))?;
|
|
|
|
let ts_compact = {
|
|
let fmt = time::format_description::parse_borrowed::<2>(
|
|
"[year][month][day]T[hour][minute][second]Z",
|
|
)
|
|
.map_err(|e| format!("format description parse failed: {e}"))?;
|
|
time::OffsetDateTime::now_utc()
|
|
.format(&fmt)
|
|
.map_err(|e| format!("format timestamp failed: {e}"))?
|
|
};
|
|
|
|
let out_dir = args.analysis_out_path.clone().unwrap_or_else(|| {
|
|
std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
|
.join("target")
|
|
.join("live")
|
|
.join("analyze")
|
|
.join(ts_compact)
|
|
});
|
|
std::fs::create_dir_all(&out_dir)
|
|
.map_err(|e| format!("create analyze out dir failed: {}: {e}", out_dir.display()))?;
|
|
|
|
let handle = TimingHandle::new(TimingMeta {
|
|
recorded_at_utc_rfc3339,
|
|
validation_time_utc_rfc3339,
|
|
tal_url: None,
|
|
db_path: None,
|
|
});
|
|
handle.set_meta(TimingMetaUpdate {
|
|
tal_url: args.tal_url.as_deref(),
|
|
db_path: Some(args.db_path.to_string_lossy().as_ref()),
|
|
});
|
|
timing = Some((out_dir, handle));
|
|
}
|
|
|
|
let total_started = std::time::Instant::now();
|
|
let mut memory_checkpoints: Vec<MemoryTelemetryCheckpoint> = Vec::new();
|
|
let mut malloc_trim_probes: Vec<MallocTrimProbe> = Vec::new();
|
|
let enable_memory_trim_probe = memory_trim_probe_enabled() || args.memory_trim_after_validation;
|
|
record_memory_checkpoint(
|
|
&mut memory_checkpoints,
|
|
"after_store_open",
|
|
&total_started,
|
|
store.as_ref(),
|
|
);
|
|
let validation_started = std::time::Instant::now();
|
|
let collect_current_repo_objects = false;
|
|
let out = if let Some(dir) = args.rsync_local_dir.as_ref() {
|
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
|
timeout: std::time::Duration::from_secs(args.http_timeout_secs.max(1)),
|
|
extra_root_certificates_pem: http_root_certificates_pem.clone(),
|
|
..HttpFetcherConfig::default()
|
|
})
|
|
.map_err(|e| e.to_string())?;
|
|
let rsync = LocalDirRsyncFetcher::new(dir);
|
|
run_online_validation_with_fetchers(
|
|
Arc::clone(&store),
|
|
&policy,
|
|
&args,
|
|
&http,
|
|
&rsync,
|
|
validation_time,
|
|
&config,
|
|
collect_current_repo_objects,
|
|
timing.as_ref().map(|(_, t)| t),
|
|
)?
|
|
} else {
|
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
|
timeout: std::time::Duration::from_secs(args.http_timeout_secs.max(1)),
|
|
extra_root_certificates_pem: http_root_certificates_pem.clone(),
|
|
..HttpFetcherConfig::default()
|
|
})
|
|
.map_err(|e| e.to_string())?;
|
|
let rsync = SystemRsyncFetcher::new(SystemRsyncConfig {
|
|
rsync_bin: args
|
|
.rsync_command
|
|
.clone()
|
|
.unwrap_or_else(|| PathBuf::from("rsync")),
|
|
timeout: std::time::Duration::from_secs(args.rsync_timeout_secs.max(1)),
|
|
mirror_root: args.rsync_mirror_root.clone(),
|
|
scope_policy: args.rsync_scope_policy,
|
|
..SystemRsyncConfig::default()
|
|
});
|
|
run_online_validation_with_fetchers(
|
|
Arc::clone(&store),
|
|
&policy,
|
|
&args,
|
|
&http,
|
|
&rsync,
|
|
validation_time,
|
|
&config,
|
|
collect_current_repo_objects,
|
|
timing.as_ref().map(|(_, t)| t),
|
|
)?
|
|
};
|
|
|
|
let validation_ms = validation_started.elapsed().as_millis() as u64;
|
|
crate::logging::emit(crate::logging::Level::Info, "validation_phase_completed", || serde_json::json!({"elapsed_ms": validation_ms}));
|
|
let shared = PostValidationShared::from_run_output(out);
|
|
record_memory_checkpoint(
|
|
&mut memory_checkpoints,
|
|
"after_validation",
|
|
&total_started,
|
|
store.as_ref(),
|
|
);
|
|
if enable_memory_trim_probe {
|
|
malloc_trim_probes.push(crate::output::memory::malloc_trim_probe());
|
|
record_memory_checkpoint(
|
|
&mut memory_checkpoints,
|
|
"after_validation_malloc_trim",
|
|
&total_started,
|
|
store.as_ref(),
|
|
);
|
|
}
|
|
|
|
if let Some((_out_dir, t)) = timing.as_ref() {
|
|
t.record_count("instances_processed", shared.instances_processed as u64);
|
|
t.record_count("instances_failed", shared.instances_failed as u64);
|
|
}
|
|
|
|
let publication_points = shared.publication_points.len();
|
|
let publication_point_repo_sync_ms_total: u64 = shared
|
|
.publication_points
|
|
.iter()
|
|
.map(|pp| pp.repo_sync_duration_ms.unwrap_or(0))
|
|
.sum();
|
|
let download_event_count = shared.download_stats.events_total;
|
|
let rrdp_download_ms_total: u64 = ["rrdp_notification", "rrdp_snapshot", "rrdp_delta"]
|
|
.iter()
|
|
.map(|key| {
|
|
shared
|
|
.download_stats
|
|
.by_kind
|
|
.get(*key)
|
|
.map(|item| item.duration_ms_total)
|
|
.unwrap_or(0)
|
|
})
|
|
.sum();
|
|
let rsync_download_ms_total = shared
|
|
.download_stats
|
|
.by_kind
|
|
.get("rsync")
|
|
.map(|item| item.duration_ms_total)
|
|
.unwrap_or(0);
|
|
let repo_sync_ms_total = rrdp_download_ms_total + rsync_download_ms_total;
|
|
let download_bytes_total: u64 = shared
|
|
.download_stats
|
|
.by_kind
|
|
.values()
|
|
.map(|item| item.bytes_total.unwrap_or(0))
|
|
.sum();
|
|
|
|
let report_json_format = if args.report_json_compact {
|
|
ReportJsonFormat::Compact
|
|
} else {
|
|
ReportJsonFormat::Pretty
|
|
};
|
|
let ccr_produced_at = time::OffsetDateTime::now_utc();
|
|
let compare_view_trust_anchor = args
|
|
.compare_view_trust_anchor
|
|
.as_deref()
|
|
.unwrap_or("unknown");
|
|
let (report_result, ccr_result, compare_view_result) =
|
|
std::thread::scope(|scope| {
|
|
// Reborrow `shared` as a plain reference so the scoped output tasks
|
|
// capture the reference instead of moving fields out of the owned value.
|
|
let shared = &shared;
|
|
let report_handle = if args.skip_report_build {
|
|
None
|
|
} else {
|
|
Some(scope.spawn(|| {
|
|
run_report_task(
|
|
&policy,
|
|
validation_time,
|
|
shared,
|
|
args.report_json_path.as_deref(),
|
|
report_json_format,
|
|
)
|
|
}))
|
|
};
|
|
let ccr_handle = scope.spawn(|| {
|
|
run_ccr_task(
|
|
shared,
|
|
args.ccr_out_path.as_deref(),
|
|
ccr_produced_at,
|
|
)
|
|
});
|
|
let compare_view_handle = scope.spawn(|| {
|
|
run_compare_view_task(
|
|
shared,
|
|
args.vrps_csv_out_path.as_deref(),
|
|
args.vaps_csv_out_path.as_deref(),
|
|
compare_view_trust_anchor,
|
|
)
|
|
});
|
|
let report_result = match report_handle {
|
|
Some(handle) => handle
|
|
.join()
|
|
.map_err(|_| "report task panicked".to_string())
|
|
.and_then(|result| result),
|
|
None => Ok(ReportTaskOutput::skipped()),
|
|
};
|
|
let ccr_result = ccr_handle
|
|
.join()
|
|
.map_err(|_| "ccr task panicked".to_string())
|
|
.and_then(|result| result);
|
|
let compare_view_result = compare_view_handle
|
|
.join()
|
|
.map_err(|_| "compare view task panicked".to_string())
|
|
.and_then(|result| result);
|
|
(report_result, ccr_result, compare_view_result)
|
|
});
|
|
let report_output = report_result?;
|
|
let ccr_output = ccr_result?;
|
|
let compare_view_output = compare_view_result?;
|
|
record_memory_checkpoint(
|
|
&mut memory_checkpoints,
|
|
"after_report_and_ccr",
|
|
&total_started,
|
|
store.as_ref(),
|
|
);
|
|
if enable_memory_trim_probe {
|
|
malloc_trim_probes.push(crate::output::memory::malloc_trim_probe());
|
|
record_memory_checkpoint(
|
|
&mut memory_checkpoints,
|
|
"after_report_and_ccr_malloc_trim",
|
|
&total_started,
|
|
store.as_ref(),
|
|
);
|
|
}
|
|
let report_build_ms = report_output.report_build_ms;
|
|
let report_write_ms = report_output.report_write_ms;
|
|
let ccr_build_ms = ccr_output.ccr_build_ms;
|
|
let ccr_write_ms = ccr_output.ccr_write_ms;
|
|
let compare_view_build_ms = compare_view_output.build_ms;
|
|
let compare_view_write_ms = compare_view_output.write_ms;
|
|
record_memory_checkpoint(
|
|
&mut memory_checkpoints,
|
|
"after_compare_view",
|
|
&total_started,
|
|
store.as_ref(),
|
|
);
|
|
|
|
record_memory_checkpoint(
|
|
&mut memory_checkpoints,
|
|
"before_stage_timing",
|
|
&total_started,
|
|
store.as_ref(),
|
|
);
|
|
let timing_report_snapshot = timing
|
|
.as_ref()
|
|
.map(|(_, handle)| handle.report_snapshot(50));
|
|
let stage_timing = RunStageTiming {
|
|
validation_ms,
|
|
report_build_ms,
|
|
report_write_ms,
|
|
ccr_build_ms,
|
|
ccr_write_ms,
|
|
compare_view_build_ms,
|
|
compare_view_write_ms,
|
|
total_ms: total_started.elapsed().as_millis() as u64,
|
|
publication_points,
|
|
repo_sync_ms_total,
|
|
publication_point_repo_sync_ms_total,
|
|
download_event_count,
|
|
rrdp_download_ms_total,
|
|
rsync_download_ms_total,
|
|
download_bytes_total,
|
|
analysis_counts: timing
|
|
.as_ref()
|
|
.map(|(_, handle)| handle.counts_snapshot())
|
|
.unwrap_or_default(),
|
|
analysis_phases: timing_report_snapshot
|
|
.as_ref()
|
|
.map(|report| report.phases.clone())
|
|
.unwrap_or_default(),
|
|
analysis_top_publication_points: timing_report_snapshot
|
|
.as_ref()
|
|
.map(|report| report.top_publication_points.clone())
|
|
.unwrap_or_default(),
|
|
analysis_top_publication_point_steps: timing_report_snapshot
|
|
.as_ref()
|
|
.map(|report| report.top_publication_point_steps.clone())
|
|
.unwrap_or_default(),
|
|
memory_telemetry: Some(MemoryTelemetrySummary {
|
|
checkpoints: memory_checkpoints,
|
|
object_graph: Some(estimate_shared_object_graph(&shared)),
|
|
malloc_trim_probes,
|
|
}),
|
|
};
|
|
let stage_timing_anchor_path = args
|
|
.report_json_path
|
|
.as_deref()
|
|
.or(args.ccr_out_path.as_deref())
|
|
.or(args.vrps_csv_out_path.as_deref());
|
|
write_stage_timing(stage_timing_anchor_path, &stage_timing)?;
|
|
|
|
if let Some((out_dir, t)) = timing.as_ref() {
|
|
t.record_count("vrps", shared.vrps.len() as u64);
|
|
t.record_count("aspas", shared.aspas.len() as u64);
|
|
t.record_count(
|
|
"audit_publication_points",
|
|
shared.publication_points.len() as u64,
|
|
);
|
|
let timing_json_path = out_dir.join("timing.json");
|
|
t.write_json(&timing_json_path, 20)?;
|
|
crate::logging::info!("analysis: wrote {}", timing_json_path.display());
|
|
}
|
|
|
|
// Write the compatibility summary after analysis timing has been flushed.
|
|
// The RRDP counters (notably delta operations) are recorded by
|
|
// the transport/validation stages rather than by the compact publication
|
|
// point audit, so the final summary can use those authoritative counts.
|
|
if let Some(path) = args.summary_out_path.as_deref() {
|
|
let analysis_counts = timing
|
|
.as_ref()
|
|
.map(|(_, handle)| handle.counts_snapshot());
|
|
write_summary(
|
|
path,
|
|
&shared,
|
|
&args,
|
|
validation_ms,
|
|
analysis_counts.as_ref(),
|
|
)?;
|
|
}
|
|
|
|
print_summary_from_shared(validation_time, &shared);
|
|
Ok(())
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct ValidationSummary {
|
|
backend: &'static str,
|
|
worker_count: usize,
|
|
repo_sync_worker_count: usize,
|
|
trust_anchors: usize,
|
|
synchronized_repositories: usize,
|
|
synchronized_objects: usize,
|
|
validated_ca_certificates: usize,
|
|
validated_manifests: usize,
|
|
validated_crls: usize,
|
|
validated_roas: usize,
|
|
rejected_ca_certificates: usize,
|
|
rejected_manifests: usize,
|
|
rejected_crls: usize,
|
|
rejected_roas: usize,
|
|
rrdp_snapshot_repositories: usize,
|
|
rrdp_snapshot_fallbacks: usize,
|
|
rrdp_delta_repositories: usize,
|
|
rrdp_noop_repositories: usize,
|
|
rrdp_delta_updates: usize,
|
|
vrps: usize,
|
|
publication_points: usize,
|
|
validation_ms: u64,
|
|
}
|
|
|
|
fn write_summary(
|
|
path: &Path,
|
|
shared: &PostValidationShared,
|
|
args: &RunConfig,
|
|
validation_ms: u64,
|
|
analysis_counts: Option<&std::collections::HashMap<String, u64>>,
|
|
) -> Result<(), String> {
|
|
use crate::output::audit::{AuditDownloadKind, AuditObjectKind, AuditObjectResult};
|
|
use std::collections::BTreeSet;
|
|
|
|
let mut summary = ValidationSummary {
|
|
backend: "panda-rpki",
|
|
worker_count: args.parallel_phase2_config.object_workers,
|
|
repo_sync_worker_count: args.parallel_phase1_config.max_repo_sync_workers_global,
|
|
trust_anchors: shared.discoveries.len().max(1),
|
|
synchronized_repositories: 0,
|
|
synchronized_objects: 0,
|
|
validated_ca_certificates: shared.instances_processed,
|
|
validated_manifests: 0,
|
|
validated_crls: 0,
|
|
validated_roas: 0,
|
|
rejected_ca_certificates: shared.instances_failed,
|
|
rejected_manifests: 0,
|
|
rejected_crls: 0,
|
|
rejected_roas: 0,
|
|
rrdp_snapshot_repositories: 0,
|
|
rrdp_snapshot_fallbacks: 0,
|
|
rrdp_delta_repositories: 0,
|
|
rrdp_noop_repositories: 0,
|
|
rrdp_delta_updates: 0,
|
|
vrps: shared.vrps.len(),
|
|
publication_points: shared.publication_points.len(),
|
|
validation_ms,
|
|
};
|
|
let mut notification_uris = BTreeSet::new();
|
|
for point in shared.publication_points.iter() {
|
|
if let Some(uri) = point.rrdp_notification_uri.as_deref() {
|
|
notification_uris.insert(uri);
|
|
}
|
|
summary.synchronized_objects += point.objects.len();
|
|
match point.repo_sync_phase.as_deref() {
|
|
Some(phase) if phase.contains("fallback") => summary.rrdp_snapshot_fallbacks += 1,
|
|
Some(phase) if phase.contains("snapshot") => {
|
|
summary.rrdp_snapshot_repositories += 1
|
|
}
|
|
Some(phase) if phase.contains("delta") => summary.rrdp_delta_repositories += 1,
|
|
Some(phase) if phase.contains("noop") => summary.rrdp_noop_repositories += 1,
|
|
_ => {}
|
|
}
|
|
for object in &point.objects {
|
|
let accepted = object.result == AuditObjectResult::Ok;
|
|
match &object.kind {
|
|
AuditObjectKind::Manifest => {
|
|
if accepted { summary.validated_manifests += 1; } else { summary.rejected_manifests += 1; }
|
|
}
|
|
AuditObjectKind::Crl => {
|
|
if accepted { summary.validated_crls += 1; } else { summary.rejected_crls += 1; }
|
|
}
|
|
AuditObjectKind::Roa => {
|
|
if accepted { summary.validated_roas += 1; } else { summary.rejected_roas += 1; }
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
}
|
|
// Compact audit mode is not used by the Panda RPKI frontend, but deriving the
|
|
// transport counters from download statistics keeps the summary correct
|
|
// even when a future caller chooses a reduced publication-point audit.
|
|
let download_stat = |kind: &str| shared.download_stats.by_kind.get(kind);
|
|
if let Some(stats) = download_stat("rrdp_snapshot") {
|
|
summary.rrdp_snapshot_repositories = stats.ok_total as usize;
|
|
}
|
|
if let Some(stats) = download_stat("rrdp_delta") {
|
|
summary.rrdp_delta_repositories = stats.ok_total as usize;
|
|
let counted_delta_updates = stats.objects_count_total.unwrap_or(0);
|
|
summary.rrdp_delta_updates = if counted_delta_updates > 0 {
|
|
counted_delta_updates as usize
|
|
} else {
|
|
analysis_counts
|
|
.and_then(|counts| counts.get("rrdp_delta_ops_applied_total"))
|
|
.copied()
|
|
.unwrap_or_else(|| {
|
|
shared
|
|
.downloads
|
|
.iter()
|
|
.filter(|event| event.kind == AuditDownloadKind::RrdpDelta && event.success)
|
|
.filter_map(|event| event.objects.as_ref())
|
|
.map(|objects| objects.objects_count)
|
|
.sum()
|
|
}) as usize
|
|
};
|
|
}
|
|
if let Some(stats) = download_stat("rsync") {
|
|
summary.rrdp_snapshot_fallbacks = stats.ok_total as usize;
|
|
}
|
|
if let Some(stats) = download_stat("rrdp_notification") {
|
|
let transport_repositories = stats.ok_total as usize;
|
|
summary.rrdp_noop_repositories = transport_repositories
|
|
.saturating_sub(summary.rrdp_snapshot_repositories)
|
|
.saturating_sub(summary.rrdp_delta_repositories)
|
|
.saturating_sub(summary.rrdp_snapshot_fallbacks);
|
|
}
|
|
let downloaded_objects = analysis_counts
|
|
.and_then(|counts| {
|
|
let snapshot = counts.get("rrdp_snapshot_objects_applied_total").copied();
|
|
let delta = counts.get("rrdp_delta_ops_applied_total").copied();
|
|
snapshot.zip(delta).map(|(snapshot, delta)| snapshot + delta)
|
|
})
|
|
.unwrap_or_else(|| {
|
|
shared
|
|
.download_stats
|
|
.by_kind
|
|
.values()
|
|
.filter_map(|stats| stats.objects_count_total)
|
|
.sum()
|
|
});
|
|
if downloaded_objects > 0 {
|
|
summary.synchronized_objects = downloaded_objects as usize;
|
|
}
|
|
summary.synchronized_repositories = notification_uris.len();
|
|
|
|
if let Some(parent) = path.parent() {
|
|
std::fs::create_dir_all(parent)
|
|
.map_err(|error| format!("create Panda RPKI summary parent {}: {error}", parent.display()))?;
|
|
}
|
|
let bytes = serde_json::to_vec_pretty(&summary)
|
|
.map_err(|error| format!("serialize Panda RPKI summary: {error}"))?;
|
|
std::fs::write(path, bytes)
|
|
.map_err(|error| format!("write Panda RPKI summary {}: {error}", path.display()))
|
|
}
|