初始化 Panda RPKI v0.1.0 开源候选版本

src/runtime/run.rs

@@ -0,0 +1,548 @@
+// Top-level CLI execution pipeline.
+
+pub(crate) fn run_config(args: RunConfig) -> Result<(), String> {
+    let mut policy = read_policy(args.policy_path.as_deref())?;
+    if let Some(strict_policy) = args.strict_policy {
+        policy.strict = strict_policy;
+    }
+    if let Some(resource_validation_mode) = args.resource_validation_mode {
+        policy.resource_validation_mode = resource_validation_mode;
+    }
+    if args.disable_rrdp {
+        policy.sync_preference = crate::validation::policy::SyncPreference::RsyncOnly;
+    }
+    policy.ta_constraints = args.ta_constraints.clone();
+    for warning in policy.ta_constraints.configuration_warnings() {
+        crate::logging::emit(crate::logging::Level::Warn, "constraint_warning", || serde_json::json!({"reason": warning}));
+    }
+    let validation_time = args
+        .validation_time
+        .unwrap_or_else(time::OffsetDateTime::now_utc);
+    let validation_time =
+        time::OffsetDateTime::from_unix_timestamp(validation_time.unix_timestamp())
+            .map_err(|error| format!("normalize validation time failed: {error}"))?;
+    let http_root_certificates_pem = args
+        .http_root_cert_paths
+        .iter()
+        .map(|path| {
+            std::fs::read(path)
+                .map_err(|e| format!("read HTTP root certificate failed: {}: {e}", path.display()))
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+
+    let store = if args.raw_store_db.is_some() || args.repo_bytes_db.is_some() {
+        Arc::new(
+            RocksStore::open_with_external_stores(
+                &args.db_path,
+                args.raw_store_db.as_deref(),
+                args.repo_bytes_db.as_deref(),
+            )
+            .map_err(|e| e.to_string())?,
+        )
+    } else {
+        Arc::new(RocksStore::open(&args.db_path).map_err(|e| e.to_string())?)
+    };
+    let config = TreeRunConfig {
+        max_depth: Some(args.max_ca_depth),
+        max_instances: args.max_instances,
+        compact_audit: args.skip_report_build
+            && args.report_json_path.is_none(),
+        build_ccr_accumulator: args.ccr_out_path.is_some(),
+    };
+
+    use time::format_description::well_known::Rfc3339;
+    let mut timing: Option<(std::path::PathBuf, TimingHandle)> = None;
+    if args.analyze {
+        let recorded_at_utc_rfc3339 = time::OffsetDateTime::now_utc()
+            .to_offset(time::UtcOffset::UTC)
+            .format(&Rfc3339)
+            .map_err(|e| format!("format recorded_at_utc failed: {e}"))?;
+        let validation_time_utc_rfc3339 = validation_time
+            .to_offset(time::UtcOffset::UTC)
+            .format(&Rfc3339)
+            .map_err(|e| format!("format validation_time failed: {e}"))?;
+
+        let ts_compact = {
+            let fmt = time::format_description::parse_borrowed::<2>(
+                "[year][month][day]T[hour][minute][second]Z",
+            )
+                .map_err(|e| format!("format description parse failed: {e}"))?;
+            time::OffsetDateTime::now_utc()
+                .format(&fmt)
+                .map_err(|e| format!("format timestamp failed: {e}"))?
+        };
+
+        let out_dir = args.analysis_out_path.clone().unwrap_or_else(|| {
+            std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+                .join("target")
+                .join("live")
+                .join("analyze")
+                .join(ts_compact)
+        });
+        std::fs::create_dir_all(&out_dir)
+            .map_err(|e| format!("create analyze out dir failed: {}: {e}", out_dir.display()))?;
+
+        let handle = TimingHandle::new(TimingMeta {
+            recorded_at_utc_rfc3339,
+            validation_time_utc_rfc3339,
+            tal_url: None,
+            db_path: None,
+        });
+        handle.set_meta(TimingMetaUpdate {
+            tal_url: args.tal_url.as_deref(),
+            db_path: Some(args.db_path.to_string_lossy().as_ref()),
+        });
+        timing = Some((out_dir, handle));
+    }
+
+    let total_started = std::time::Instant::now();
+    let mut memory_checkpoints: Vec<MemoryTelemetryCheckpoint> = Vec::new();
+    let mut malloc_trim_probes: Vec<MallocTrimProbe> = Vec::new();
+    let enable_memory_trim_probe = memory_trim_probe_enabled() || args.memory_trim_after_validation;
+    record_memory_checkpoint(
+        &mut memory_checkpoints,
+        "after_store_open",
+        &total_started,
+        store.as_ref(),
+    );
+    let validation_started = std::time::Instant::now();
+    let collect_current_repo_objects = false;
+    let out = if let Some(dir) = args.rsync_local_dir.as_ref() {
+        let http = BlockingHttpFetcher::new(HttpFetcherConfig {
+            timeout: std::time::Duration::from_secs(args.http_timeout_secs.max(1)),
+            extra_root_certificates_pem: http_root_certificates_pem.clone(),
+            ..HttpFetcherConfig::default()
+        })
+        .map_err(|e| e.to_string())?;
+        let rsync = LocalDirRsyncFetcher::new(dir);
+        run_online_validation_with_fetchers(
+            Arc::clone(&store),
+            &policy,
+            &args,
+            &http,
+            &rsync,
+            validation_time,
+            &config,
+            collect_current_repo_objects,
+            timing.as_ref().map(|(_, t)| t),
+        )?
+    } else {
+        let http = BlockingHttpFetcher::new(HttpFetcherConfig {
+            timeout: std::time::Duration::from_secs(args.http_timeout_secs.max(1)),
+            extra_root_certificates_pem: http_root_certificates_pem.clone(),
+            ..HttpFetcherConfig::default()
+        })
+        .map_err(|e| e.to_string())?;
+        let rsync = SystemRsyncFetcher::new(SystemRsyncConfig {
+            rsync_bin: args
+                .rsync_command
+                .clone()
+                .unwrap_or_else(|| PathBuf::from("rsync")),
+            timeout: std::time::Duration::from_secs(args.rsync_timeout_secs.max(1)),
+            mirror_root: args.rsync_mirror_root.clone(),
+            scope_policy: args.rsync_scope_policy,
+            ..SystemRsyncConfig::default()
+        });
+        run_online_validation_with_fetchers(
+            Arc::clone(&store),
+            &policy,
+            &args,
+            &http,
+            &rsync,
+            validation_time,
+            &config,
+            collect_current_repo_objects,
+            timing.as_ref().map(|(_, t)| t),
+        )?
+    };
+
+    let validation_ms = validation_started.elapsed().as_millis() as u64;
+    crate::logging::emit(crate::logging::Level::Info, "validation_phase_completed", || serde_json::json!({"elapsed_ms": validation_ms}));
+    let shared = PostValidationShared::from_run_output(out);
+    record_memory_checkpoint(
+        &mut memory_checkpoints,
+        "after_validation",
+        &total_started,
+        store.as_ref(),
+    );
+    if enable_memory_trim_probe {
+        malloc_trim_probes.push(crate::output::memory::malloc_trim_probe());
+        record_memory_checkpoint(
+            &mut memory_checkpoints,
+            "after_validation_malloc_trim",
+            &total_started,
+            store.as_ref(),
+        );
+    }
+
+    if let Some((_out_dir, t)) = timing.as_ref() {
+        t.record_count("instances_processed", shared.instances_processed as u64);
+        t.record_count("instances_failed", shared.instances_failed as u64);
+    }
+
+    let publication_points = shared.publication_points.len();
+    let publication_point_repo_sync_ms_total: u64 = shared
+        .publication_points
+        .iter()
+        .map(|pp| pp.repo_sync_duration_ms.unwrap_or(0))
+        .sum();
+    let download_event_count = shared.download_stats.events_total;
+    let rrdp_download_ms_total: u64 = ["rrdp_notification", "rrdp_snapshot", "rrdp_delta"]
+        .iter()
+        .map(|key| {
+            shared
+                .download_stats
+                .by_kind
+                .get(*key)
+                .map(|item| item.duration_ms_total)
+                .unwrap_or(0)
+        })
+        .sum();
+    let rsync_download_ms_total = shared
+        .download_stats
+        .by_kind
+        .get("rsync")
+        .map(|item| item.duration_ms_total)
+        .unwrap_or(0);
+    let repo_sync_ms_total = rrdp_download_ms_total + rsync_download_ms_total;
+    let download_bytes_total: u64 = shared
+        .download_stats
+        .by_kind
+        .values()
+        .map(|item| item.bytes_total.unwrap_or(0))
+        .sum();
+
+    let report_json_format = if args.report_json_compact {
+        ReportJsonFormat::Compact
+    } else {
+        ReportJsonFormat::Pretty
+    };
+    let ccr_produced_at = time::OffsetDateTime::now_utc();
+    let compare_view_trust_anchor = args
+        .compare_view_trust_anchor
+        .as_deref()
+        .unwrap_or("unknown");
+    let (report_result, ccr_result, compare_view_result) =
+        std::thread::scope(|scope| {
+            // Reborrow `shared` as a plain reference so the scoped output tasks
+            // capture the reference instead of moving fields out of the owned value.
+            let shared = &shared;
+            let report_handle = if args.skip_report_build {
+                None
+            } else {
+                Some(scope.spawn(|| {
+                    run_report_task(
+                        &policy,
+                        validation_time,
+                        shared,
+                        args.report_json_path.as_deref(),
+                        report_json_format,
+                    )
+                }))
+            };
+            let ccr_handle = scope.spawn(|| {
+                run_ccr_task(
+                    shared,
+                    args.ccr_out_path.as_deref(),
+                    ccr_produced_at,
+                )
+            });
+            let compare_view_handle = scope.spawn(|| {
+                run_compare_view_task(
+                    shared,
+                    args.vrps_csv_out_path.as_deref(),
+                    args.vaps_csv_out_path.as_deref(),
+                    compare_view_trust_anchor,
+                )
+            });
+            let report_result = match report_handle {
+                Some(handle) => handle
+                    .join()
+                    .map_err(|_| "report task panicked".to_string())
+                    .and_then(|result| result),
+                None => Ok(ReportTaskOutput::skipped()),
+            };
+            let ccr_result = ccr_handle
+                .join()
+                .map_err(|_| "ccr task panicked".to_string())
+                .and_then(|result| result);
+            let compare_view_result = compare_view_handle
+                .join()
+                .map_err(|_| "compare view task panicked".to_string())
+                .and_then(|result| result);
+            (report_result, ccr_result, compare_view_result)
+        });
+    let report_output = report_result?;
+    let ccr_output = ccr_result?;
+    let compare_view_output = compare_view_result?;
+    record_memory_checkpoint(
+        &mut memory_checkpoints,
+        "after_report_and_ccr",
+        &total_started,
+        store.as_ref(),
+    );
+    if enable_memory_trim_probe {
+        malloc_trim_probes.push(crate::output::memory::malloc_trim_probe());
+        record_memory_checkpoint(
+            &mut memory_checkpoints,
+            "after_report_and_ccr_malloc_trim",
+            &total_started,
+            store.as_ref(),
+        );
+    }
+    let report_build_ms = report_output.report_build_ms;
+    let report_write_ms = report_output.report_write_ms;
+    let ccr_build_ms = ccr_output.ccr_build_ms;
+    let ccr_write_ms = ccr_output.ccr_write_ms;
+    let compare_view_build_ms = compare_view_output.build_ms;
+    let compare_view_write_ms = compare_view_output.write_ms;
+    record_memory_checkpoint(
+        &mut memory_checkpoints,
+        "after_compare_view",
+        &total_started,
+        store.as_ref(),
+    );
+
+    record_memory_checkpoint(
+        &mut memory_checkpoints,
+        "before_stage_timing",
+        &total_started,
+        store.as_ref(),
+    );
+    let timing_report_snapshot = timing
+        .as_ref()
+        .map(|(_, handle)| handle.report_snapshot(50));
+    let stage_timing = RunStageTiming {
+        validation_ms,
+        report_build_ms,
+        report_write_ms,
+        ccr_build_ms,
+        ccr_write_ms,
+        compare_view_build_ms,
+        compare_view_write_ms,
+        total_ms: total_started.elapsed().as_millis() as u64,
+        publication_points,
+        repo_sync_ms_total,
+        publication_point_repo_sync_ms_total,
+        download_event_count,
+        rrdp_download_ms_total,
+        rsync_download_ms_total,
+        download_bytes_total,
+        analysis_counts: timing
+            .as_ref()
+            .map(|(_, handle)| handle.counts_snapshot())
+            .unwrap_or_default(),
+        analysis_phases: timing_report_snapshot
+            .as_ref()
+            .map(|report| report.phases.clone())
+            .unwrap_or_default(),
+        analysis_top_publication_points: timing_report_snapshot
+            .as_ref()
+            .map(|report| report.top_publication_points.clone())
+            .unwrap_or_default(),
+        analysis_top_publication_point_steps: timing_report_snapshot
+            .as_ref()
+            .map(|report| report.top_publication_point_steps.clone())
+            .unwrap_or_default(),
+        memory_telemetry: Some(MemoryTelemetrySummary {
+            checkpoints: memory_checkpoints,
+            object_graph: Some(estimate_shared_object_graph(&shared)),
+            malloc_trim_probes,
+        }),
+    };
+    let stage_timing_anchor_path = args
+        .report_json_path
+        .as_deref()
+        .or(args.ccr_out_path.as_deref())
+        .or(args.vrps_csv_out_path.as_deref());
+    write_stage_timing(stage_timing_anchor_path, &stage_timing)?;
+
+    if let Some((out_dir, t)) = timing.as_ref() {
+        t.record_count("vrps", shared.vrps.len() as u64);
+        t.record_count("aspas", shared.aspas.len() as u64);
+        t.record_count(
+            "audit_publication_points",
+            shared.publication_points.len() as u64,
+        );
+        let timing_json_path = out_dir.join("timing.json");
+        t.write_json(&timing_json_path, 20)?;
+        crate::logging::info!("analysis: wrote {}", timing_json_path.display());
+    }
+
+    // Write the compatibility summary after analysis timing has been flushed.
+    // The RRDP counters (notably delta operations) are recorded by
+    // the transport/validation stages rather than by the compact publication
+    // point audit, so the final summary can use those authoritative counts.
+    if let Some(path) = args.summary_out_path.as_deref() {
+        let analysis_counts = timing
+            .as_ref()
+            .map(|(_, handle)| handle.counts_snapshot());
+        write_summary(
+            path,
+            &shared,
+            &args,
+            validation_ms,
+            analysis_counts.as_ref(),
+        )?;
+    }
+
+    print_summary_from_shared(validation_time, &shared);
+    Ok(())
+}
+
+#[derive(Serialize)]
+struct ValidationSummary {
+    backend: &'static str,
+    worker_count: usize,
+    repo_sync_worker_count: usize,
+    trust_anchors: usize,
+    synchronized_repositories: usize,
+    synchronized_objects: usize,
+    validated_ca_certificates: usize,
+    validated_manifests: usize,
+    validated_crls: usize,
+    validated_roas: usize,
+    rejected_ca_certificates: usize,
+    rejected_manifests: usize,
+    rejected_crls: usize,
+    rejected_roas: usize,
+    rrdp_snapshot_repositories: usize,
+    rrdp_snapshot_fallbacks: usize,
+    rrdp_delta_repositories: usize,
+    rrdp_noop_repositories: usize,
+    rrdp_delta_updates: usize,
+    vrps: usize,
+    publication_points: usize,
+    validation_ms: u64,
+}
+
+fn write_summary(
+    path: &Path,
+    shared: &PostValidationShared,
+    args: &RunConfig,
+    validation_ms: u64,
+    analysis_counts: Option<&std::collections::HashMap<String, u64>>,
+) -> Result<(), String> {
+    use crate::output::audit::{AuditDownloadKind, AuditObjectKind, AuditObjectResult};
+    use std::collections::BTreeSet;
+
+    let mut summary = ValidationSummary {
+        backend: "panda-rpki",
+        worker_count: args.parallel_phase2_config.object_workers,
+        repo_sync_worker_count: args.parallel_phase1_config.max_repo_sync_workers_global,
+        trust_anchors: shared.discoveries.len().max(1),
+        synchronized_repositories: 0,
+        synchronized_objects: 0,
+        validated_ca_certificates: shared.instances_processed,
+        validated_manifests: 0,
+        validated_crls: 0,
+        validated_roas: 0,
+        rejected_ca_certificates: shared.instances_failed,
+        rejected_manifests: 0,
+        rejected_crls: 0,
+        rejected_roas: 0,
+        rrdp_snapshot_repositories: 0,
+        rrdp_snapshot_fallbacks: 0,
+        rrdp_delta_repositories: 0,
+        rrdp_noop_repositories: 0,
+        rrdp_delta_updates: 0,
+        vrps: shared.vrps.len(),
+        publication_points: shared.publication_points.len(),
+        validation_ms,
+    };
+    let mut notification_uris = BTreeSet::new();
+    for point in shared.publication_points.iter() {
+        if let Some(uri) = point.rrdp_notification_uri.as_deref() {
+            notification_uris.insert(uri);
+        }
+        summary.synchronized_objects += point.objects.len();
+        match point.repo_sync_phase.as_deref() {
+            Some(phase) if phase.contains("fallback") => summary.rrdp_snapshot_fallbacks += 1,
+            Some(phase) if phase.contains("snapshot") => {
+                summary.rrdp_snapshot_repositories += 1
+            }
+            Some(phase) if phase.contains("delta") => summary.rrdp_delta_repositories += 1,
+            Some(phase) if phase.contains("noop") => summary.rrdp_noop_repositories += 1,
+            _ => {}
+        }
+        for object in &point.objects {
+            let accepted = object.result == AuditObjectResult::Ok;
+            match &object.kind {
+                AuditObjectKind::Manifest => {
+                    if accepted { summary.validated_manifests += 1; } else { summary.rejected_manifests += 1; }
+                }
+                AuditObjectKind::Crl => {
+                    if accepted { summary.validated_crls += 1; } else { summary.rejected_crls += 1; }
+                }
+                AuditObjectKind::Roa => {
+                    if accepted { summary.validated_roas += 1; } else { summary.rejected_roas += 1; }
+                }
+                _ => {}
+            }
+        }
+    }
+    // Compact audit mode is not used by the Panda RPKI frontend, but deriving the
+    // transport counters from download statistics keeps the summary correct
+    // even when a future caller chooses a reduced publication-point audit.
+    let download_stat = |kind: &str| shared.download_stats.by_kind.get(kind);
+    if let Some(stats) = download_stat("rrdp_snapshot") {
+        summary.rrdp_snapshot_repositories = stats.ok_total as usize;
+    }
+    if let Some(stats) = download_stat("rrdp_delta") {
+        summary.rrdp_delta_repositories = stats.ok_total as usize;
+        let counted_delta_updates = stats.objects_count_total.unwrap_or(0);
+        summary.rrdp_delta_updates = if counted_delta_updates > 0 {
+            counted_delta_updates as usize
+        } else {
+            analysis_counts
+                .and_then(|counts| counts.get("rrdp_delta_ops_applied_total"))
+                .copied()
+                .unwrap_or_else(|| {
+                    shared
+                        .downloads
+                        .iter()
+                        .filter(|event| event.kind == AuditDownloadKind::RrdpDelta && event.success)
+                        .filter_map(|event| event.objects.as_ref())
+                        .map(|objects| objects.objects_count)
+                        .sum()
+                }) as usize
+        };
+    }
+    if let Some(stats) = download_stat("rsync") {
+        summary.rrdp_snapshot_fallbacks = stats.ok_total as usize;
+    }
+    if let Some(stats) = download_stat("rrdp_notification") {
+        let transport_repositories = stats.ok_total as usize;
+        summary.rrdp_noop_repositories = transport_repositories
+            .saturating_sub(summary.rrdp_snapshot_repositories)
+            .saturating_sub(summary.rrdp_delta_repositories)
+            .saturating_sub(summary.rrdp_snapshot_fallbacks);
+    }
+    let downloaded_objects = analysis_counts
+        .and_then(|counts| {
+            let snapshot = counts.get("rrdp_snapshot_objects_applied_total").copied();
+            let delta = counts.get("rrdp_delta_ops_applied_total").copied();
+            snapshot.zip(delta).map(|(snapshot, delta)| snapshot + delta)
+        })
+        .unwrap_or_else(|| {
+            shared
+        .download_stats
+        .by_kind
+        .values()
+        .filter_map(|stats| stats.objects_count_total)
+        .sum()
+        });
+    if downloaded_objects > 0 {
+        summary.synchronized_objects = downloaded_objects as usize;
+    }
+    summary.synchronized_repositories = notification_uris.len();
+
+    if let Some(parent) = path.parent() {
+        std::fs::create_dir_all(parent)
+            .map_err(|error| format!("create Panda RPKI summary parent {}: {error}", parent.display()))?;
+    }
+    let bytes = serde_json::to_vec_pretty(&summary)
+        .map_err(|error| format!("serialize Panda RPKI summary: {error}"))?;
+    std::fs::write(path, bytes)
+        .map_err(|error| format!("write Panda RPKI summary {}: {error}", path.display()))
+}