全量同步测试增加download过程审计输出

src/validation/run.rs

@@ -52,6 +52,7 @@ pub fn run_publication_point_once(
         http_fetcher,
         rsync_fetcher,
         None,
+        None,
     )?;
 
     let publication_point = process_manifest_publication_point(

src/validation/run_tree_from_tal.rs

@@ -2,6 +2,7 @@ use url::Url;
 
 use crate::analysis::timing::TimingHandle;
 use crate::audit::PublicationPointAudit;
+use crate::audit_downloads::DownloadLogHandle;
 use crate::data_model::ta::TrustAnchor;
 use crate::sync::rrdp::Fetcher;
 use crate::validation::from_tal::{
@@ -27,6 +28,8 @@ pub struct RunTreeFromTalAuditOutput {
     pub discovery: DiscoveredRootCaInstance,
     pub tree: TreeRunOutput,
     pub publication_points: Vec<PublicationPointAudit>,
+    pub downloads: Vec<crate::audit::AuditDownloadEvent>,
+    pub download_stats: crate::audit::AuditDownloadStats,
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -75,6 +78,7 @@ pub fn run_tree_from_tal_url_serial(
         rsync_fetcher,
         validation_time,
         timing: None,
+        download_log: None,
         revalidate_only: config.revalidate_only,
         rrdp_dedup: true,
         rrdp_repo_cache: Mutex::new(HashMap::new()),
@@ -99,6 +103,7 @@ pub fn run_tree_from_tal_url_serial_audit(
 ) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError> {
     let discovery = discover_root_ca_instance_from_tal_url(http_fetcher, tal_url)?;
 
+    let download_log = DownloadLogHandle::new();
     let runner = Rpkiv1PublicationPointRunner {
         store,
         policy,
@@ -106,6 +111,7 @@ pub fn run_tree_from_tal_url_serial_audit(
         rsync_fetcher,
         validation_time,
         timing: None,
+        download_log: Some(download_log.clone()),
         revalidate_only: config.revalidate_only,
         rrdp_dedup: true,
         rrdp_repo_cache: Mutex::new(HashMap::new()),
@@ -119,10 +125,14 @@ pub fn run_tree_from_tal_url_serial_audit(
         publication_points,
     } = run_tree_serial_audit(root, &runner, config)?;
 
+    let downloads = download_log.snapshot_events();
+    let download_stats = DownloadLogHandle::stats_from_events(&downloads);
     Ok(RunTreeFromTalAuditOutput {
         discovery,
         tree,
         publication_points,
+        downloads,
+        download_stats,
     })
 }
 
@@ -140,6 +150,7 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing(
     let discovery = discover_root_ca_instance_from_tal_url(http_fetcher, tal_url)?;
     drop(_tal);
 
+    let download_log = DownloadLogHandle::new();
     let runner = Rpkiv1PublicationPointRunner {
         store,
         policy,
@@ -147,6 +158,7 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing(
         rsync_fetcher,
         validation_time,
         timing: Some(timing.clone()),
+        download_log: Some(download_log.clone()),
         revalidate_only: config.revalidate_only,
         rrdp_dedup: true,
         rrdp_repo_cache: Mutex::new(HashMap::new()),
@@ -161,10 +173,14 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing(
         publication_points,
     } = run_tree_serial_audit(root, &runner, config)?;
 
+    let downloads = download_log.snapshot_events();
+    let download_stats = DownloadLogHandle::stats_from_events(&downloads);
     Ok(RunTreeFromTalAuditOutput {
         discovery,
         tree,
         publication_points,
+        downloads,
+        download_stats,
     })
 }
 
@@ -189,6 +205,7 @@ pub fn run_tree_from_tal_and_ta_der_serial(
         rsync_fetcher,
         validation_time,
         timing: None,
+        download_log: None,
         revalidate_only: config.revalidate_only,
         rrdp_dedup: true,
         rrdp_repo_cache: Mutex::new(HashMap::new()),
@@ -216,6 +233,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit(
     let discovery =
         discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri)?;
 
+    let download_log = DownloadLogHandle::new();
     let runner = Rpkiv1PublicationPointRunner {
         store,
         policy,
@@ -223,6 +241,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit(
         rsync_fetcher,
         validation_time,
         timing: None,
+        download_log: Some(download_log.clone()),
         revalidate_only: config.revalidate_only,
         rrdp_dedup: true,
         rrdp_repo_cache: Mutex::new(HashMap::new()),
@@ -236,10 +255,14 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit(
         publication_points,
     } = run_tree_serial_audit(root, &runner, config)?;
 
+    let downloads = download_log.snapshot_events();
+    let download_stats = DownloadLogHandle::stats_from_events(&downloads);
     Ok(RunTreeFromTalAuditOutput {
         discovery,
         tree,
         publication_points,
+        downloads,
+        download_stats,
     })
 }
 
@@ -260,6 +283,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing(
         discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri)?;
     drop(_tal);
 
+    let download_log = DownloadLogHandle::new();
     let runner = Rpkiv1PublicationPointRunner {
         store,
         policy,
@@ -267,6 +291,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing(
         rsync_fetcher,
         validation_time,
         timing: Some(timing.clone()),
+        download_log: Some(download_log.clone()),
         revalidate_only: config.revalidate_only,
         rrdp_dedup: true,
         rrdp_repo_cache: Mutex::new(HashMap::new()),
@@ -281,9 +306,13 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing(
         publication_points,
     } = run_tree_serial_audit(root, &runner, config)?;
 
+    let downloads = download_log.snapshot_events();
+    let download_stats = DownloadLogHandle::stats_from_events(&downloads);
     Ok(RunTreeFromTalAuditOutput {
         discovery,
         tree,
         publication_points,
+        downloads,
+        download_stats,
     })
 }

src/validation/tree_runner.rs

@@ -3,6 +3,7 @@ use crate::audit::{
     AuditObjectKind, AuditObjectResult, AuditWarning, ObjectAuditEntry, PublicationPointAudit,
     sha256_hex, sha256_hex_from_32,
 };
+use crate::audit_downloads::DownloadLogHandle;
 use crate::fetch::rsync::RsyncFetcher;
 use crate::policy::Policy;
 use crate::report::{RfcRef, Warning};
@@ -33,6 +34,7 @@ pub struct Rpkiv1PublicationPointRunner<'a> {
     pub rsync_fetcher: &'a dyn RsyncFetcher,
     pub validation_time: time::OffsetDateTime,
     pub timing: Option<TimingHandle>,
+    pub download_log: Option<DownloadLogHandle>,
     pub revalidate_only: bool,
     /// In-run RRDP dedup: when RRDP is enabled, only sync each `rrdp_notification_uri` once per run.
     ///
@@ -157,6 +159,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
                     self.http_fetcher,
                     self.rsync_fetcher,
                     self.timing.as_ref(),
+                    self.download_log.as_ref(),
                 ) {
                     Ok(res) => {
                         // Populate rsync dedup cache when we actually used rsync.
@@ -1390,6 +1393,7 @@ authorityKeyIdentifier = keyid:always
             rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir),
             validation_time,
             timing: None,
+            download_log: None,
             revalidate_only: false,
             rrdp_dedup: false,
             rrdp_repo_cache: Mutex::new(HashMap::new()),
@@ -1502,6 +1506,7 @@ authorityKeyIdentifier = keyid:always
             rsync_fetcher: &rsync,
             validation_time,
             timing: None,
+            download_log: None,
             revalidate_only: false,
             rrdp_dedup: false,
             rrdp_repo_cache: Mutex::new(HashMap::new()),
@@ -1576,6 +1581,7 @@ authorityKeyIdentifier = keyid:always
             rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir),
             validation_time,
             timing: None,
+            download_log: None,
             revalidate_only: false,
             rrdp_dedup: false,
             rrdp_repo_cache: Mutex::new(HashMap::new()),
@@ -1599,6 +1605,7 @@ authorityKeyIdentifier = keyid:always
             rsync_fetcher: &FailingRsyncFetcher,
             validation_time,
             timing: None,
+            download_log: None,
             revalidate_only: false,
             rrdp_dedup: false,
             rrdp_repo_cache: Mutex::new(HashMap::new()),