20260407 & 20260408 基于cir 三方replay对齐，并且materialize 使用hard link优化

src/validation/from_tal.rs

@@ -2,6 +2,7 @@ use url::Url;
 
 use crate::data_model::ta::{TrustAnchor, TrustAnchorError};
 use crate::data_model::tal::{Tal, TalDecodeError};
+use crate::fetch::rsync::RsyncFetcher;
 use crate::sync::rrdp::Fetcher;
 use crate::validation::ca_instance::{
     CaInstanceUris, CaInstanceUrisError, ca_instance_uris_from_ca_certificate,
@@ -104,6 +105,108 @@ pub fn discover_root_ca_instance_from_tal(
     })))
 }
 
+pub fn discover_root_ca_instance_from_tal_with_fetchers(
+    http_fetcher: &dyn Fetcher,
+    rsync_fetcher: &dyn RsyncFetcher,
+    tal: Tal,
+    tal_url: Option<String>,
+) -> Result<DiscoveredRootCaInstance, FromTalError> {
+    if tal.ta_uris.is_empty() {
+        return Err(FromTalError::NoTaUris);
+    }
+
+    let mut last_err: Option<String> = None;
+    let mut ta_uris = tal.ta_uris.clone();
+    ta_uris.sort_by_key(|uri| if uri.scheme() == "rsync" { 0 } else { 1 });
+    for ta_uri in ta_uris.iter() {
+        let ta_der = match fetch_ta_der(http_fetcher, rsync_fetcher, ta_uri) {
+            Ok(b) => b,
+            Err(e) => {
+                last_err = Some(format!("fetch {ta_uri} failed: {e}"));
+                continue;
+            }
+        };
+
+        let trust_anchor = match TrustAnchor::bind_der(tal.clone(), &ta_der, Some(ta_uri)) {
+            Ok(ta) => ta,
+            Err(e) => {
+                last_err = Some(format!("bind {ta_uri} failed: {e}"));
+                continue;
+            }
+        };
+
+        let ca_instance =
+            match ca_instance_uris_from_ca_certificate(&trust_anchor.ta_certificate.rc_ca) {
+                Ok(v) => v,
+                Err(e) => {
+                    last_err = Some(format!("CA instance discovery failed: {e}"));
+                    continue;
+                }
+            };
+
+        return Ok(DiscoveredRootCaInstance {
+            tal_url,
+            trust_anchor,
+            ca_instance,
+        });
+    }
+
+    Err(FromTalError::TaFetch(last_err.unwrap_or_else(|| {
+        "unknown TA candidate error".to_string()
+    })))
+}
+
+fn fetch_ta_der(
+    http_fetcher: &dyn Fetcher,
+    rsync_fetcher: &dyn RsyncFetcher,
+    ta_uri: &Url,
+) -> Result<Vec<u8>, String> {
+    match ta_uri.scheme() {
+        "https" | "http" => http_fetcher.fetch(ta_uri.as_str()),
+        "rsync" => fetch_ta_der_via_rsync(rsync_fetcher, ta_uri.as_str()),
+        scheme => Err(format!("unsupported TA URI scheme: {scheme}")),
+    }
+}
+
+fn fetch_ta_der_via_rsync(
+    rsync_fetcher: &dyn RsyncFetcher,
+    ta_rsync_uri: &str,
+) -> Result<Vec<u8>, String> {
+    let base = rsync_parent_uri(ta_rsync_uri)?;
+    let objects = rsync_fetcher
+        .fetch_objects(&base)
+        .map_err(|e| e.to_string())?;
+    objects
+        .into_iter()
+        .find(|(uri, _)| uri == ta_rsync_uri)
+        .map(|(_, bytes)| bytes)
+        .ok_or_else(|| format!("TA rsync object not found in fetched subtree: {ta_rsync_uri}"))
+}
+
+fn rsync_parent_uri(ta_rsync_uri: &str) -> Result<String, String> {
+    let url = Url::parse(ta_rsync_uri).map_err(|e| e.to_string())?;
+    if url.scheme() != "rsync" {
+        return Err(format!("not an rsync URI: {ta_rsync_uri}"));
+    }
+    let host = url
+        .host_str()
+        .ok_or_else(|| format!("missing host in rsync URI: {ta_rsync_uri}"))?;
+    let segments = url
+        .path_segments()
+        .ok_or_else(|| format!("missing path in rsync URI: {ta_rsync_uri}"))?
+        .collect::<Vec<_>>();
+    if segments.is_empty() || segments.last().copied().unwrap_or_default().is_empty() {
+        return Err(format!("rsync URI must reference a file object: {ta_rsync_uri}"));
+    }
+    let parent_segments = &segments[..segments.len() - 1];
+    let mut parent = format!("rsync://{host}/");
+    if !parent_segments.is_empty() {
+        parent.push_str(&parent_segments.join("/"));
+        parent.push('/');
+    }
+    Ok(parent)
+}
+
 pub fn discover_root_ca_instance_from_tal_and_ta_der(
     tal_bytes: &[u8],
     ta_der: &[u8],
@@ -119,6 +222,58 @@ pub fn discover_root_ca_instance_from_tal_and_ta_der(
     })
 }
 
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::fetch::rsync::LocalDirRsyncFetcher;
+
+    #[test]
+    fn discover_root_ca_instance_from_tal_with_fetchers_supports_rsync_ta_uri() {
+        let tal_bytes = std::fs::read(
+            std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+                .join("tests/fixtures/tal/apnic-rfc7730-https.tal"),
+        )
+        .unwrap();
+        let ta_der = std::fs::read(
+            std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+                .join("tests/fixtures/ta/apnic-ta.cer"),
+        )
+        .unwrap();
+        let tal = Tal::decode_bytes(&tal_bytes).unwrap();
+        let rsync_uri = tal
+            .ta_uris
+            .iter()
+            .find(|uri| uri.scheme() == "rsync")
+            .unwrap()
+            .clone();
+
+        let td = tempfile::tempdir().unwrap();
+        let mirror_root = td.path().join(rsync_uri.host_str().unwrap()).join("repository");
+        std::fs::create_dir_all(&mirror_root).unwrap();
+        std::fs::write(
+            mirror_root.join("apnic-rpki-root-iana-origin.cer"),
+            ta_der,
+        )
+        .unwrap();
+
+        let http = crate::fetch::http::BlockingHttpFetcher::new(
+            crate::fetch::http::HttpFetcherConfig::default(),
+        )
+        .unwrap();
+        let rsync = LocalDirRsyncFetcher::new(
+            td.path().join(rsync_uri.host_str().unwrap()).join("repository"),
+        );
+        let discovery = discover_root_ca_instance_from_tal_with_fetchers(&http, &rsync, tal, None)
+            .expect("discover via rsync TA");
+        assert!(discovery
+            .trust_anchor
+            .resolved_ta_uri
+            .unwrap()
+            .as_str()
+            .starts_with("rsync://"));
+    }
+}
+
 pub fn run_root_from_tal_url_once(
     store: &crate::storage::RocksStore,
     policy: &crate::policy::Policy,

src/validation/run_tree_from_tal.rs

@@ -15,6 +15,7 @@ use crate::replay::fetch_rsync::PayloadReplayRsyncFetcher;
 use crate::sync::rrdp::Fetcher;
 use crate::validation::from_tal::{
     DiscoveredRootCaInstance, FromTalError, discover_root_ca_instance_from_tal_and_ta_der,
+    discover_root_ca_instance_from_tal_with_fetchers,
     discover_root_ca_instance_from_tal_url,
 };
 use crate::validation::tree::{
@@ -295,6 +296,117 @@ pub fn run_tree_from_tal_and_ta_der_serial(
     Ok(RunTreeFromTalOutput { discovery, tree })
 }
 
+pub fn run_tree_from_tal_bytes_serial_audit(
+    store: &crate::storage::RocksStore,
+    policy: &crate::policy::Policy,
+    tal_bytes: &[u8],
+    tal_uri: Option<String>,
+    http_fetcher: &dyn Fetcher,
+    rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher,
+    validation_time: time::OffsetDateTime,
+    config: &TreeRunConfig,
+) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError> {
+    let tal = crate::data_model::tal::Tal::decode_bytes(tal_bytes).map_err(FromTalError::from)?;
+    let discovery =
+        discover_root_ca_instance_from_tal_with_fetchers(http_fetcher, rsync_fetcher, tal, tal_uri)?;
+
+    let download_log = DownloadLogHandle::new();
+    let runner = Rpkiv1PublicationPointRunner {
+        store,
+        policy,
+        http_fetcher,
+        rsync_fetcher,
+        validation_time,
+        timing: None,
+        download_log: Some(download_log.clone()),
+        replay_archive_index: None,
+        replay_delta_index: None,
+        rrdp_dedup: true,
+        rrdp_repo_cache: Mutex::new(HashMap::new()),
+        rsync_dedup: true,
+        rsync_repo_cache: Mutex::new(HashMap::new()),
+    };
+
+    let root = root_handle_from_trust_anchor(
+        &discovery.trust_anchor,
+        derive_tal_id(&discovery),
+        None,
+        &discovery.ca_instance,
+    );
+    let TreeRunAuditOutput {
+        tree,
+        publication_points,
+    } = run_tree_serial_audit(root, &runner, config)?;
+
+    let downloads = download_log.snapshot_events();
+    let download_stats = DownloadLogHandle::stats_from_events(&downloads);
+    Ok(RunTreeFromTalAuditOutput {
+        discovery,
+        tree,
+        publication_points,
+        downloads,
+        download_stats,
+    })
+}
+
+pub fn run_tree_from_tal_bytes_serial_audit_with_timing(
+    store: &crate::storage::RocksStore,
+    policy: &crate::policy::Policy,
+    tal_bytes: &[u8],
+    tal_uri: Option<String>,
+    http_fetcher: &dyn Fetcher,
+    rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher,
+    validation_time: time::OffsetDateTime,
+    config: &TreeRunConfig,
+    timing: &TimingHandle,
+) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError> {
+    let _tal = timing.span_phase("tal_bootstrap");
+    let tal = crate::data_model::tal::Tal::decode_bytes(tal_bytes).map_err(FromTalError::from)?;
+    let discovery =
+        discover_root_ca_instance_from_tal_with_fetchers(http_fetcher, rsync_fetcher, tal, tal_uri)?;
+    drop(_tal);
+
+    let download_log = DownloadLogHandle::new();
+    let runner = Rpkiv1PublicationPointRunner {
+        store,
+        policy,
+        http_fetcher,
+        rsync_fetcher,
+        validation_time,
+        timing: Some(timing.clone()),
+        download_log: Some(download_log.clone()),
+        replay_archive_index: None,
+        replay_delta_index: None,
+        rrdp_dedup: true,
+        rrdp_repo_cache: Mutex::new(HashMap::new()),
+        rsync_dedup: true,
+        rsync_repo_cache: Mutex::new(HashMap::new()),
+    };
+
+    let root = root_handle_from_trust_anchor(
+        &discovery.trust_anchor,
+        derive_tal_id(&discovery),
+        None,
+        &discovery.ca_instance,
+    );
+    let _tree = timing.span_phase("tree_run");
+    let TreeRunAuditOutput {
+        tree,
+        publication_points,
+    } = run_tree_serial_audit(root, &runner, config)?;
+    drop(_tree);
+
+    let downloads = download_log.snapshot_events();
+    let download_stats = DownloadLogHandle::stats_from_events(&downloads);
+    Ok(RunTreeFromTalAuditOutput {
+        discovery,
+        tree,
+        publication_points,
+        downloads,
+        download_stats,
+    })
+}
+
 pub fn run_tree_from_tal_and_ta_der_serial_audit(
     store: &crate::storage::RocksStore,
     policy: &crate::policy::Policy,

src/validation/tree_runner.rs

@@ -111,7 +111,11 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
         let attempted_rrdp =
             self.policy.sync_preference == crate::policy::SyncPreference::RrdpThenRsync;
         let original_notification_uri = ca.rrdp_notification_uri.as_deref();
-        let mut effective_notification_uri = original_notification_uri;
+        let mut effective_notification_uri = if attempted_rrdp {
+            original_notification_uri
+        } else {
+            None
+        };
         let mut skip_sync_due_to_dedup = false;
 
         if attempted_rrdp && self.rrdp_dedup {
@@ -4129,6 +4133,116 @@ authorityKeyIdentifier = keyid:always
         assert_eq!(calls.load(Ordering::SeqCst), 1, "module-scope dedup should skip second sync");
     }
 
+    #[test]
+    fn runner_rsync_dedup_works_in_rsync_only_mode_even_when_rrdp_notify_exists() {
+        let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+            .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0");
+        assert!(fixture_dir.is_dir(), "fixture directory must exist");
+
+        let first_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string();
+        let second_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/sub/".to_string();
+        let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft";
+        let manifest_rsync_uri = format!("{first_base_uri}{manifest_file}");
+
+        let fixture_manifest_bytes =
+            std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture");
+        let fixture_manifest =
+            crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes)
+                .expect("decode manifest fixture");
+        let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60);
+
+        let store_dir = tempfile::tempdir().expect("store dir");
+        let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
+        let policy = Policy {
+            sync_preference: crate::policy::SyncPreference::RsyncOnly,
+            ..Policy::default()
+        };
+
+        let issuer_ca_der = std::fs::read(
+            std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(
+                "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer",
+            ),
+        )
+        .expect("read issuer ca fixture");
+        let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca");
+
+        let handle = CaInstanceHandle {
+            depth: 0,
+            tal_id: "test-tal".to_string(),
+            parent_manifest_rsync_uri: None,
+            ca_certificate_der: issuer_ca_der,
+            ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()),
+            effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(),
+            effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(),
+            rsync_base_uri: first_base_uri.clone(),
+            manifest_rsync_uri: manifest_rsync_uri.clone(),
+            publication_point_rsync_uri: first_base_uri.clone(),
+            rrdp_notification_uri: Some("https://rrdp.example.test/notification.xml".to_string()),
+        };
+        let second_handle = CaInstanceHandle {
+            rsync_base_uri: second_base_uri.clone(),
+            publication_point_rsync_uri: second_base_uri.clone(),
+            ..handle.clone()
+        };
+
+        struct ModuleScopeRsyncFetcher {
+            inner: LocalDirRsyncFetcher,
+            calls: Arc<AtomicUsize>,
+        }
+        impl RsyncFetcher for ModuleScopeRsyncFetcher {
+            fn fetch_objects(
+                &self,
+                rsync_base_uri: &str,
+            ) -> Result<Vec<(String, Vec<u8>)>, RsyncFetchError> {
+                self.calls.fetch_add(1, Ordering::SeqCst);
+                self.inner.fetch_objects(rsync_base_uri)
+            }
+
+            fn dedup_key(&self, _rsync_base_uri: &str) -> String {
+                "rsync://rpki.cernet.net/repo/".to_string()
+            }
+        }
+
+        let calls = Arc::new(AtomicUsize::new(0));
+        let rsync = ModuleScopeRsyncFetcher {
+            inner: LocalDirRsyncFetcher::new(&fixture_dir),
+            calls: calls.clone(),
+        };
+
+        let runner = Rpkiv1PublicationPointRunner {
+            store: &store,
+            policy: &policy,
+            http_fetcher: &NeverHttpFetcher,
+            rsync_fetcher: &rsync,
+            validation_time,
+            timing: None,
+            download_log: None,
+            replay_archive_index: None,
+            replay_delta_index: None,
+            rrdp_dedup: true,
+            rrdp_repo_cache: Mutex::new(HashMap::new()),
+            rsync_dedup: true,
+            rsync_repo_cache: Mutex::new(HashMap::new()),
+        };
+
+        let first = runner.run_publication_point(&handle).expect("first run ok");
+        assert_eq!(first.source, PublicationPointSource::Fresh);
+
+        let second = runner
+            .run_publication_point(&second_handle)
+            .expect("second run ok");
+        assert!(matches!(
+            second.source,
+            PublicationPointSource::Fresh | PublicationPointSource::VcirCurrentInstance
+        ));
+
+        assert_eq!(
+            calls.load(Ordering::SeqCst),
+            1,
+            "rsync-only mode must deduplicate by rsync scope even when RRDP notification is present"
+        );
+    }
+
     #[test]
     fn runner_when_repo_sync_fails_uses_current_instance_vcir_and_keeps_children_empty_for_fixture()
     {