20260620 优化delta长尾控制面和PP cache快路径

This commit is contained in:
2026-06-20 15:31:20 +08:00
parent bd266ef2a5
commit 184d3cb95b
33 changed files with 1951 additions and 109 deletions

View File

@@ -1,3 +1,4 @@
use std::cell::RefCell;
use std::io::Write;
use std::time::Duration;
@@ -6,6 +7,19 @@ use reqwest::header::HeaderMap;
use crate::sync::rrdp::Fetcher;
thread_local! {
static HTTP_TIMEOUT_OVERRIDE: RefCell<Option<Duration>> = const { RefCell::new(None) };
}
pub fn with_scoped_http_timeout_override<R>(timeout: Duration, f: impl FnOnce() -> R) -> R {
HTTP_TIMEOUT_OVERRIDE.with(|cell| {
let previous = cell.replace(Some(timeout));
let result = f();
let _ = cell.replace(previous);
result
})
}
#[derive(Clone, Debug)]
pub struct HttpFetcherConfig {
/// Connection-establishment timeout for HTTP requests.
@@ -37,6 +51,7 @@ impl Default for HttpFetcherConfig {
pub struct BlockingHttpFetcher {
short_client: Client,
large_body_client: Client,
retry_short_client: Client,
short_timeout: Duration,
large_body_timeout: Duration,
}
@@ -58,9 +73,16 @@ impl BlockingHttpFetcher {
.user_agent(config.user_agent)
.build()
.map_err(|e| e.to_string())?;
let retry_short_client = Client::builder()
.connect_timeout(Duration::from_secs(1))
.timeout(Duration::from_secs(1))
.user_agent("rpki-dev/0.1 (stage2)")
.build()
.map_err(|e| e.to_string())?;
Ok(Self {
short_client,
large_body_client,
retry_short_client,
short_timeout,
large_body_timeout,
})
@@ -176,6 +198,9 @@ impl BlockingHttpFetcher {
}
fn client_for_uri(&self, uri: &str) -> (&Client, &'static str, Duration) {
if let Some(timeout) = HTTP_TIMEOUT_OVERRIDE.with(|cell| *cell.borrow()) {
return (&self.retry_short_client, "retry_short", timeout);
}
if uses_large_body_timeout(uri) {
(
&self.large_body_client,

View File

@@ -24,6 +24,20 @@ pub trait RsyncFetcher: Send + Sync {
/// Return a list of objects as `(rsync_uri, bytes)` pairs.
fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult<Vec<(String, Vec<u8>)>>;
/// Fetch one object by exact rsync URI.
///
/// The default implementation fetches the parent directory and filters the exact
/// object. Live fetchers should override this to avoid widening one-object TAL
/// bootstrap fetches into whole publication point or module synchronizations.
fn fetch_object(&self, rsync_uri: &str) -> RsyncFetchResult<Vec<u8>> {
let base = parent_rsync_uri(rsync_uri).map_err(RsyncFetchError::Fetch)?;
self.fetch_objects(&base)?
.into_iter()
.find(|(uri, _)| uri == rsync_uri)
.map(|(_, bytes)| bytes)
.ok_or_else(|| RsyncFetchError::Fetch(format!("rsync object not found: {rsync_uri}")))
}
/// Stream fetched objects to a visitor without requiring callers to materialize the
/// full result vector in memory.
fn visit_objects(
@@ -62,6 +76,32 @@ pub trait RsyncFetcher: Send + Sync {
}
}
fn parent_rsync_uri(rsync_uri: &str) -> Result<String, String> {
let parsed = url::Url::parse(rsync_uri).map_err(|e| e.to_string())?;
if parsed.scheme() != "rsync" {
return Err(format!("not an rsync URI: {rsync_uri}"));
}
let host = parsed
.host_str()
.ok_or_else(|| format!("missing host in rsync URI: {rsync_uri}"))?;
let segments = parsed
.path_segments()
.ok_or_else(|| format!("missing path in rsync URI: {rsync_uri}"))?
.collect::<Vec<_>>();
if segments.is_empty() || segments.last().copied().unwrap_or_default().is_empty() {
return Err(format!(
"rsync URI must reference a file object: {rsync_uri}"
));
}
let parent_segments = &segments[..segments.len() - 1];
let mut parent = format!("rsync://{host}/");
if !parent_segments.is_empty() {
parent.push_str(&parent_segments.join("/"));
parent.push('/');
}
Ok(parent)
}
/// A simple "rsync" implementation backed by a local directory.
///
/// This is primarily meant for offline tests and fixtures. The key generation mimics rsync URIs:

View File

@@ -317,6 +317,34 @@ impl RsyncFetcher for SystemRsyncFetcher {
Ok(out)
}
fn fetch_object(&self, rsync_uri: &str) -> RsyncFetchResult<Vec<u8>> {
let parsed =
url::Url::parse(rsync_uri).map_err(|e| RsyncFetchError::Fetch(e.to_string()))?;
if parsed.scheme() != "rsync" {
return Err(RsyncFetchError::Fetch(format!(
"not an rsync URI: {rsync_uri}"
)));
}
let file_name = parsed
.path_segments()
.and_then(|segments| segments.filter(|segment| !segment.is_empty()).next_back())
.ok_or_else(|| {
RsyncFetchError::Fetch(format!(
"rsync URI must reference a file object: {rsync_uri}"
))
})?;
let tmp = TempDir::new().map_err(RsyncFetchError::Fetch)?;
self.run_rsync(rsync_uri, tmp.path())
.map_err(RsyncFetchError::Fetch)?;
let object_path = tmp.path().join(file_name);
std::fs::read(&object_path).map_err(|e| {
RsyncFetchError::Fetch(format!(
"read fetched rsync object failed: {}: {e}",
object_path.display()
))
})
}
fn visit_objects(
&self,
rsync_base_uri: &str,