20260620 优化delta长尾控制面和PP cache快路径
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
use std::cell::RefCell;
|
||||
use std::io::Write;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -6,6 +7,19 @@ use reqwest::header::HeaderMap;
|
||||
|
||||
use crate::sync::rrdp::Fetcher;
|
||||
|
||||
thread_local! {
|
||||
static HTTP_TIMEOUT_OVERRIDE: RefCell<Option<Duration>> = const { RefCell::new(None) };
|
||||
}
|
||||
|
||||
pub fn with_scoped_http_timeout_override<R>(timeout: Duration, f: impl FnOnce() -> R) -> R {
|
||||
HTTP_TIMEOUT_OVERRIDE.with(|cell| {
|
||||
let previous = cell.replace(Some(timeout));
|
||||
let result = f();
|
||||
let _ = cell.replace(previous);
|
||||
result
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct HttpFetcherConfig {
|
||||
/// Connection-establishment timeout for HTTP requests.
|
||||
@@ -37,6 +51,7 @@ impl Default for HttpFetcherConfig {
|
||||
pub struct BlockingHttpFetcher {
|
||||
short_client: Client,
|
||||
large_body_client: Client,
|
||||
retry_short_client: Client,
|
||||
short_timeout: Duration,
|
||||
large_body_timeout: Duration,
|
||||
}
|
||||
@@ -58,9 +73,16 @@ impl BlockingHttpFetcher {
|
||||
.user_agent(config.user_agent)
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let retry_short_client = Client::builder()
|
||||
.connect_timeout(Duration::from_secs(1))
|
||||
.timeout(Duration::from_secs(1))
|
||||
.user_agent("rpki-dev/0.1 (stage2)")
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(Self {
|
||||
short_client,
|
||||
large_body_client,
|
||||
retry_short_client,
|
||||
short_timeout,
|
||||
large_body_timeout,
|
||||
})
|
||||
@@ -176,6 +198,9 @@ impl BlockingHttpFetcher {
|
||||
}
|
||||
|
||||
fn client_for_uri(&self, uri: &str) -> (&Client, &'static str, Duration) {
|
||||
if let Some(timeout) = HTTP_TIMEOUT_OVERRIDE.with(|cell| *cell.borrow()) {
|
||||
return (&self.retry_short_client, "retry_short", timeout);
|
||||
}
|
||||
if uses_large_body_timeout(uri) {
|
||||
(
|
||||
&self.large_body_client,
|
||||
|
||||
@@ -24,6 +24,20 @@ pub trait RsyncFetcher: Send + Sync {
|
||||
/// Return a list of objects as `(rsync_uri, bytes)` pairs.
|
||||
fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult<Vec<(String, Vec<u8>)>>;
|
||||
|
||||
/// Fetch one object by exact rsync URI.
|
||||
///
|
||||
/// The default implementation fetches the parent directory and filters the exact
|
||||
/// object. Live fetchers should override this to avoid widening one-object TAL
|
||||
/// bootstrap fetches into whole publication point or module synchronizations.
|
||||
fn fetch_object(&self, rsync_uri: &str) -> RsyncFetchResult<Vec<u8>> {
|
||||
let base = parent_rsync_uri(rsync_uri).map_err(RsyncFetchError::Fetch)?;
|
||||
self.fetch_objects(&base)?
|
||||
.into_iter()
|
||||
.find(|(uri, _)| uri == rsync_uri)
|
||||
.map(|(_, bytes)| bytes)
|
||||
.ok_or_else(|| RsyncFetchError::Fetch(format!("rsync object not found: {rsync_uri}")))
|
||||
}
|
||||
|
||||
/// Stream fetched objects to a visitor without requiring callers to materialize the
|
||||
/// full result vector in memory.
|
||||
fn visit_objects(
|
||||
@@ -62,6 +76,32 @@ pub trait RsyncFetcher: Send + Sync {
|
||||
}
|
||||
}
|
||||
|
||||
fn parent_rsync_uri(rsync_uri: &str) -> Result<String, String> {
|
||||
let parsed = url::Url::parse(rsync_uri).map_err(|e| e.to_string())?;
|
||||
if parsed.scheme() != "rsync" {
|
||||
return Err(format!("not an rsync URI: {rsync_uri}"));
|
||||
}
|
||||
let host = parsed
|
||||
.host_str()
|
||||
.ok_or_else(|| format!("missing host in rsync URI: {rsync_uri}"))?;
|
||||
let segments = parsed
|
||||
.path_segments()
|
||||
.ok_or_else(|| format!("missing path in rsync URI: {rsync_uri}"))?
|
||||
.collect::<Vec<_>>();
|
||||
if segments.is_empty() || segments.last().copied().unwrap_or_default().is_empty() {
|
||||
return Err(format!(
|
||||
"rsync URI must reference a file object: {rsync_uri}"
|
||||
));
|
||||
}
|
||||
let parent_segments = &segments[..segments.len() - 1];
|
||||
let mut parent = format!("rsync://{host}/");
|
||||
if !parent_segments.is_empty() {
|
||||
parent.push_str(&parent_segments.join("/"));
|
||||
parent.push('/');
|
||||
}
|
||||
Ok(parent)
|
||||
}
|
||||
|
||||
/// A simple "rsync" implementation backed by a local directory.
|
||||
///
|
||||
/// This is primarily meant for offline tests and fixtures. The key generation mimics rsync URIs:
|
||||
|
||||
@@ -317,6 +317,34 @@ impl RsyncFetcher for SystemRsyncFetcher {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn fetch_object(&self, rsync_uri: &str) -> RsyncFetchResult<Vec<u8>> {
|
||||
let parsed =
|
||||
url::Url::parse(rsync_uri).map_err(|e| RsyncFetchError::Fetch(e.to_string()))?;
|
||||
if parsed.scheme() != "rsync" {
|
||||
return Err(RsyncFetchError::Fetch(format!(
|
||||
"not an rsync URI: {rsync_uri}"
|
||||
)));
|
||||
}
|
||||
let file_name = parsed
|
||||
.path_segments()
|
||||
.and_then(|segments| segments.filter(|segment| !segment.is_empty()).next_back())
|
||||
.ok_or_else(|| {
|
||||
RsyncFetchError::Fetch(format!(
|
||||
"rsync URI must reference a file object: {rsync_uri}"
|
||||
))
|
||||
})?;
|
||||
let tmp = TempDir::new().map_err(RsyncFetchError::Fetch)?;
|
||||
self.run_rsync(rsync_uri, tmp.path())
|
||||
.map_err(RsyncFetchError::Fetch)?;
|
||||
let object_path = tmp.path().join(file_name);
|
||||
std::fs::read(&object_path).map_err(|e| {
|
||||
RsyncFetchError::Fetch(format!(
|
||||
"read fetched rsync object failed: {}: {e}",
|
||||
object_path.display()
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn visit_objects(
|
||||
&self,
|
||||
rsync_base_uri: &str,
|
||||
|
||||
Reference in New Issue
Block a user