NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260410 完成五个rir 基于cir的三方回放，raw by hash 独立db，发现内存占用大，连续大rir 录制发生oom

Browse Source
This commit is contained in:
yuyr
2026-04-11 11:24:32 +08:00
parent e083fe4daa
commit 77fc2f1a41
96 changed files with 5159 additions and 1720 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
src/fetch/http.rs
View File

@@ -7,6 +7,8 @@ use crate::sync::rrdp::Fetcher;
#[derive(Clone, Debug)]
pub struct HttpFetcherConfig {
/// Connection-establishment timeout for HTTP requests.
pub connect_timeout: Duration,
/// Short timeout used for connection establishment and small metadata objects.
pub timeout: Duration,
/// Larger timeout used for RRDP snapshot / delta bodies.
@@ -17,7 +19,8 @@ pub struct HttpFetcherConfig {
impl Default for HttpFetcherConfig {
fn default() -> Self {
Self {
timeout: Duration::from_secs(20),
connect_timeout: Duration::from_secs(15),
timeout: Duration::from_secs(30),
large_body_timeout: Duration::from_secs(180),
user_agent: "rpki-dev/0.1 (stage2)".to_string(),
}
@@ -41,14 +44,15 @@ impl BlockingHttpFetcher {
pub fn new(config: HttpFetcherConfig) -> Result<Self, String> {
let short_timeout = config.timeout;
let large_body_timeout = std::cmp::max(config.large_body_timeout, config.timeout);
let connect_timeout = std::cmp::min(config.connect_timeout, config.timeout);
let short_client = Client::builder()
.connect_timeout(config.timeout)
.connect_timeout(connect_timeout)
.timeout(config.timeout)
.user_agent(config.user_agent.clone())
.build()
.map_err(|e| e.to_string())?;
let large_body_client = Client::builder()
.connect_timeout(config.timeout)
.connect_timeout(connect_timeout)
.timeout(large_body_timeout)
.user_agent(config.user_agent)
.build()
@@ -64,24 +68,21 @@ impl BlockingHttpFetcher {
pub fn fetch_bytes(&self, uri: &str) -> Result<Vec<u8>, String> {
let started = std::time::Instant::now();
let (client, timeout_profile, timeout_value) = self.client_for_uri(uri);
let resp = client
.get(uri)
.send()
.map_err(|e| {
let msg = format!("http request failed: {e}");
crate::progress_log::emit(
"http_fetch_failed",
serde_json::json!({
"uri": uri,
"stage": "request",
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": started.elapsed().as_millis() as u64,
"error": msg,
}),
);
msg
})?;
let resp = client.get(uri).send().map_err(|e| {
let msg = format!("http request failed: {e}");
crate::progress_log::emit(
"http_fetch_failed",
serde_json::json!({
"uri": uri,
"stage": "request",
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": started.elapsed().as_millis() as u64,
"error": msg,
}),
);
msg
})?;
let status = resp.status();
let headers = resp.headers().clone();
@@ -175,7 +176,11 @@ impl BlockingHttpFetcher {
fn client_for_uri(&self, uri: &str) -> (&Client, &'static str, Duration) {
if uses_large_body_timeout(uri) {
(&self.large_body_client, "large_body", self.large_body_timeout)
(
&self.large_body_client,
"large_body",
self.large_body_timeout,
)
} else {
(&self.short_client, "short", self.short_timeout)
}
@@ -200,9 +205,7 @@ fn header_value_opt(headers: &HeaderMap, name: &str) -> Option<String> {
}
fn uses_large_body_timeout(uri: &str) -> bool {
uri.starts_with("https://")
&& uri.ends_with(".xml")
&& !uri.ends_with("notification.xml")
uri.starts_with("https://") && uri.ends_with(".xml") && !uri.ends_with("notification.xml")
}
#[cfg(test)]
@@ -280,13 +283,17 @@ mod tests {
#[test]
fn uses_large_body_timeout_selects_rrdp_snapshot_and_delta_not_notification() {
assert!(!uses_large_body_timeout("https://rrdp.example.test/notification.xml"));
assert!(!uses_large_body_timeout(
"https://rrdp.example.test/notification.xml"
));
assert!(uses_large_body_timeout(
"https://rrdp.example.test/session/123/snapshot.xml"
));
assert!(uses_large_body_timeout(
"https://rrdp.example.test/session/123/delta-42.xml"
));
assert!(!uses_large_body_timeout("https://tal.example.test/example.tal"));
assert!(!uses_large_body_timeout(
"https://tal.example.test/example.tal"
));
}
}