NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260411 apply snapahot内存优化，采用流式写文件和分块处理降低运行内存需求

Browse Source
This commit is contained in:
yuyr
2026-04-11 14:45:08 +08:00
parent 77fc2f1a41
commit e45830d79f
3 changed files with 523 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
src/fetch/http.rs
View File

@@ -1,3 +1,4 @@
use std::io::Write;
use std::time::Duration;
use reqwest::blocking::Client;
@@ -191,6 +192,116 @@ impl Fetcher for BlockingHttpFetcher {
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
self.fetch_bytes(uri)
}
fn fetch_to_writer(&self, uri: &str, out: &mut dyn Write) -> Result<u64, String> {
let started = std::time::Instant::now();
let (client, timeout_profile, timeout_value) = self.client_for_uri(uri);
let resp = client.get(uri).send().map_err(|e| {
let msg = format!("http request failed: {e}");
crate::progress_log::emit(
"http_fetch_failed",
serde_json::json!({
"uri": uri,
"stage": "request",
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": started.elapsed().as_millis() as u64,
"error": msg,
}),
);
msg
})?;
let status = resp.status();
let headers = resp.headers().clone();
if !status.is_success() {
let body_preview = resp
.text()
.ok()
.map(|text| text.chars().take(160).collect::<String>());
let body_prefix = body_preview
.clone()
.unwrap_or_else(|| "<unavailable>".to_string());
let msg = format!(
"http status {status}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}; body_prefix={}",
header_value(&headers, "content-type"),
header_value(&headers, "content-encoding"),
header_value(&headers, "content-length"),
header_value(&headers, "transfer-encoding"),
body_prefix,
);
crate::progress_log::emit(
"http_fetch_failed",
serde_json::json!({
"uri": uri,
"stage": "status",
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": started.elapsed().as_millis() as u64,
"status": status.as_u16(),
"content_type": header_value_opt(&headers, "content-type"),
"content_encoding": header_value_opt(&headers, "content-encoding"),
"content_length": header_value_opt(&headers, "content-length"),
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
"body_prefix": body_preview,
"error": msg,
}),
);
return Err(msg);
}
let mut resp = resp;
match resp.copy_to(out) {
Ok(bytes) => {
let duration_ms = started.elapsed().as_millis() as u64;
if (duration_ms as f64) / 1000.0 >= crate::progress_log::slow_threshold_secs() {
crate::progress_log::emit(
"http_fetch_slow",
serde_json::json!({
"uri": uri,
"status": status.as_u16(),
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": duration_ms,
"bytes": bytes,
"content_type": header_value_opt(&headers, "content-type"),
"content_encoding": header_value_opt(&headers, "content-encoding"),
"content_length": header_value_opt(&headers, "content-length"),
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
}),
);
}
Ok(bytes)
}
Err(e) => {
let msg = format!(
"http stream body failed: {e}; status={}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}",
status,
header_value(&headers, "content-type"),
header_value(&headers, "content-encoding"),
header_value(&headers, "content-length"),
header_value(&headers, "transfer-encoding"),
);
crate::progress_log::emit(
"http_fetch_failed",
serde_json::json!({
"uri": uri,
"stage": "stream_body",
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": started.elapsed().as_millis() as u64,
"status": status.as_u16(),
"content_type": header_value_opt(&headers, "content-type"),
"content_encoding": header_value_opt(&headers, "content-encoding"),
"content_length": header_value_opt(&headers, "content-length"),
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
"error": msg,
}),
);
Err(msg)
}
}
}
}
fn header_value(headers: &HeaderMap, name: &str) -> String {