NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260401 live recorder 扩展到1+N个delta，完成5个RIR录制，以及三方replay，结果三方均对齐vrps和vaps

Browse Source
This commit is contained in:
yuyr
2026-04-03 16:44:27 +08:00
parent 6edc420ce2
commit 34fb9657f1
27 changed files with 4641 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

199
src/fetch/http.rs
View File

@@ -1,12 +1,16 @@
use std::time::Duration;
use reqwest::blocking::Client;
use reqwest::header::HeaderMap;
use crate::sync::rrdp::Fetcher;
#[derive(Clone, Debug)]
pub struct HttpFetcherConfig {
/// Short timeout used for connection establishment and small metadata objects.
pub timeout: Duration,
/// Larger timeout used for RRDP snapshot / delta bodies.
pub large_body_timeout: Duration,
pub user_agent: String,
}
@@ -14,6 +18,7 @@ impl Default for HttpFetcherConfig {
fn default() -> Self {
Self {
timeout: Duration::from_secs(20),
large_body_timeout: Duration::from_secs(180),
user_agent: "rpki-dev/0.1 (stage2)".to_string(),
}
}
@@ -26,34 +31,154 @@ impl Default for HttpFetcherConfig {
/// - fetching RRDP notification/snapshot files (RFC 8182 §3.4)
#[derive(Clone, Debug)]
pub struct BlockingHttpFetcher {
client: Client,
short_client: Client,
large_body_client: Client,
short_timeout: Duration,
large_body_timeout: Duration,
}
impl BlockingHttpFetcher {
pub fn new(config: HttpFetcherConfig) -> Result<Self, String> {
let client = Client::builder()
let short_timeout = config.timeout;
let large_body_timeout = std::cmp::max(config.large_body_timeout, config.timeout);
let short_client = Client::builder()
.connect_timeout(config.timeout)
.timeout(config.timeout)
.user_agent(config.user_agent.clone())
.build()
.map_err(|e| e.to_string())?;
let large_body_client = Client::builder()
.connect_timeout(config.timeout)
.timeout(large_body_timeout)
.user_agent(config.user_agent)
.build()
.map_err(|e| e.to_string())?;
Ok(Self { client })
Ok(Self {
short_client,
large_body_client,
short_timeout,
large_body_timeout,
})
}
pub fn fetch_bytes(&self, uri: &str) -> Result<Vec<u8>, String> {
let resp = self
.client
let started = std::time::Instant::now();
let (client, timeout_profile, timeout_value) = self.client_for_uri(uri);
let resp = client
.get(uri)
.send()
.map_err(|e| format!("http request failed: {e}"))?;
.map_err(|e| {
let msg = format!("http request failed: {e}");
crate::progress_log::emit(
"http_fetch_failed",
serde_json::json!({
"uri": uri,
"stage": "request",
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": started.elapsed().as_millis() as u64,
"error": msg,
}),
);
msg
})?;
let status = resp.status();
let headers = resp.headers().clone();
if !status.is_success() {
return Err(format!("http status {status}"));
let body_preview = resp
.text()
.ok()
.map(|text| text.chars().take(160).collect::<String>());
let body_prefix = body_preview
.clone()
.unwrap_or_else(|| "<unavailable>".to_string());
let msg = format!(
"http status {status}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}; body_prefix={}",
header_value(&headers, "content-type"),
header_value(&headers, "content-encoding"),
header_value(&headers, "content-length"),
header_value(&headers, "transfer-encoding"),
body_prefix,
);
crate::progress_log::emit(
"http_fetch_failed",
serde_json::json!({
"uri": uri,
"stage": "status",
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": started.elapsed().as_millis() as u64,
"status": status.as_u16(),
"content_type": header_value_opt(&headers, "content-type"),
"content_encoding": header_value_opt(&headers, "content-encoding"),
"content_length": header_value_opt(&headers, "content-length"),
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
"body_prefix": body_preview,
"error": msg,
}),
);
return Err(msg);
}
resp.bytes()
.map(|b| b.to_vec())
.map_err(|e| format!("http read body failed: {e}"))
match resp.bytes() {
Ok(bytes) => {
let duration_ms = started.elapsed().as_millis() as u64;
if (duration_ms as f64) / 1000.0 >= crate::progress_log::slow_threshold_secs() {
crate::progress_log::emit(
"http_fetch_slow",
serde_json::json!({
"uri": uri,
"status": status.as_u16(),
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": duration_ms,
"bytes": bytes.len(),
"content_type": header_value_opt(&headers, "content-type"),
"content_encoding": header_value_opt(&headers, "content-encoding"),
"content_length": header_value_opt(&headers, "content-length"),
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
}),
);
}
Ok(bytes.to_vec())
}
Err(e) => {
let msg = format!(
"http read body failed: {e}; status={}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}",
status,
header_value(&headers, "content-type"),
header_value(&headers, "content-encoding"),
header_value(&headers, "content-length"),
header_value(&headers, "transfer-encoding"),
);
crate::progress_log::emit(
"http_fetch_failed",
serde_json::json!({
"uri": uri,
"stage": "read_body",
"timeout_profile": timeout_profile,
"request_timeout_ms": timeout_value.as_millis() as u64,
"duration_ms": started.elapsed().as_millis() as u64,
"status": status.as_u16(),
"content_type": header_value_opt(&headers, "content-type"),
"content_encoding": header_value_opt(&headers, "content-encoding"),
"content_length": header_value_opt(&headers, "content-length"),
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
"error": msg,
}),
);
Err(msg)
}
}
}
fn client_for_uri(&self, uri: &str) -> (&Client, &'static str, Duration) {
if uses_large_body_timeout(uri) {
(&self.large_body_client, "large_body", self.large_body_timeout)
} else {
(&self.short_client, "short", self.short_timeout)
}
}
}
@@ -63,12 +188,30 @@ impl Fetcher for BlockingHttpFetcher {
}
}
fn header_value(headers: &HeaderMap, name: &str) -> String {
header_value_opt(headers, name).unwrap_or_else(|| "<none>".to_string())
}
fn header_value_opt(headers: &HeaderMap, name: &str) -> Option<String> {
headers
.get(name)
.and_then(|v| v.to_str().ok())
.map(|v| v.to_string())
}
fn uses_large_body_timeout(uri: &str) -> bool {
uri.starts_with("https://")
&& uri.ends_with(".xml")
&& !uri.ends_with("notification.xml")
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::{Read, Write};
use std::net::TcpListener;
use std::thread;
use std::time::Duration as StdDuration;
fn spawn_one_shot_http_server(status_line: &'static str, body: &'static [u8]) -> String {
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
@@ -110,4 +253,40 @@ mod tests {
let err = http.fetch_bytes(&url).unwrap_err();
assert!(err.contains("http status"), "{err}");
}
#[test]
fn fetch_bytes_times_out_on_idle_body_read() {
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
let addr = listener.local_addr().expect("addr");
thread::spawn(move || {
let (mut stream, _) = listener.accept().expect("accept");
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf);
stream
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nh")
.expect("write partial body");
std::thread::sleep(StdDuration::from_secs(2));
let _ = stream.write_all(b"ello");
});
let url = format!("http://{}/", addr);
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
timeout: Duration::from_secs(1),
..HttpFetcherConfig::default()
})
.expect("http");
let err = http.fetch_bytes(&url).unwrap_err();
assert!(err.contains("http read body failed"), "{err}");
}
#[test]
fn uses_large_body_timeout_selects_rrdp_snapshot_and_delta_not_notification() {
assert!(!uses_large_body_timeout("https://rrdp.example.test/notification.xml"));
assert!(uses_large_body_timeout(
"https://rrdp.example.test/session/123/snapshot.xml"
));
assert!(uses_large_body_timeout(
"https://rrdp.example.test/session/123/delta-42.xml"
));
assert!(!uses_large_body_timeout("https://tal.example.test/example.tal"));
}
}