923 lines
36 KiB
Rust
923 lines
36 KiB
Rust
use std::cell::RefCell;
|
|
use std::io::Write;
|
|
use std::time::Duration;
|
|
|
|
use reqwest::blocking::Client;
|
|
use reqwest::header::{HeaderMap, LOCATION};
|
|
use url::Url;
|
|
|
|
use crate::repository::sync::rrdp::{
|
|
Fetcher, RrdpError, RrdpFetchError, RrdpOrigin, RrdpResourceKind,
|
|
};
|
|
|
|
const MAX_RRDP_REDIRECTS: usize = 10;
|
|
|
|
thread_local! {
|
|
static HTTP_TIMEOUT_OVERRIDE: RefCell<Option<Duration>> = const { RefCell::new(None) };
|
|
}
|
|
|
|
pub fn with_scoped_http_timeout_override<R>(timeout: Duration, f: impl FnOnce() -> R) -> R {
|
|
HTTP_TIMEOUT_OVERRIDE.with(|cell| {
|
|
let previous = cell.replace(Some(timeout));
|
|
let result = f();
|
|
let _ = cell.replace(previous);
|
|
result
|
|
})
|
|
}
|
|
|
|
/// Default User-Agent sent with every outgoing HTTP request (RRDP
|
|
/// notification/snapshot/delta and TAL/TA downloads).
|
|
pub const DEFAULT_HTTP_USER_AGENT: &str = "panda-rpki/0.2";
|
|
|
|
/// Environment variable that overrides the HTTP User-Agent. Unset, empty, or
|
|
/// values containing characters that are illegal in a header value fall back
|
|
/// to [`DEFAULT_HTTP_USER_AGENT`].
|
|
pub const HTTP_USER_AGENT_ENV: &str = "RPKI_HTTP_USER_AGENT";
|
|
|
|
fn resolve_http_user_agent(env_value: Option<String>) -> String {
|
|
let Some(value) = env_value else {
|
|
return DEFAULT_HTTP_USER_AGENT.to_string();
|
|
};
|
|
let trimmed = value.trim();
|
|
let valid = !trimmed.is_empty() && trimmed.bytes().all(|b| (0x20..=0x7e).contains(&b));
|
|
if valid {
|
|
trimmed.to_string()
|
|
} else {
|
|
DEFAULT_HTTP_USER_AGENT.to_string()
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug)]
|
|
pub struct HttpFetcherConfig {
|
|
/// Connection-establishment timeout for HTTP requests.
|
|
pub connect_timeout: Duration,
|
|
/// Short timeout used for connection establishment and small metadata objects.
|
|
pub timeout: Duration,
|
|
/// Larger timeout used for RRDP snapshot / delta bodies.
|
|
pub large_body_timeout: Duration,
|
|
pub user_agent: String,
|
|
/// Extra PEM trust anchors for HTTPS transport tests or controlled endpoints.
|
|
pub extra_root_certificates_pem: Vec<Vec<u8>>,
|
|
}
|
|
|
|
impl Default for HttpFetcherConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
connect_timeout: Duration::from_secs(15),
|
|
timeout: Duration::from_secs(30),
|
|
large_body_timeout: Duration::from_secs(180),
|
|
user_agent: resolve_http_user_agent(std::env::var(HTTP_USER_AGENT_ENV).ok()),
|
|
extra_root_certificates_pem: Vec::new(),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Minimal blocking HTTP(S) fetcher for validation runs.
|
|
///
|
|
/// This is used for:
|
|
/// - downloading TAL / TA certificates (RFC 8630 §2)
|
|
/// - fetching RRDP notification/snapshot files (RFC 8182 §3.4)
|
|
#[derive(Clone, Debug)]
|
|
pub struct BlockingHttpFetcher {
|
|
short_client: Client,
|
|
large_body_client: Client,
|
|
retry_short_client: Client,
|
|
rrdp_short_client: Client,
|
|
rrdp_large_body_client: Client,
|
|
rrdp_retry_short_client: Client,
|
|
short_timeout: Duration,
|
|
large_body_timeout: Duration,
|
|
}
|
|
|
|
impl BlockingHttpFetcher {
|
|
pub fn new(config: HttpFetcherConfig) -> Result<Self, String> {
|
|
let short_timeout = config.timeout;
|
|
let large_body_timeout = std::cmp::max(config.large_body_timeout, config.timeout);
|
|
let connect_timeout = std::cmp::min(config.connect_timeout, config.timeout);
|
|
let short_client = Self::client_builder(
|
|
&config,
|
|
connect_timeout,
|
|
config.timeout,
|
|
config.user_agent.clone(),
|
|
)?
|
|
.build()
|
|
.map_err(|e| e.to_string())?;
|
|
let large_body_client = Self::client_builder(
|
|
&config,
|
|
connect_timeout,
|
|
large_body_timeout,
|
|
config.user_agent.clone(),
|
|
)?
|
|
.build()
|
|
.map_err(|e| e.to_string())?;
|
|
let retry_short_client = Self::client_builder(
|
|
&config,
|
|
Duration::from_secs(1),
|
|
Duration::from_secs(1),
|
|
config.user_agent.clone(),
|
|
)?
|
|
.build()
|
|
.map_err(|e| e.to_string())?;
|
|
let rrdp_short_client = Self::rrdp_client_builder(
|
|
&config,
|
|
connect_timeout,
|
|
config.timeout,
|
|
config.user_agent.clone(),
|
|
)?
|
|
.build()
|
|
.map_err(|e| e.to_string())?;
|
|
let rrdp_large_body_client = Self::rrdp_client_builder(
|
|
&config,
|
|
connect_timeout,
|
|
large_body_timeout,
|
|
config.user_agent.clone(),
|
|
)?
|
|
.build()
|
|
.map_err(|e| e.to_string())?;
|
|
let rrdp_retry_short_client = Self::rrdp_client_builder(
|
|
&config,
|
|
Duration::from_secs(1),
|
|
Duration::from_secs(1),
|
|
config.user_agent.clone(),
|
|
)?
|
|
.build()
|
|
.map_err(|e| e.to_string())?;
|
|
Ok(Self {
|
|
short_client,
|
|
large_body_client,
|
|
retry_short_client,
|
|
rrdp_short_client,
|
|
rrdp_large_body_client,
|
|
rrdp_retry_short_client,
|
|
short_timeout,
|
|
large_body_timeout,
|
|
})
|
|
}
|
|
|
|
fn client_builder(
|
|
config: &HttpFetcherConfig,
|
|
connect_timeout: Duration,
|
|
timeout: Duration,
|
|
user_agent: String,
|
|
) -> Result<reqwest::blocking::ClientBuilder, String> {
|
|
let mut builder = Client::builder()
|
|
.connect_timeout(connect_timeout)
|
|
.timeout(timeout)
|
|
.user_agent(user_agent);
|
|
for (idx, pem) in config.extra_root_certificates_pem.iter().enumerate() {
|
|
let certificate = reqwest::Certificate::from_pem(pem)
|
|
.map_err(|e| format!("parse HTTP root certificate #{idx} failed: {e}"))?;
|
|
builder = builder.add_root_certificate(certificate);
|
|
}
|
|
Ok(builder)
|
|
}
|
|
|
|
fn rrdp_client_builder(
|
|
config: &HttpFetcherConfig,
|
|
connect_timeout: Duration,
|
|
timeout: Duration,
|
|
user_agent: String,
|
|
) -> Result<reqwest::blocking::ClientBuilder, String> {
|
|
Ok(
|
|
Self::client_builder(config, connect_timeout, timeout, user_agent)?
|
|
.redirect(reqwest::redirect::Policy::none()),
|
|
)
|
|
}
|
|
|
|
pub fn fetch_bytes(&self, uri: &str) -> Result<Vec<u8>, String> {
|
|
let started = std::time::Instant::now();
|
|
let (client, timeout_profile, timeout_value) = self.client_for_uri(uri);
|
|
let resp = client.get(uri).send().map_err(|e| {
|
|
let msg = format!("http request failed: {e:?}");
|
|
crate::logging::progress::emit!(
|
|
"http_fetch_failed",
|
|
serde_json::json!({
|
|
"uri": uri,
|
|
"stage": "request",
|
|
"timeout_profile": timeout_profile,
|
|
"request_timeout_ms": timeout_value.as_millis() as u64,
|
|
"duration_ms": started.elapsed().as_millis() as u64,
|
|
"error": msg,
|
|
}),
|
|
);
|
|
msg
|
|
})?;
|
|
|
|
let status = resp.status();
|
|
let headers = resp.headers().clone();
|
|
if !status.is_success() {
|
|
let msg = format!(
|
|
"http status {status}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}",
|
|
header_value(&headers, "content-type"),
|
|
header_value(&headers, "content-encoding"),
|
|
header_value(&headers, "content-length"),
|
|
header_value(&headers, "transfer-encoding"),
|
|
);
|
|
crate::logging::progress::emit!(
|
|
"http_fetch_failed",
|
|
serde_json::json!({
|
|
"uri": uri,
|
|
"stage": "status",
|
|
"timeout_profile": timeout_profile,
|
|
"request_timeout_ms": timeout_value.as_millis() as u64,
|
|
"duration_ms": started.elapsed().as_millis() as u64,
|
|
"status": status.as_u16(),
|
|
"content_type": header_value_opt(&headers, "content-type"),
|
|
"content_encoding": header_value_opt(&headers, "content-encoding"),
|
|
"content_length": header_value_opt(&headers, "content-length"),
|
|
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
|
|
"error": msg,
|
|
}),
|
|
);
|
|
return Err(msg);
|
|
}
|
|
|
|
match resp.bytes() {
|
|
Ok(bytes) => {
|
|
let duration_ms = started.elapsed().as_millis() as u64;
|
|
if (duration_ms as f64) / 1000.0 >= crate::logging::progress::slow_threshold_secs()
|
|
{
|
|
crate::logging::progress::emit!(
|
|
"http_fetch_slow",
|
|
serde_json::json!({
|
|
"uri": uri,
|
|
"status": status.as_u16(),
|
|
"timeout_profile": timeout_profile,
|
|
"request_timeout_ms": timeout_value.as_millis() as u64,
|
|
"duration_ms": duration_ms,
|
|
"bytes": bytes.len(),
|
|
"content_type": header_value_opt(&headers, "content-type"),
|
|
"content_encoding": header_value_opt(&headers, "content-encoding"),
|
|
"content_length": header_value_opt(&headers, "content-length"),
|
|
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
|
|
}),
|
|
);
|
|
}
|
|
Ok(bytes.to_vec())
|
|
}
|
|
Err(e) => {
|
|
let msg = format!(
|
|
"http read body failed: {e}; status={}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}",
|
|
status,
|
|
header_value(&headers, "content-type"),
|
|
header_value(&headers, "content-encoding"),
|
|
header_value(&headers, "content-length"),
|
|
header_value(&headers, "transfer-encoding"),
|
|
);
|
|
crate::logging::progress::emit!(
|
|
"http_fetch_failed",
|
|
serde_json::json!({
|
|
"uri": uri,
|
|
"stage": "read_body",
|
|
"timeout_profile": timeout_profile,
|
|
"request_timeout_ms": timeout_value.as_millis() as u64,
|
|
"duration_ms": started.elapsed().as_millis() as u64,
|
|
"status": status.as_u16(),
|
|
"content_type": header_value_opt(&headers, "content-type"),
|
|
"content_encoding": header_value_opt(&headers, "content-encoding"),
|
|
"content_length": header_value_opt(&headers, "content-length"),
|
|
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
|
|
"error": msg,
|
|
}),
|
|
);
|
|
Err(msg)
|
|
}
|
|
}
|
|
}
|
|
|
|
fn client_for_uri(&self, uri: &str) -> (&Client, &'static str, Duration) {
|
|
if let Some(timeout) = HTTP_TIMEOUT_OVERRIDE.with(|cell| *cell.borrow()) {
|
|
return (&self.retry_short_client, "retry_short", timeout);
|
|
}
|
|
if uses_large_body_timeout(uri) {
|
|
(
|
|
&self.large_body_client,
|
|
"large_body",
|
|
self.large_body_timeout,
|
|
)
|
|
} else {
|
|
(&self.short_client, "short", self.short_timeout)
|
|
}
|
|
}
|
|
|
|
fn rrdp_client_for_uri(&self, uri: &str) -> (&Client, &'static str, Duration) {
|
|
if let Some(timeout) = HTTP_TIMEOUT_OVERRIDE.with(|cell| *cell.borrow()) {
|
|
return (&self.rrdp_retry_short_client, "retry_short", timeout);
|
|
}
|
|
if uses_large_body_timeout(uri) {
|
|
(
|
|
&self.rrdp_large_body_client,
|
|
"large_body",
|
|
self.large_body_timeout,
|
|
)
|
|
} else {
|
|
(&self.rrdp_short_client, "short", self.short_timeout)
|
|
}
|
|
}
|
|
|
|
fn rrdp_response(
|
|
&self,
|
|
resource: RrdpResourceKind,
|
|
uri: &str,
|
|
notification_origin: &RrdpOrigin,
|
|
) -> Result<(reqwest::blocking::Response, &'static str, Duration), RrdpFetchError> {
|
|
let mut current = Url::parse(uri).map_err(|e| RrdpError::InvalidRrdpUri {
|
|
resource,
|
|
detail: e.to_string(),
|
|
})?;
|
|
let current_origin = RrdpOrigin::from_url(¤t, resource)?;
|
|
if ¤t_origin != notification_origin {
|
|
return Err(RrdpError::CrossOriginReference {
|
|
resource,
|
|
notification_origin: notification_origin.to_string(),
|
|
resource_origin: current_origin.to_string(),
|
|
}
|
|
.into());
|
|
}
|
|
|
|
for redirects in 0..=MAX_RRDP_REDIRECTS {
|
|
let (client, profile, timeout) = self.rrdp_client_for_uri(current.as_str());
|
|
let response = client
|
|
.get(current.clone())
|
|
.send()
|
|
.map_err(|e| RrdpFetchError::Fetch(format!("http request failed: {e:?}")))?;
|
|
if !response.status().is_redirection() {
|
|
return Ok((response, profile, timeout));
|
|
}
|
|
if redirects == MAX_RRDP_REDIRECTS {
|
|
return Err(RrdpError::RedirectLimitExceeded {
|
|
resource,
|
|
max: MAX_RRDP_REDIRECTS,
|
|
}
|
|
.into());
|
|
}
|
|
let location = response
|
|
.headers()
|
|
.get(LOCATION)
|
|
.ok_or(RrdpError::RedirectLocationMissing { resource })?
|
|
.to_str()
|
|
.map_err(|_| RrdpError::InvalidRedirectLocation {
|
|
resource,
|
|
location: "<non-UTF-8 header>".to_string(),
|
|
})?;
|
|
let next = current
|
|
.join(location)
|
|
.map_err(|_| RrdpError::InvalidRedirectLocation {
|
|
resource,
|
|
location: location.to_string(),
|
|
})?;
|
|
let next_origin = RrdpOrigin::from_url(&next, resource)?;
|
|
crate::logging::progress::emit!(
|
|
"rrdp_redirect_checked",
|
|
serde_json::json!({
|
|
"resource": resource.to_string(),
|
|
"from": current.as_str(),
|
|
"to": next.as_str(),
|
|
"notification_origin": notification_origin.to_string(),
|
|
"same_origin": &next_origin == notification_origin,
|
|
"rfc": "RFC 9674 §3.2",
|
|
}),
|
|
);
|
|
if &next_origin != notification_origin {
|
|
crate::logging::progress::emit!(
|
|
"rrdp_cross_origin_rejected",
|
|
serde_json::json!({
|
|
"resource": resource.to_string(),
|
|
"notification_origin": notification_origin.to_string(),
|
|
"resource_origin": next_origin.to_string(),
|
|
"reason": "redirect",
|
|
"rfc": "RFC 9674 §3.2",
|
|
}),
|
|
);
|
|
return Err(RrdpError::CrossOriginRedirect {
|
|
resource,
|
|
notification_origin: notification_origin.to_string(),
|
|
redirect_origin: next_origin.to_string(),
|
|
}
|
|
.into());
|
|
}
|
|
current = next;
|
|
}
|
|
unreachable!("redirect loop either returns or reaches its limit")
|
|
}
|
|
|
|
fn rrdp_fetch_bytes(
|
|
&self,
|
|
resource: RrdpResourceKind,
|
|
uri: &str,
|
|
notification_origin: &RrdpOrigin,
|
|
) -> Result<Vec<u8>, RrdpFetchError> {
|
|
let (response, _profile, _timeout) =
|
|
self.rrdp_response(resource, uri, notification_origin)?;
|
|
let status = response.status();
|
|
if !status.is_success() {
|
|
return Err(RrdpFetchError::Fetch(format!("http status {status}")));
|
|
}
|
|
response
|
|
.bytes()
|
|
.map(|b| b.to_vec())
|
|
.map_err(|e| RrdpFetchError::Fetch(format!("http read body failed: {e}")))
|
|
}
|
|
|
|
fn rrdp_fetch_to_writer(
|
|
&self,
|
|
resource: RrdpResourceKind,
|
|
uri: &str,
|
|
notification_origin: &RrdpOrigin,
|
|
out: &mut dyn Write,
|
|
) -> Result<u64, RrdpFetchError> {
|
|
let (mut response, _profile, _timeout) =
|
|
self.rrdp_response(resource, uri, notification_origin)?;
|
|
let status = response.status();
|
|
if !status.is_success() {
|
|
return Err(RrdpFetchError::Fetch(format!("http status {status}")));
|
|
}
|
|
response
|
|
.copy_to(out)
|
|
.map_err(|e| RrdpFetchError::Fetch(format!("http stream body failed: {e}")))
|
|
}
|
|
}
|
|
|
|
impl Fetcher for BlockingHttpFetcher {
|
|
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
|
|
self.fetch_bytes(uri)
|
|
}
|
|
|
|
fn fetch_to_writer(&self, uri: &str, out: &mut dyn Write) -> Result<u64, String> {
|
|
let started = std::time::Instant::now();
|
|
let (client, timeout_profile, timeout_value) = self.client_for_uri(uri);
|
|
let resp = client.get(uri).send().map_err(|e| {
|
|
let msg = format!("http request failed: {e:?}");
|
|
crate::logging::progress::emit!(
|
|
"http_fetch_failed",
|
|
serde_json::json!({
|
|
"uri": uri,
|
|
"stage": "request",
|
|
"timeout_profile": timeout_profile,
|
|
"request_timeout_ms": timeout_value.as_millis() as u64,
|
|
"duration_ms": started.elapsed().as_millis() as u64,
|
|
"error": msg,
|
|
}),
|
|
);
|
|
msg
|
|
})?;
|
|
|
|
let status = resp.status();
|
|
let headers = resp.headers().clone();
|
|
if !status.is_success() {
|
|
let msg = format!(
|
|
"http status {status}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}",
|
|
header_value(&headers, "content-type"),
|
|
header_value(&headers, "content-encoding"),
|
|
header_value(&headers, "content-length"),
|
|
header_value(&headers, "transfer-encoding"),
|
|
);
|
|
crate::logging::progress::emit!(
|
|
"http_fetch_failed",
|
|
serde_json::json!({
|
|
"uri": uri,
|
|
"stage": "status",
|
|
"timeout_profile": timeout_profile,
|
|
"request_timeout_ms": timeout_value.as_millis() as u64,
|
|
"duration_ms": started.elapsed().as_millis() as u64,
|
|
"status": status.as_u16(),
|
|
"content_type": header_value_opt(&headers, "content-type"),
|
|
"content_encoding": header_value_opt(&headers, "content-encoding"),
|
|
"content_length": header_value_opt(&headers, "content-length"),
|
|
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
|
|
"error": msg,
|
|
}),
|
|
);
|
|
return Err(msg);
|
|
}
|
|
|
|
let mut resp = resp;
|
|
match resp.copy_to(out) {
|
|
Ok(bytes) => {
|
|
let duration_ms = started.elapsed().as_millis() as u64;
|
|
if (duration_ms as f64) / 1000.0 >= crate::logging::progress::slow_threshold_secs()
|
|
{
|
|
crate::logging::progress::emit!(
|
|
"http_fetch_slow",
|
|
serde_json::json!({
|
|
"uri": uri,
|
|
"status": status.as_u16(),
|
|
"timeout_profile": timeout_profile,
|
|
"request_timeout_ms": timeout_value.as_millis() as u64,
|
|
"duration_ms": duration_ms,
|
|
"bytes": bytes,
|
|
"content_type": header_value_opt(&headers, "content-type"),
|
|
"content_encoding": header_value_opt(&headers, "content-encoding"),
|
|
"content_length": header_value_opt(&headers, "content-length"),
|
|
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
|
|
}),
|
|
);
|
|
}
|
|
Ok(bytes)
|
|
}
|
|
Err(e) => {
|
|
let msg = format!(
|
|
"http stream body failed: {e}; status={}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}",
|
|
status,
|
|
header_value(&headers, "content-type"),
|
|
header_value(&headers, "content-encoding"),
|
|
header_value(&headers, "content-length"),
|
|
header_value(&headers, "transfer-encoding"),
|
|
);
|
|
crate::logging::progress::emit!(
|
|
"http_fetch_failed",
|
|
serde_json::json!({
|
|
"uri": uri,
|
|
"stage": "stream_body",
|
|
"timeout_profile": timeout_profile,
|
|
"request_timeout_ms": timeout_value.as_millis() as u64,
|
|
"duration_ms": started.elapsed().as_millis() as u64,
|
|
"status": status.as_u16(),
|
|
"content_type": header_value_opt(&headers, "content-type"),
|
|
"content_encoding": header_value_opt(&headers, "content-encoding"),
|
|
"content_length": header_value_opt(&headers, "content-length"),
|
|
"transfer_encoding": header_value_opt(&headers, "transfer-encoding"),
|
|
"error": msg,
|
|
}),
|
|
);
|
|
Err(msg)
|
|
}
|
|
}
|
|
}
|
|
|
|
fn fetch_rrdp(
|
|
&self,
|
|
resource: RrdpResourceKind,
|
|
uri: &str,
|
|
notification_origin: &RrdpOrigin,
|
|
) -> Result<Vec<u8>, RrdpFetchError> {
|
|
self.rrdp_fetch_bytes(resource, uri, notification_origin)
|
|
}
|
|
|
|
fn fetch_rrdp_to_writer(
|
|
&self,
|
|
resource: RrdpResourceKind,
|
|
uri: &str,
|
|
notification_origin: &RrdpOrigin,
|
|
out: &mut dyn Write,
|
|
) -> Result<u64, RrdpFetchError> {
|
|
self.rrdp_fetch_to_writer(resource, uri, notification_origin, out)
|
|
}
|
|
}
|
|
|
|
fn header_value(headers: &HeaderMap, name: &str) -> String {
|
|
header_value_opt(headers, name).unwrap_or_else(|| "<none>".to_string())
|
|
}
|
|
|
|
fn header_value_opt(headers: &HeaderMap, name: &str) -> Option<String> {
|
|
headers
|
|
.get(name)
|
|
.and_then(|v| v.to_str().ok())
|
|
.map(|v| v.to_string())
|
|
}
|
|
|
|
fn uses_large_body_timeout(uri: &str) -> bool {
|
|
uri.starts_with("https://") && uri.ends_with(".xml") && !uri.ends_with("notification.xml")
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::io::{Read, Write};
|
|
use std::net::TcpListener;
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
use std::thread;
|
|
use std::time::Duration as StdDuration;
|
|
|
|
fn spawn_one_shot_http_server(status_line: &'static str, body: &'static [u8]) -> String {
|
|
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
|
|
let addr = listener.local_addr().expect("addr");
|
|
thread::spawn(move || {
|
|
let (mut stream, _) = listener.accept().expect("accept");
|
|
let mut buf = [0u8; 1024];
|
|
let _ = stream.read(&mut buf);
|
|
let hdr = format!(
|
|
"{status_line}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
|
body.len()
|
|
);
|
|
stream.write_all(hdr.as_bytes()).expect("write hdr");
|
|
stream.write_all(body).expect("write body");
|
|
});
|
|
format!("http://{}/", addr)
|
|
}
|
|
|
|
#[test]
|
|
fn fetch_bytes_returns_body_on_success() {
|
|
let url = spawn_one_shot_http_server("HTTP/1.1 200 OK", b"hello");
|
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
|
timeout: Duration::from_secs(2),
|
|
..HttpFetcherConfig::default()
|
|
})
|
|
.expect("http");
|
|
let got = http.fetch_bytes(&url).expect("fetch");
|
|
assert_eq!(got, b"hello");
|
|
}
|
|
|
|
#[test]
|
|
fn fetch_bytes_rejects_non_success_status() {
|
|
let url = spawn_one_shot_http_server("HTTP/1.1 404 Not Found", b"");
|
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
|
timeout: Duration::from_secs(2),
|
|
..HttpFetcherConfig::default()
|
|
})
|
|
.expect("http");
|
|
let err = http.fetch_bytes(&url).unwrap_err();
|
|
assert!(err.contains("http status"), "{err}");
|
|
}
|
|
|
|
#[test]
|
|
fn fetch_bytes_times_out_on_idle_body_read() {
|
|
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
|
|
let addr = listener.local_addr().expect("addr");
|
|
thread::spawn(move || {
|
|
let (mut stream, _) = listener.accept().expect("accept");
|
|
let mut buf = [0u8; 1024];
|
|
let _ = stream.read(&mut buf);
|
|
stream
|
|
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nh")
|
|
.expect("write partial body");
|
|
std::thread::sleep(StdDuration::from_secs(2));
|
|
let _ = stream.write_all(b"ello");
|
|
});
|
|
let url = format!("http://{}/", addr);
|
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
|
timeout: Duration::from_secs(1),
|
|
..HttpFetcherConfig::default()
|
|
})
|
|
.expect("http");
|
|
let err = http.fetch_bytes(&url).unwrap_err();
|
|
assert!(err.contains("http read body failed"), "{err}");
|
|
}
|
|
|
|
#[test]
|
|
fn fetch_to_writer_streams_body_on_success() {
|
|
let url = spawn_one_shot_http_server("HTTP/1.1 200 OK", b"writer-body");
|
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
|
timeout: Duration::from_secs(2),
|
|
..HttpFetcherConfig::default()
|
|
})
|
|
.expect("http");
|
|
let mut out = Vec::new();
|
|
let bytes = http.fetch_to_writer(&url, &mut out).expect("stream");
|
|
assert_eq!(bytes, 11);
|
|
assert_eq!(out, b"writer-body");
|
|
}
|
|
|
|
#[test]
|
|
fn fetch_to_writer_rejects_non_success_status() {
|
|
let url = spawn_one_shot_http_server("HTTP/1.1 500 Internal Server Error", b"boom");
|
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
|
timeout: Duration::from_secs(2),
|
|
..HttpFetcherConfig::default()
|
|
})
|
|
.expect("http");
|
|
let mut out = Vec::new();
|
|
let err = http.fetch_to_writer(&url, &mut out).unwrap_err();
|
|
assert!(err.contains("http status"), "{err}");
|
|
assert!(out.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn fetch_to_writer_times_out_on_idle_stream_read() {
|
|
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
|
|
let addr = listener.local_addr().expect("addr");
|
|
thread::spawn(move || {
|
|
let (mut stream, _) = listener.accept().expect("accept");
|
|
let mut buf = [0u8; 1024];
|
|
let _ = stream.read(&mut buf);
|
|
stream
|
|
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nh")
|
|
.expect("write partial body");
|
|
std::thread::sleep(StdDuration::from_secs(2));
|
|
let _ = stream.write_all(b"ello");
|
|
});
|
|
let url = format!("http://{}/", addr);
|
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
|
timeout: Duration::from_secs(1),
|
|
..HttpFetcherConfig::default()
|
|
})
|
|
.expect("http");
|
|
let mut out = Vec::new();
|
|
let err = http.fetch_to_writer(&url, &mut out).unwrap_err();
|
|
assert!(err.contains("http stream body failed"), "{err}");
|
|
}
|
|
|
|
#[test]
|
|
fn uses_large_body_timeout_selects_rrdp_snapshot_and_delta_not_notification() {
|
|
assert!(!uses_large_body_timeout(
|
|
"https://rrdp.example.test/notification.xml"
|
|
));
|
|
assert!(uses_large_body_timeout(
|
|
"https://rrdp.example.test/session/123/snapshot.xml"
|
|
));
|
|
assert!(uses_large_body_timeout(
|
|
"https://rrdp.example.test/session/123/delta-42.xml"
|
|
));
|
|
assert!(!uses_large_body_timeout(
|
|
"https://tal.example.test/example.tal"
|
|
));
|
|
}
|
|
|
|
#[test]
|
|
fn client_for_uri_selects_expected_timeout_profile() {
|
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
|
timeout: Duration::from_secs(3),
|
|
large_body_timeout: Duration::from_secs(9),
|
|
..HttpFetcherConfig::default()
|
|
})
|
|
.expect("http");
|
|
|
|
let (_, profile_short, timeout_short) =
|
|
http.client_for_uri("https://example.test/root.tal");
|
|
assert_eq!(profile_short, "short");
|
|
assert_eq!(timeout_short, Duration::from_secs(3));
|
|
|
|
let (_, profile_large, timeout_large) =
|
|
http.client_for_uri("https://rrdp.example.test/session/1/snapshot.xml");
|
|
assert_eq!(profile_large, "large_body");
|
|
assert_eq!(timeout_large, Duration::from_secs(9));
|
|
}
|
|
|
|
#[test]
|
|
fn resolve_http_user_agent_falls_back_to_default() {
|
|
assert_eq!(resolve_http_user_agent(None), DEFAULT_HTTP_USER_AGENT);
|
|
assert_eq!(
|
|
resolve_http_user_agent(Some(String::new())),
|
|
DEFAULT_HTTP_USER_AGENT
|
|
);
|
|
assert_eq!(
|
|
resolve_http_user_agent(Some(" ".to_string())),
|
|
DEFAULT_HTTP_USER_AGENT
|
|
);
|
|
// Control characters are illegal in header values; fall back instead
|
|
// of letting reqwest fail to build the client.
|
|
assert_eq!(
|
|
resolve_http_user_agent(Some("bad\u{1}ua".to_string())),
|
|
DEFAULT_HTTP_USER_AGENT
|
|
);
|
|
assert_eq!(
|
|
resolve_http_user_agent(Some("non-ascii-ua-中".to_string())),
|
|
DEFAULT_HTTP_USER_AGENT
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn resolve_http_user_agent_accepts_custom_value() {
|
|
assert_eq!(
|
|
resolve_http_user_agent(Some("panda-rpki/9.9 (test)".to_string())),
|
|
"panda-rpki/9.9 (test)"
|
|
);
|
|
// Surrounding whitespace is trimmed.
|
|
assert_eq!(
|
|
resolve_http_user_agent(Some(" custom/1.0 ".to_string())),
|
|
"custom/1.0"
|
|
);
|
|
}
|
|
|
|
fn spawn_user_agent_capture_server() -> (String, std::sync::mpsc::Receiver<String>) {
|
|
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
|
|
let addr = listener.local_addr().expect("addr");
|
|
let (tx, rx) = std::sync::mpsc::channel();
|
|
thread::spawn(move || {
|
|
let (mut stream, _) = listener.accept().expect("accept");
|
|
let mut buf = [0u8; 4096];
|
|
let read = stream.read(&mut buf).expect("read request");
|
|
let request = String::from_utf8_lossy(&buf[..read]).to_string();
|
|
let body = b"ok";
|
|
let hdr = format!(
|
|
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
|
body.len()
|
|
);
|
|
stream.write_all(hdr.as_bytes()).expect("write hdr");
|
|
stream.write_all(body).expect("write body");
|
|
let _ = tx.send(request);
|
|
});
|
|
(format!("http://{}/", addr), rx)
|
|
}
|
|
|
|
#[test]
|
|
fn fetch_bytes_sends_configured_user_agent_header() {
|
|
let (url, rx) = spawn_user_agent_capture_server();
|
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
|
timeout: Duration::from_secs(2),
|
|
user_agent: "panda-rpki/0.2".to_string(),
|
|
..HttpFetcherConfig::default()
|
|
})
|
|
.expect("http");
|
|
http.fetch_bytes(&url).expect("fetch");
|
|
let request = rx.recv().expect("request captured");
|
|
assert!(
|
|
request
|
|
.to_ascii_lowercase()
|
|
.contains("user-agent: panda-rpki/0.2"),
|
|
"{request}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn fetch_bytes_sends_custom_user_agent_header() {
|
|
let (url, rx) = spawn_user_agent_capture_server();
|
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
|
timeout: Duration::from_secs(2),
|
|
user_agent: "custom-marker/7.7".to_string(),
|
|
..HttpFetcherConfig::default()
|
|
})
|
|
.expect("http");
|
|
http.fetch_bytes(&url).expect("fetch");
|
|
let request = rx.recv().expect("request captured");
|
|
assert!(
|
|
request
|
|
.to_ascii_lowercase()
|
|
.contains("user-agent: custom-marker/7.7"),
|
|
"{request}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn rrdp_same_origin_redirect_is_followed_manually() {
|
|
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
|
|
let addr = listener.local_addr().expect("addr");
|
|
thread::spawn(move || {
|
|
for response in [
|
|
b"HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
|
.as_slice(),
|
|
b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok".as_slice(),
|
|
] {
|
|
let (mut stream, _) = listener.accept().expect("accept");
|
|
let mut buf = [0u8; 1024];
|
|
let _ = stream.read(&mut buf);
|
|
stream.write_all(response).expect("response");
|
|
}
|
|
});
|
|
let uri = format!("http://{addr}/start");
|
|
let origin = RrdpOrigin::parse(&uri, RrdpResourceKind::Notification).expect("origin");
|
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
|
timeout: Duration::from_secs(2),
|
|
..HttpFetcherConfig::default()
|
|
})
|
|
.expect("http");
|
|
let body = http
|
|
.fetch_rrdp(RrdpResourceKind::Notification, &uri, &origin)
|
|
.expect("same-origin redirect accepted");
|
|
assert_eq!(body, b"ok");
|
|
}
|
|
|
|
#[test]
|
|
fn rrdp_cross_origin_redirect_is_rejected_before_target_request() {
|
|
let target = TcpListener::bind(("127.0.0.1", 0)).expect("bind target");
|
|
target.set_nonblocking(true).expect("nonblocking target");
|
|
let target_addr = target.local_addr().expect("target addr");
|
|
let target_hits = Arc::new(AtomicUsize::new(0));
|
|
let target_hits_worker = Arc::clone(&target_hits);
|
|
let target_thread = thread::spawn(move || {
|
|
let deadline = std::time::Instant::now() + StdDuration::from_millis(400);
|
|
while std::time::Instant::now() < deadline {
|
|
match target.accept() {
|
|
Ok((mut stream, _)) => {
|
|
target_hits_worker.fetch_add(1, Ordering::SeqCst);
|
|
let _ = stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n");
|
|
}
|
|
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
|
|
std::thread::sleep(StdDuration::from_millis(5));
|
|
}
|
|
Err(e) => panic!("target accept: {e}"),
|
|
}
|
|
}
|
|
});
|
|
|
|
let source = TcpListener::bind(("127.0.0.1", 0)).expect("bind source");
|
|
let source_addr = source.local_addr().expect("source addr");
|
|
thread::spawn(move || {
|
|
let (mut stream, _) = source.accept().expect("accept source");
|
|
let mut buf = [0u8; 1024];
|
|
let _ = stream.read(&mut buf);
|
|
let response = format!(
|
|
"HTTP/1.1 302 Found\r\nLocation: http://{target_addr}/foreign\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
|
);
|
|
stream.write_all(response.as_bytes()).expect("redirect");
|
|
});
|
|
let uri = format!("http://{source_addr}/start");
|
|
let origin = RrdpOrigin::parse(&uri, RrdpResourceKind::Notification).expect("origin");
|
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
|
timeout: Duration::from_secs(2),
|
|
..HttpFetcherConfig::default()
|
|
})
|
|
.expect("http");
|
|
let err = http
|
|
.fetch_rrdp(RrdpResourceKind::Notification, &uri, &origin)
|
|
.unwrap_err();
|
|
assert!(matches!(
|
|
err,
|
|
RrdpFetchError::Rrdp(RrdpError::CrossOriginRedirect { .. })
|
|
));
|
|
target_thread.join().expect("target thread");
|
|
assert_eq!(target_hits.load(Ordering::SeqCst), 0);
|
|
}
|
|
}
|