增加必要的数据记录
This commit is contained in:
103
src/rtr/cache/core.rs
vendored
103
src/rtr/cache/core.rs
vendored
@@ -443,7 +443,10 @@ impl RtrCache {
|
||||
}
|
||||
|
||||
pub fn snapshot_for_version(&self, version: u8) -> Snapshot {
|
||||
self.versions[version_index(version)].snapshot.as_ref().clone()
|
||||
self.versions[version_index(version)]
|
||||
.snapshot
|
||||
.as_ref()
|
||||
.clone()
|
||||
}
|
||||
|
||||
pub fn rtr_payloads_for_version(&self, version: u8) -> Arc<Vec<Payload>> {
|
||||
@@ -661,6 +664,66 @@ pub struct CacheMemoryStats {
|
||||
pub rtr_payloads_arc_strong_counts: [usize; VERSION_COUNT],
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, Serialize)]
|
||||
pub struct PayloadCounts {
|
||||
pub total: usize,
|
||||
pub vrp: usize,
|
||||
pub router_key: usize,
|
||||
pub aspa: usize,
|
||||
}
|
||||
|
||||
impl PayloadCounts {
|
||||
fn add_payload(&mut self, payload: &Payload) {
|
||||
self.total += 1;
|
||||
match payload {
|
||||
Payload::RouteOrigin(_) => self.vrp += 1,
|
||||
Payload::RouterKey(_) => self.router_key += 1,
|
||||
Payload::Aspa(_) => self.aspa += 1,
|
||||
}
|
||||
}
|
||||
|
||||
fn add_counts(&mut self, other: Self) {
|
||||
self.total += other.total;
|
||||
self.vrp += other.vrp;
|
||||
self.router_key += other.router_key;
|
||||
self.aspa += other.aspa;
|
||||
}
|
||||
|
||||
fn from_payloads(payloads: &[Payload]) -> Self {
|
||||
let mut counts = Self::default();
|
||||
for payload in payloads {
|
||||
counts.add_payload(payload);
|
||||
}
|
||||
counts
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
pub struct DeltaReportStats {
|
||||
pub serial: u32,
|
||||
pub announced: PayloadCounts,
|
||||
pub withdrawn: PayloadCounts,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
pub struct DeltaWindowReportStats {
|
||||
pub length: usize,
|
||||
pub oldest_serial: Option<u32>,
|
||||
pub newest_serial: Option<u32>,
|
||||
pub announced: PayloadCounts,
|
||||
pub withdrawn: PayloadCounts,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
pub struct VersionReportStats {
|
||||
pub version: u8,
|
||||
pub session_id: u16,
|
||||
pub serial: u32,
|
||||
pub snapshot: PayloadCounts,
|
||||
pub latest_delta: Option<DeltaReportStats>,
|
||||
pub delta_window: DeltaWindowReportStats,
|
||||
}
|
||||
|
||||
impl RtrCache {
|
||||
pub fn memory_stats(&self) -> CacheMemoryStats {
|
||||
let snapshot_payload_counts =
|
||||
@@ -687,6 +750,44 @@ impl RtrCache {
|
||||
rtr_payloads_arc_strong_counts,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn version_report_stats(&self) -> [VersionReportStats; VERSION_COUNT] {
|
||||
std::array::from_fn(|idx| {
|
||||
let state = &self.versions[idx];
|
||||
let snapshot = PayloadCounts {
|
||||
total: state.rtr_payloads.len(),
|
||||
vrp: state.snapshot.origins().len(),
|
||||
router_key: state.snapshot.router_keys().len(),
|
||||
aspa: state.snapshot.aspas().len(),
|
||||
};
|
||||
let latest_delta = state.deltas.back().map(|delta| DeltaReportStats {
|
||||
serial: delta.serial(),
|
||||
announced: PayloadCounts::from_payloads(delta.announced()),
|
||||
withdrawn: PayloadCounts::from_payloads(delta.withdrawn()),
|
||||
});
|
||||
let mut window_announced = PayloadCounts::default();
|
||||
let mut window_withdrawn = PayloadCounts::default();
|
||||
for delta in &state.deltas {
|
||||
window_announced.add_counts(PayloadCounts::from_payloads(delta.announced()));
|
||||
window_withdrawn.add_counts(PayloadCounts::from_payloads(delta.withdrawn()));
|
||||
}
|
||||
|
||||
VersionReportStats {
|
||||
version: idx as u8,
|
||||
session_id: state.session_id,
|
||||
serial: state.serial,
|
||||
snapshot,
|
||||
latest_delta,
|
||||
delta_window: DeltaWindowReportStats {
|
||||
length: state.deltas.len(),
|
||||
oldest_serial: state.deltas.front().map(|delta| delta.serial()),
|
||||
newest_serial: state.deltas.back().map(|delta| delta.serial()),
|
||||
announced: window_announced,
|
||||
withdrawn: window_withdrawn,
|
||||
},
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
|
||||
3
src/rtr/cache/mod.rs
vendored
3
src/rtr/cache/mod.rs
vendored
@@ -4,7 +4,8 @@ mod ordering;
|
||||
mod store;
|
||||
|
||||
pub use core::{
|
||||
CacheAvailability, CacheMemoryStats, RtrCache, RtrCacheBuilder, SerialResult, SessionIds,
|
||||
CacheAvailability, CacheMemoryStats, DeltaReportStats, DeltaWindowReportStats, PayloadCounts,
|
||||
RtrCache, RtrCacheBuilder, SerialResult, SessionIds, VersionReportStats,
|
||||
};
|
||||
pub use model::{Delta, DualTime, Snapshot};
|
||||
pub use ordering::{
|
||||
|
||||
474
src/rtr/config.rs
Normal file
474
src/rtr/config.rs
Normal file
@@ -0,0 +1,474 @@
|
||||
use std::env;
|
||||
use std::net::SocketAddr;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Result, anyhow};
|
||||
use chrono_tz::Tz;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::rtr::payload::Timing;
|
||||
use crate::rtr::server::RtrServiceConfig;
|
||||
use crate::rtr::server::ssh::SshAuthMode;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AppConfig {
|
||||
pub enable_tls: bool,
|
||||
pub enable_ssh: bool,
|
||||
pub tcp_addr: SocketAddr,
|
||||
pub tls_addr: SocketAddr,
|
||||
pub ssh_addr: SocketAddr,
|
||||
|
||||
pub db_path: String,
|
||||
pub ccr_dir: String,
|
||||
pub slurm_dir: Option<String>,
|
||||
pub tls_cert_path: String,
|
||||
pub tls_key_path: String,
|
||||
pub tls_client_ca_path: String,
|
||||
pub ssh_host_key_path: String,
|
||||
pub ssh_authorized_keys_path: String,
|
||||
pub ssh_username: String,
|
||||
pub ssh_subsystem_name: String,
|
||||
pub ssh_auth_mode: SshAuthMode,
|
||||
pub ssh_password: Option<String>,
|
||||
|
||||
pub max_delta: u8,
|
||||
pub prune_delta_by_snapshot_size: bool,
|
||||
pub strict_ccr_validation: bool,
|
||||
pub source_refresh_interval: Duration,
|
||||
pub report_dir: String,
|
||||
pub report_interval: Duration,
|
||||
pub timezone: Tz,
|
||||
pub timing: Timing,
|
||||
|
||||
pub service_config: RtrServiceConfig,
|
||||
}
|
||||
|
||||
impl Default for AppConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enable_tls: false,
|
||||
enable_ssh: false,
|
||||
tcp_addr: "0.0.0.0:323".parse().expect("invalid default tcp_addr"),
|
||||
tls_addr: "0.0.0.0:324".parse().expect("invalid default tls_addr"),
|
||||
ssh_addr: "0.0.0.0:22".parse().expect("invalid default ssh_addr"),
|
||||
|
||||
db_path: "./rtr-db".to_string(),
|
||||
ccr_dir: "./data".to_string(),
|
||||
slurm_dir: None,
|
||||
tls_cert_path: "./certs/server.crt".to_string(),
|
||||
tls_key_path: "./certs/server.key".to_string(),
|
||||
tls_client_ca_path: "./certs/client-ca.crt".to_string(),
|
||||
ssh_host_key_path: "./certs/ssh_host_ed25519_key".to_string(),
|
||||
ssh_authorized_keys_path: "./certs/rtr-authorized_keys".to_string(),
|
||||
ssh_username: "rpki-rtr".to_string(),
|
||||
ssh_subsystem_name: "rpki-rtr".to_string(),
|
||||
ssh_auth_mode: SshAuthMode::Key,
|
||||
ssh_password: None,
|
||||
|
||||
max_delta: 100,
|
||||
prune_delta_by_snapshot_size: false,
|
||||
strict_ccr_validation: false,
|
||||
source_refresh_interval: Duration::from_secs(300),
|
||||
report_dir: "./report".to_string(),
|
||||
report_interval: Duration::from_secs(60),
|
||||
timezone: default_timezone(),
|
||||
timing: Timing::default(),
|
||||
|
||||
service_config: RtrServiceConfig {
|
||||
max_connections: 512,
|
||||
max_concurrent_handshakes: 128,
|
||||
notify_queue_size: 1024,
|
||||
tcp_keepalive: Some(Duration::from_secs(60)),
|
||||
warn_insecure_tcp: true,
|
||||
require_tls_server_dns_name_san: false,
|
||||
enforce_tls_client_san_ip_match: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AppConfig {
|
||||
pub fn from_env() -> Result<Self> {
|
||||
let mut config = Self::default();
|
||||
|
||||
if let Some(value) = env_var("RPKI_RTR_ENABLE_TLS")? {
|
||||
config.enable_tls = parse_bool(&value, "RPKI_RTR_ENABLE_TLS")?;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_ENABLE_SSH")? {
|
||||
config.enable_ssh = parse_bool(&value, "RPKI_RTR_ENABLE_SSH")?;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_TCP_ADDR")? {
|
||||
config.tcp_addr = value
|
||||
.parse()
|
||||
.map_err(|err| anyhow!("invalid RPKI_RTR_TCP_ADDR '{}': {}", value, err))?;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_TLS_ADDR")? {
|
||||
config.tls_addr = value
|
||||
.parse()
|
||||
.map_err(|err| anyhow!("invalid RPKI_RTR_TLS_ADDR '{}': {}", value, err))?;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_SSH_ADDR")? {
|
||||
config.ssh_addr = value
|
||||
.parse()
|
||||
.map_err(|err| anyhow!("invalid RPKI_RTR_SSH_ADDR '{}': {}", value, err))?;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_SSH_PORT")? {
|
||||
let port: u16 = value
|
||||
.parse()
|
||||
.map_err(|err| anyhow!("invalid RPKI_RTR_SSH_PORT '{}': {}", value, err))?;
|
||||
config.ssh_addr.set_port(port);
|
||||
}
|
||||
|
||||
if let Some(value) = env_var("RPKI_RTR_DB_PATH")? {
|
||||
config.db_path = value;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_CCR_DIR")? {
|
||||
config.ccr_dir = value;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_SLURM_DIR")? {
|
||||
let value = value.trim();
|
||||
config.slurm_dir = if value.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(value.to_string())
|
||||
};
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_TLS_CERT_PATH")? {
|
||||
config.tls_cert_path = value;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_TLS_KEY_PATH")? {
|
||||
config.tls_key_path = value;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_TLS_CLIENT_CA_PATH")? {
|
||||
config.tls_client_ca_path = value;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_SSH_HOST_KEY_PATH")? {
|
||||
config.ssh_host_key_path = value;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_SSH_AUTHORIZED_KEYS_PATH")? {
|
||||
config.ssh_authorized_keys_path = value;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_SSH_USERNAME")? {
|
||||
config.ssh_username = value;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_SSH_SUBSYSTEM_NAME")? {
|
||||
config.ssh_subsystem_name = value;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_SSH_AUTH_MODE")? {
|
||||
config.ssh_auth_mode = SshAuthMode::parse(&value).ok_or_else(|| {
|
||||
anyhow!(
|
||||
"invalid RPKI_RTR_SSH_AUTH_MODE '{}': expected key|password|both",
|
||||
value
|
||||
)
|
||||
})?;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_SSH_PASSWORD")? {
|
||||
let value = value.trim().to_string();
|
||||
config.ssh_password = if value.is_empty() { None } else { Some(value) };
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_MAX_DELTA")? {
|
||||
let parsed: u8 = value
|
||||
.parse()
|
||||
.map_err(|err| anyhow!("invalid RPKI_RTR_MAX_DELTA '{}': {}", value, err))?;
|
||||
if parsed == 0 {
|
||||
return Err(anyhow!(
|
||||
"invalid RPKI_RTR_MAX_DELTA '{}': must be >= 1",
|
||||
value
|
||||
));
|
||||
}
|
||||
config.max_delta = parsed;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_PRUNE_DELTA_BY_SNAPSHOT_SIZE")? {
|
||||
config.prune_delta_by_snapshot_size =
|
||||
parse_bool(&value, "RPKI_RTR_PRUNE_DELTA_BY_SNAPSHOT_SIZE")?;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_STRICT_CCR_VALIDATION")? {
|
||||
config.strict_ccr_validation = parse_bool(&value, "RPKI_RTR_STRICT_CCR_VALIDATION")?;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_REPORT_DIR")? {
|
||||
config.report_dir = value;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_REPORT_INTERVAL_SECS")? {
|
||||
let secs = parse_positive_u64(&value, "RPKI_RTR_REPORT_INTERVAL_SECS")?;
|
||||
config.report_interval = Duration::from_secs(secs);
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_TIMEZONE")? {
|
||||
config.timezone = parse_timezone(&value, "RPKI_RTR_TIMEZONE")?;
|
||||
}
|
||||
|
||||
let source_refresh_interval_new = env_var("RPKI_RTR_SOURCE_REFRESH_INTERVAL_SECS")?;
|
||||
let source_refresh_interval_legacy = env_var("RPKI_RTR_REFRESH_INTERVAL_SECS")?;
|
||||
match (
|
||||
source_refresh_interval_new.as_deref(),
|
||||
source_refresh_interval_legacy.as_deref(),
|
||||
) {
|
||||
(Some(new_value), Some(_)) => {
|
||||
let secs = parse_positive_u64(new_value, "RPKI_RTR_SOURCE_REFRESH_INTERVAL_SECS")?;
|
||||
config.source_refresh_interval = Duration::from_secs(secs);
|
||||
warn!(
|
||||
"both RPKI_RTR_SOURCE_REFRESH_INTERVAL_SECS and legacy RPKI_RTR_REFRESH_INTERVAL_SECS are set; using RPKI_RTR_SOURCE_REFRESH_INTERVAL_SECS"
|
||||
);
|
||||
}
|
||||
(Some(new_value), None) => {
|
||||
let secs = parse_positive_u64(new_value, "RPKI_RTR_SOURCE_REFRESH_INTERVAL_SECS")?;
|
||||
config.source_refresh_interval = Duration::from_secs(secs);
|
||||
}
|
||||
(None, Some(legacy_value)) => {
|
||||
let secs = parse_positive_u64(legacy_value, "RPKI_RTR_REFRESH_INTERVAL_SECS")?;
|
||||
config.source_refresh_interval = Duration::from_secs(secs);
|
||||
warn!(
|
||||
"RPKI_RTR_REFRESH_INTERVAL_SECS is deprecated; use RPKI_RTR_SOURCE_REFRESH_INTERVAL_SECS"
|
||||
);
|
||||
}
|
||||
(None, None) => {}
|
||||
}
|
||||
|
||||
if let Some(value) = env_var("RPKI_RTR_TIMING_REFRESH_SECS")? {
|
||||
config.timing.refresh = parse_positive_u32(&value, "RPKI_RTR_TIMING_REFRESH_SECS")?;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_TIMING_RETRY_SECS")? {
|
||||
config.timing.retry = parse_positive_u32(&value, "RPKI_RTR_TIMING_RETRY_SECS")?;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_TIMING_EXPIRE_SECS")? {
|
||||
config.timing.expire = parse_positive_u32(&value, "RPKI_RTR_TIMING_EXPIRE_SECS")?;
|
||||
}
|
||||
config
|
||||
.timing
|
||||
.validate()
|
||||
.map_err(|err| anyhow!("invalid RTR timing configuration: {}", err))?;
|
||||
|
||||
if let Some(value) = env_var("RPKI_RTR_MAX_CONNECTIONS")? {
|
||||
config.service_config.max_connections = value
|
||||
.parse()
|
||||
.map_err(|err| anyhow!("invalid RPKI_RTR_MAX_CONNECTIONS '{}': {}", value, err))?;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_MAX_CONCURRENT_HANDSHAKES")? {
|
||||
config.service_config.max_concurrent_handshakes = value.parse().map_err(|err| {
|
||||
anyhow!(
|
||||
"invalid RPKI_RTR_MAX_CONCURRENT_HANDSHAKES '{}': {}",
|
||||
value,
|
||||
err
|
||||
)
|
||||
})?;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_NOTIFY_QUEUE_SIZE")? {
|
||||
config.service_config.notify_queue_size = value.parse().map_err(|err| {
|
||||
anyhow!("invalid RPKI_RTR_NOTIFY_QUEUE_SIZE '{}': {}", value, err)
|
||||
})?;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_TCP_KEEPALIVE_SECS")? {
|
||||
let secs: u64 = value.parse().map_err(|err| {
|
||||
anyhow!("invalid RPKI_RTR_TCP_KEEPALIVE_SECS '{}': {}", value, err)
|
||||
})?;
|
||||
config.service_config.tcp_keepalive = if secs == 0 {
|
||||
None
|
||||
} else {
|
||||
Some(Duration::from_secs(secs))
|
||||
};
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_WARN_INSECURE_TCP")? {
|
||||
config.service_config.warn_insecure_tcp =
|
||||
parse_bool(&value, "RPKI_RTR_WARN_INSECURE_TCP")?;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_REQUIRE_TLS_SERVER_DNS_NAME_SAN")? {
|
||||
config.service_config.require_tls_server_dns_name_san =
|
||||
parse_bool(&value, "RPKI_RTR_REQUIRE_TLS_SERVER_DNS_NAME_SAN")?;
|
||||
}
|
||||
if let Some(value) = env_var("RPKI_RTR_ENFORCE_TLS_CLIENT_SAN_IP_MATCH")? {
|
||||
config.service_config.enforce_tls_client_san_ip_match =
|
||||
parse_bool(&value, "RPKI_RTR_ENFORCE_TLS_CLIENT_SAN_IP_MATCH")?;
|
||||
}
|
||||
|
||||
if config.service_config.max_connections == 0 {
|
||||
return Err(anyhow!(
|
||||
"invalid RPKI_RTR_MAX_CONNECTIONS '{}': must be >= 1",
|
||||
config.service_config.max_connections
|
||||
));
|
||||
}
|
||||
if config.service_config.max_concurrent_handshakes == 0 {
|
||||
return Err(anyhow!(
|
||||
"invalid RPKI_RTR_MAX_CONCURRENT_HANDSHAKES '{}': must be >= 1",
|
||||
config.service_config.max_concurrent_handshakes
|
||||
));
|
||||
}
|
||||
if config.service_config.max_concurrent_handshakes > config.service_config.max_connections {
|
||||
return Err(anyhow!(
|
||||
"invalid handshake/connection limits: RPKI_RTR_MAX_CONCURRENT_HANDSHAKES ({}) must be <= RPKI_RTR_MAX_CONNECTIONS ({})",
|
||||
config.service_config.max_concurrent_handshakes,
|
||||
config.service_config.max_connections
|
||||
));
|
||||
}
|
||||
|
||||
Ok(config)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn log_startup_config(config: &AppConfig) {
|
||||
info!("starting RTR service");
|
||||
info!("db_path={}", config.db_path);
|
||||
info!("tcp_addr={}", config.tcp_addr);
|
||||
info!("tls_enabled={}", config.enable_tls);
|
||||
info!("ssh_enabled={}", config.enable_ssh);
|
||||
|
||||
if config.enable_tls {
|
||||
info!("tls_addr={}", config.tls_addr);
|
||||
info!("tls_cert_path={}", config.tls_cert_path);
|
||||
info!("tls_key_path={}", config.tls_key_path);
|
||||
info!("tls_client_ca_path={}", config.tls_client_ca_path);
|
||||
}
|
||||
if config.enable_ssh {
|
||||
info!("ssh_addr={}", config.ssh_addr);
|
||||
info!("ssh_host_key_path={}", config.ssh_host_key_path);
|
||||
info!(
|
||||
"ssh_authorized_keys_path={}",
|
||||
config.ssh_authorized_keys_path
|
||||
);
|
||||
info!("ssh_username={}", config.ssh_username);
|
||||
info!("ssh_subsystem_name={}", config.ssh_subsystem_name);
|
||||
info!("ssh_auth_mode={}", config.ssh_auth_mode.as_str());
|
||||
info!("ssh_password_enabled={}", config.ssh_password.is_some());
|
||||
}
|
||||
|
||||
info!("ccr_dir={}", config.ccr_dir);
|
||||
info!(
|
||||
"slurm_dir={}",
|
||||
config.slurm_dir.as_deref().unwrap_or("disabled")
|
||||
);
|
||||
info!("max_delta={}", config.max_delta);
|
||||
info!("strict_ccr_validation={}", config.strict_ccr_validation);
|
||||
info!(
|
||||
"source_refresh_interval_secs={}",
|
||||
config.source_refresh_interval.as_secs()
|
||||
);
|
||||
info!("report_dir={}", config.report_dir);
|
||||
info!("report_interval_secs={}", config.report_interval.as_secs());
|
||||
info!("timezone={}", format_timezone(config.timezone));
|
||||
info!("rtr_timing_refresh_secs={}", config.timing.refresh);
|
||||
info!("rtr_timing_retry_secs={}", config.timing.retry);
|
||||
info!("rtr_timing_expire_secs={}", config.timing.expire);
|
||||
info!("max_connections={}", config.service_config.max_connections);
|
||||
info!(
|
||||
"max_concurrent_handshakes={}",
|
||||
config.service_config.max_concurrent_handshakes
|
||||
);
|
||||
info!(
|
||||
"notify_queue_size={}",
|
||||
config.service_config.notify_queue_size
|
||||
);
|
||||
info!(
|
||||
"tcp_keepalive_secs={}",
|
||||
config
|
||||
.service_config
|
||||
.tcp_keepalive
|
||||
.map(|duration| duration.as_secs().to_string())
|
||||
.unwrap_or_else(|| "disabled".to_string())
|
||||
);
|
||||
info!(
|
||||
"warn_insecure_tcp={}",
|
||||
config.service_config.warn_insecure_tcp
|
||||
);
|
||||
info!(
|
||||
"require_tls_server_dns_name_san={}",
|
||||
config.service_config.require_tls_server_dns_name_san
|
||||
);
|
||||
info!(
|
||||
"enforce_tls_client_san_ip_match={}",
|
||||
config.service_config.enforce_tls_client_san_ip_match
|
||||
);
|
||||
}
|
||||
|
||||
pub fn default_timezone() -> Tz {
|
||||
chrono_tz::Asia::Shanghai
|
||||
}
|
||||
|
||||
pub fn format_timezone(timezone: Tz) -> String {
|
||||
timezone.name().to_string()
|
||||
}
|
||||
|
||||
fn env_var(name: &str) -> Result<Option<String>> {
|
||||
match env::var(name) {
|
||||
Ok(value) => Ok(Some(value)),
|
||||
Err(env::VarError::NotPresent) => Ok(None),
|
||||
Err(err) => Err(anyhow!("failed to read {}: {}", name, err)),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_bool(value: &str, name: &str) -> Result<bool> {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"1" | "true" | "yes" | "on" => Ok(true),
|
||||
"0" | "false" | "no" | "off" => Ok(false),
|
||||
_ => Err(anyhow!("invalid {} '{}': expected boolean", name, value)),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_positive_u64(value: &str, name: &str) -> Result<u64> {
|
||||
let parsed = value
|
||||
.parse::<u64>()
|
||||
.map_err(|err| anyhow!("invalid {} '{}': {}", name, value, err))?;
|
||||
if parsed == 0 {
|
||||
return Err(anyhow!("invalid {} '{}': must be >= 1", name, value));
|
||||
}
|
||||
Ok(parsed)
|
||||
}
|
||||
|
||||
fn parse_positive_u32(value: &str, name: &str) -> Result<u32> {
|
||||
let parsed = value
|
||||
.parse::<u32>()
|
||||
.map_err(|err| anyhow!("invalid {} '{}': {}", name, value, err))?;
|
||||
if parsed == 0 {
|
||||
return Err(anyhow!("invalid {} '{}': must be >= 1", name, value));
|
||||
}
|
||||
Ok(parsed)
|
||||
}
|
||||
|
||||
fn parse_timezone(value: &str, name: &str) -> Result<Tz> {
|
||||
let value = value.trim();
|
||||
let normalized = match value.to_ascii_lowercase().as_str() {
|
||||
"shanghai" | "beijing" | "peking" => "Asia/Shanghai",
|
||||
"utc" | "z" => "UTC",
|
||||
_ => value,
|
||||
};
|
||||
|
||||
normalized.parse::<Tz>().map_err(|err| {
|
||||
anyhow!(
|
||||
"invalid {} '{}': expected IANA timezone like Asia/Shanghai, Europe/London, America/New_York, or UTC: {}",
|
||||
name,
|
||||
value,
|
||||
err
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_timezone_accepts_iana_names_and_common_aliases() {
|
||||
assert_eq!(
|
||||
parse_timezone("Asia/Shanghai", "TEST").unwrap(),
|
||||
chrono_tz::Asia::Shanghai
|
||||
);
|
||||
assert_eq!(
|
||||
parse_timezone("shanghai", "TEST").unwrap(),
|
||||
chrono_tz::Asia::Shanghai
|
||||
);
|
||||
assert_eq!(
|
||||
parse_timezone("Europe/London", "TEST").unwrap(),
|
||||
chrono_tz::Europe::London
|
||||
);
|
||||
assert_eq!(
|
||||
parse_timezone("America/New_York", "TEST").unwrap(),
|
||||
chrono_tz::America::New_York
|
||||
);
|
||||
assert_eq!(parse_timezone("UTC", "TEST").unwrap(), chrono_tz::UTC);
|
||||
assert_eq!(parse_timezone("Z", "TEST").unwrap(), chrono_tz::UTC);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_timezone_rejects_offset_and_invalid_values() {
|
||||
assert!(parse_timezone("+08:00", "TEST").is_err());
|
||||
assert!(parse_timezone("+0800", "TEST").is_err());
|
||||
assert!(parse_timezone("Mars/Base", "TEST").is_err());
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
pub mod cache;
|
||||
pub mod config;
|
||||
pub mod error_type;
|
||||
pub mod payload;
|
||||
pub mod pdu;
|
||||
pub mod report;
|
||||
pub mod server;
|
||||
pub mod session;
|
||||
pub mod state;
|
||||
|
||||
565
src/rtr/report.rs
Normal file
565
src/rtr/report.rs
Normal file
@@ -0,0 +1,565 @@
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::Instant;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use chrono::{DateTime, Utc};
|
||||
use chrono_tz::Tz;
|
||||
use serde::Serialize;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::rtr::cache::{CacheAvailability, SharedRtrCache, VersionReportStats};
|
||||
use crate::rtr::config::format_timezone;
|
||||
use crate::rtr::server::{RtrNotifier, RtrServiceStats, RtrTransportConnectionCounts};
|
||||
use crate::source::pipeline::{DataQualityReport, SourceLoadReport};
|
||||
|
||||
#[derive(Clone, Serialize)]
|
||||
pub struct ReportConfiguration {
|
||||
source_refresh_interval_seconds: u64,
|
||||
report_interval_seconds: u64,
|
||||
max_delta: u8,
|
||||
prune_delta_by_snapshot_size: bool,
|
||||
strict_ccr_validation: bool,
|
||||
timezone: String,
|
||||
timing: TimingReport,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Serialize)]
|
||||
struct TimingReport {
|
||||
refresh: u32,
|
||||
retry: u32,
|
||||
expire: u32,
|
||||
}
|
||||
|
||||
impl ReportConfiguration {
|
||||
pub fn new(
|
||||
source_refresh_interval_seconds: u64,
|
||||
report_interval_seconds: u64,
|
||||
max_delta: u8,
|
||||
prune_delta_by_snapshot_size: bool,
|
||||
strict_ccr_validation: bool,
|
||||
timezone: Tz,
|
||||
timing: (u32, u32, u32),
|
||||
) -> Self {
|
||||
Self {
|
||||
source_refresh_interval_seconds,
|
||||
report_interval_seconds,
|
||||
max_delta,
|
||||
prune_delta_by_snapshot_size,
|
||||
strict_ccr_validation,
|
||||
timezone: format_timezone(timezone),
|
||||
timing: TimingReport {
|
||||
refresh: timing.0,
|
||||
retry: timing.1,
|
||||
expire: timing.2,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn timezone(&self) -> Tz {
|
||||
self.timezone
|
||||
.parse::<Tz>()
|
||||
.expect("serialized timezone should be a valid IANA timezone")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ReportContext {
|
||||
started_at: DateTime<Utc>,
|
||||
started_instant: Instant,
|
||||
timezone: Tz,
|
||||
configuration: ReportConfiguration,
|
||||
runtime: Arc<RwLock<RuntimeReportState>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
struct RuntimeReportState {
|
||||
source: Option<SourceLoadReport>,
|
||||
data_quality: Option<DataQualityReport>,
|
||||
refresh: RefreshReport,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
struct RefreshReport {
|
||||
last_attempt_at: Option<DateTime<Utc>>,
|
||||
last_success_at: Option<DateTime<Utc>>,
|
||||
last_changed_at: Option<DateTime<Utc>>,
|
||||
status: &'static str,
|
||||
changed: Option<bool>,
|
||||
duration_ms: Option<u128>,
|
||||
consecutive_failures: u64,
|
||||
last_error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct RefreshReportView {
|
||||
last_attempt_at: Option<DateTime<Tz>>,
|
||||
last_success_at: Option<DateTime<Tz>>,
|
||||
last_changed_at: Option<DateTime<Tz>>,
|
||||
status: &'static str,
|
||||
changed: Option<bool>,
|
||||
duration_ms: Option<u128>,
|
||||
consecutive_failures: u64,
|
||||
last_error: Option<String>,
|
||||
}
|
||||
|
||||
impl RefreshReportView {
|
||||
fn from_report(report: RefreshReport, timezone: Tz) -> Self {
|
||||
Self {
|
||||
last_attempt_at: report
|
||||
.last_attempt_at
|
||||
.map(|time| to_report_time(time, timezone)),
|
||||
last_success_at: report
|
||||
.last_success_at
|
||||
.map(|time| to_report_time(time, timezone)),
|
||||
last_changed_at: report
|
||||
.last_changed_at
|
||||
.map(|time| to_report_time(time, timezone)),
|
||||
status: report.status,
|
||||
changed: report.changed,
|
||||
duration_ms: report.duration_ms,
|
||||
consecutive_failures: report.consecutive_failures,
|
||||
last_error: report.last_error,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for RefreshReport {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
last_attempt_at: None,
|
||||
last_success_at: None,
|
||||
last_changed_at: None,
|
||||
status: "not_attempted",
|
||||
changed: None,
|
||||
duration_ms: None,
|
||||
consecutive_failures: 0,
|
||||
last_error: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ReportContext {
|
||||
pub fn new(configuration: ReportConfiguration) -> Self {
|
||||
let timezone = configuration.timezone();
|
||||
Self {
|
||||
started_at: Utc::now(),
|
||||
started_instant: Instant::now(),
|
||||
timezone,
|
||||
configuration,
|
||||
runtime: Arc::new(RwLock::new(RuntimeReportState::default())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_refresh_success(
|
||||
&self,
|
||||
attempted_at: DateTime<Utc>,
|
||||
duration_ms: u128,
|
||||
changed: bool,
|
||||
source: SourceLoadReport,
|
||||
data_quality: DataQualityReport,
|
||||
) {
|
||||
let mut state = self
|
||||
.runtime
|
||||
.write()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
state.source = Some(source);
|
||||
state.data_quality = Some(data_quality);
|
||||
state.refresh.last_attempt_at = Some(attempted_at);
|
||||
state.refresh.last_success_at = Some(Utc::now());
|
||||
if changed {
|
||||
state.refresh.last_changed_at = Some(Utc::now());
|
||||
}
|
||||
state.refresh.status = "success";
|
||||
state.refresh.changed = Some(changed);
|
||||
state.refresh.duration_ms = Some(duration_ms);
|
||||
state.refresh.consecutive_failures = 0;
|
||||
state.refresh.last_error = None;
|
||||
}
|
||||
|
||||
pub fn record_refresh_unchanged(&self, attempted_at: DateTime<Utc>, duration_ms: u128) {
|
||||
let mut state = self
|
||||
.runtime
|
||||
.write()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
state.refresh.last_attempt_at = Some(attempted_at);
|
||||
state.refresh.last_success_at = Some(Utc::now());
|
||||
state.refresh.status = "success";
|
||||
state.refresh.changed = Some(false);
|
||||
state.refresh.duration_ms = Some(duration_ms);
|
||||
state.refresh.consecutive_failures = 0;
|
||||
state.refresh.last_error = None;
|
||||
}
|
||||
|
||||
pub fn record_refresh_failure(
|
||||
&self,
|
||||
attempted_at: DateTime<Utc>,
|
||||
duration_ms: u128,
|
||||
error: &anyhow::Error,
|
||||
) {
|
||||
let mut state = self
|
||||
.runtime
|
||||
.write()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
state.refresh.last_attempt_at = Some(attempted_at);
|
||||
state.refresh.status = "failed";
|
||||
state.refresh.changed = None;
|
||||
state.refresh.duration_ms = Some(duration_ms);
|
||||
state.refresh.consecutive_failures = state.refresh.consecutive_failures.saturating_add(1);
|
||||
state.refresh.last_error = Some(error.to_string());
|
||||
}
|
||||
|
||||
pub fn write_or_warn(
|
||||
&self,
|
||||
report_dir: &Path,
|
||||
phase: &str,
|
||||
shared_cache: &SharedRtrCache,
|
||||
notifier: &RtrNotifier,
|
||||
service_stats: &RtrServiceStats,
|
||||
) {
|
||||
if let Err(err) = self.write(report_dir, phase, shared_cache, notifier, service_stats) {
|
||||
warn!(
|
||||
"failed to write RTR report to {}: {:?}",
|
||||
report_dir.display(),
|
||||
err
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn write(
|
||||
&self,
|
||||
report_dir: &Path,
|
||||
phase: &str,
|
||||
shared_cache: &SharedRtrCache,
|
||||
notifier: &RtrNotifier,
|
||||
service_stats: &RtrServiceStats,
|
||||
) -> Result<()> {
|
||||
let cache = shared_cache.load_full();
|
||||
let runtime = self
|
||||
.runtime
|
||||
.read()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clone();
|
||||
let availability = match cache.availability() {
|
||||
CacheAvailability::Ready => "ready",
|
||||
CacheAvailability::NoDataAvailable => "no_data_available",
|
||||
};
|
||||
let active_connections = service_stats.active_connections();
|
||||
let connections_by_transport = service_stats.transport_connections();
|
||||
let max_connections = service_stats.max_connections();
|
||||
let report = RtrReport {
|
||||
schema_version: 2,
|
||||
generated_at: self.report_now(),
|
||||
phase: phase.to_string(),
|
||||
service: ServiceReport {
|
||||
started_at: self.to_report_time(self.started_at),
|
||||
uptime_seconds: self.started_instant.elapsed().as_secs(),
|
||||
active_connections,
|
||||
connections_by_transport: TransportConnectionReport::from(connections_by_transport),
|
||||
session_listeners: notifier.listener_count(),
|
||||
max_connections,
|
||||
connection_utilization: active_connections as f64 / max_connections as f64,
|
||||
},
|
||||
process: ProcessReport {
|
||||
rss_mib: current_rss_mib(),
|
||||
},
|
||||
source: runtime
|
||||
.source
|
||||
.map(|source| SourceLoadReportView::from_report(source, self.timezone)),
|
||||
refresh: RefreshReportView::from_report(runtime.refresh, self.timezone),
|
||||
data_quality: runtime.data_quality,
|
||||
configuration: self.configuration.clone(),
|
||||
cache: CacheReport {
|
||||
availability,
|
||||
created_at: self.to_report_time(cache.created_at().utc()),
|
||||
last_update_begin: self.to_report_time(cache.last_update_begin().utc()),
|
||||
last_update_end: self.to_report_time(cache.last_update_end().utc()),
|
||||
versions: cache.version_report_stats(),
|
||||
},
|
||||
};
|
||||
|
||||
fs::create_dir_all(report_dir)
|
||||
.with_context(|| format!("create report directory {}", report_dir.display()))?;
|
||||
let target = report_dir.join("rtr-server.json");
|
||||
let temporary = report_dir.join(".rtr-server.json.tmp");
|
||||
let json = serde_json::to_vec_pretty(&report).context("serialize RTR report")?;
|
||||
fs::write(&temporary, json)
|
||||
.with_context(|| format!("write temporary report {}", temporary.display()))?;
|
||||
|
||||
if let Err(err) = fs::rename(&temporary, &target) {
|
||||
if target.exists() {
|
||||
fs::remove_file(&target)
|
||||
.with_context(|| format!("replace existing report {}", target.display()))?;
|
||||
fs::rename(&temporary, &target)
|
||||
.with_context(|| format!("move report into {}", target.display()))?;
|
||||
} else {
|
||||
return Err(err).with_context(|| {
|
||||
format!(
|
||||
"move temporary report {} into {}",
|
||||
temporary.display(),
|
||||
target.display()
|
||||
)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn report_now(&self) -> DateTime<Tz> {
|
||||
self.to_report_time(Utc::now())
|
||||
}
|
||||
|
||||
fn to_report_time(&self, time: DateTime<Utc>) -> DateTime<Tz> {
|
||||
time.with_timezone(&self.timezone)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct RtrReport {
|
||||
schema_version: u16,
|
||||
generated_at: DateTime<Tz>,
|
||||
phase: String,
|
||||
service: ServiceReport,
|
||||
process: ProcessReport,
|
||||
source: Option<SourceLoadReportView>,
|
||||
refresh: RefreshReportView,
|
||||
data_quality: Option<DataQualityReport>,
|
||||
configuration: ReportConfiguration,
|
||||
cache: CacheReport,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ServiceReport {
|
||||
started_at: DateTime<Tz>,
|
||||
uptime_seconds: u64,
|
||||
active_connections: usize,
|
||||
connections_by_transport: TransportConnectionReport,
|
||||
session_listeners: usize,
|
||||
max_connections: usize,
|
||||
connection_utilization: f64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct SourceLoadReportView {
|
||||
ccr_file: String,
|
||||
ccr_file_size_bytes: u64,
|
||||
ccr_modified_at: Option<DateTime<Tz>>,
|
||||
ccr_produced_at: Option<String>,
|
||||
slurm_enabled: bool,
|
||||
slurm_file_count: usize,
|
||||
slurm_files: Vec<String>,
|
||||
slurm_version: Option<u32>,
|
||||
}
|
||||
|
||||
impl SourceLoadReportView {
|
||||
fn from_report(report: SourceLoadReport, timezone: Tz) -> Self {
|
||||
Self {
|
||||
ccr_file: report.ccr_file,
|
||||
ccr_file_size_bytes: report.ccr_file_size_bytes,
|
||||
ccr_modified_at: report
|
||||
.ccr_modified_at
|
||||
.map(|time| to_report_time(time, timezone)),
|
||||
ccr_produced_at: report.ccr_produced_at,
|
||||
slurm_enabled: report.slurm_enabled,
|
||||
slurm_file_count: report.slurm_file_count,
|
||||
slurm_files: report.slurm_files,
|
||||
slurm_version: report.slurm_version,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct TransportConnectionReport {
|
||||
tcp: usize,
|
||||
tls: usize,
|
||||
ssh: usize,
|
||||
}
|
||||
|
||||
impl From<RtrTransportConnectionCounts> for TransportConnectionReport {
|
||||
fn from(counts: RtrTransportConnectionCounts) -> Self {
|
||||
Self {
|
||||
tcp: counts.tcp,
|
||||
tls: counts.tls,
|
||||
ssh: counts.ssh,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ProcessReport {
|
||||
rss_mib: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct CacheReport {
|
||||
availability: &'static str,
|
||||
created_at: DateTime<Tz>,
|
||||
last_update_begin: DateTime<Tz>,
|
||||
last_update_end: DateTime<Tz>,
|
||||
versions: [VersionReportStats; 3],
|
||||
}
|
||||
|
||||
pub fn to_report_time(time: DateTime<Utc>, timezone: Tz) -> DateTime<Tz> {
|
||||
time.with_timezone(&timezone)
|
||||
}
|
||||
|
||||
pub fn current_rss_mib() -> Option<u64> {
|
||||
let status = fs::read_to_string("/proc/self/status").ok()?;
|
||||
let vmrss_line = status.lines().find(|line| line.starts_with("VmRSS:"))?;
|
||||
let kb = vmrss_line
|
||||
.split_whitespace()
|
||||
.nth(1)
|
||||
.and_then(|value| value.parse::<u64>().ok())?;
|
||||
Some(kb / 1024)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::rtr::cache::RtrCache;
|
||||
use crate::rtr::server::RtrService;
|
||||
use crate::source::pipeline::{
|
||||
DataQualityReport, PayloadTypeCounts, SlurmRuleCounts, SourceLoadReport,
|
||||
};
|
||||
use arc_swap::ArcSwap;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn test_configuration() -> ReportConfiguration {
|
||||
ReportConfiguration::new(
|
||||
300,
|
||||
60,
|
||||
100,
|
||||
false,
|
||||
false,
|
||||
chrono_tz::Asia::Shanghai,
|
||||
(3600, 600, 7200),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_report_creates_parseable_json() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let report_dir = temp.path().join("report");
|
||||
let shared_cache = Arc::new(ArcSwap::from_pointee(RtrCache::default()));
|
||||
let service = RtrService::new(shared_cache.clone());
|
||||
let notifier = service.notifier();
|
||||
let context = ReportContext::new(test_configuration());
|
||||
|
||||
context
|
||||
.write(
|
||||
&report_dir,
|
||||
"test",
|
||||
&shared_cache,
|
||||
¬ifier,
|
||||
&service.stats(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report: Value =
|
||||
serde_json::from_slice(&fs::read(report_dir.join("rtr-server.json")).unwrap()).unwrap();
|
||||
assert_eq!(report["schema_version"], 2);
|
||||
assert_eq!(report["phase"], "test");
|
||||
assert_report_time_offset(&report["generated_at"]);
|
||||
assert_report_time_offset(&report["service"]["started_at"]);
|
||||
assert_report_time_offset(&report["cache"]["created_at"]);
|
||||
assert_eq!(report["cache"]["availability"], "ready");
|
||||
assert_eq!(report["refresh"]["status"], "not_attempted");
|
||||
assert!(report["source"].is_null());
|
||||
assert!(report["data_quality"].is_null());
|
||||
assert_eq!(
|
||||
report["configuration"]["source_refresh_interval_seconds"],
|
||||
300
|
||||
);
|
||||
assert_eq!(report["service"]["max_connections"], 1024);
|
||||
assert_eq!(report["service"]["active_connections"], 0);
|
||||
assert_eq!(report["service"]["connections_by_transport"]["tcp"], 0);
|
||||
assert_eq!(report["service"]["connections_by_transport"]["tls"], 0);
|
||||
assert_eq!(report["service"]["connections_by_transport"]["ssh"], 0);
|
||||
assert_eq!(report["cache"]["versions"].as_array().unwrap().len(), 3);
|
||||
assert_eq!(report["cache"]["versions"][2]["snapshot"]["total"], 0);
|
||||
assert!(!report_dir.join(".rtr-server.json.tmp").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_failure_preserves_last_successful_source_data() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let report_dir = temp.path().join("report");
|
||||
let shared_cache = Arc::new(ArcSwap::from_pointee(RtrCache::default()));
|
||||
let service = RtrService::new(shared_cache.clone());
|
||||
let notifier = service.notifier();
|
||||
let context = ReportContext::new(test_configuration());
|
||||
let source = SourceLoadReport {
|
||||
ccr_file: "data/example.ccr".to_string(),
|
||||
ccr_file_size_bytes: 123,
|
||||
ccr_modified_at: Some(Utc::now()),
|
||||
ccr_produced_at: Some("20260615000000Z".to_string()),
|
||||
slurm_enabled: true,
|
||||
slurm_file_count: 1,
|
||||
slurm_files: vec!["policy.slurm".to_string()],
|
||||
slurm_version: Some(2),
|
||||
};
|
||||
let quality = DataQualityReport {
|
||||
ccr_input: PayloadTypeCounts {
|
||||
total: 11,
|
||||
vrp: 10,
|
||||
router_key: 0,
|
||||
aspa: 1,
|
||||
},
|
||||
invalid: PayloadTypeCounts::default(),
|
||||
before_slurm: PayloadTypeCounts {
|
||||
total: 11,
|
||||
vrp: 10,
|
||||
router_key: 0,
|
||||
aspa: 1,
|
||||
},
|
||||
after_slurm: PayloadTypeCounts {
|
||||
total: 10,
|
||||
vrp: 9,
|
||||
router_key: 0,
|
||||
aspa: 1,
|
||||
},
|
||||
slurm_filters: SlurmRuleCounts {
|
||||
prefix: 1,
|
||||
router_key: 0,
|
||||
aspa: 0,
|
||||
},
|
||||
slurm_assertions: SlurmRuleCounts::default(),
|
||||
};
|
||||
|
||||
context.record_refresh_success(Utc::now(), 12, true, source, quality);
|
||||
context.record_refresh_failure(Utc::now(), 5, &anyhow::anyhow!("source unavailable"));
|
||||
context
|
||||
.write(
|
||||
&report_dir,
|
||||
"refresh_failed",
|
||||
&shared_cache,
|
||||
¬ifier,
|
||||
&service.stats(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report: Value =
|
||||
serde_json::from_slice(&fs::read(report_dir.join("rtr-server.json")).unwrap()).unwrap();
|
||||
assert_eq!(report["source"]["ccr_file"], "data/example.ccr");
|
||||
assert_report_time_offset(&report["source"]["ccr_modified_at"]);
|
||||
assert_eq!(report["data_quality"]["after_slurm"]["total"], 10);
|
||||
assert_eq!(report["refresh"]["status"], "failed");
|
||||
assert_eq!(report["refresh"]["consecutive_failures"], 1);
|
||||
assert_eq!(report["refresh"]["last_error"], "source unavailable");
|
||||
assert!(!report["refresh"]["last_success_at"].is_null());
|
||||
assert_report_time_offset(&report["refresh"]["last_success_at"]);
|
||||
}
|
||||
|
||||
fn assert_report_time_offset(value: &Value) {
|
||||
let value = value.as_str().expect("report time should be a string");
|
||||
assert!(
|
||||
value.ends_with("+08:00"),
|
||||
"report time should use +08:00 offset, got {value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -16,16 +16,73 @@ use tokio_rustls::TlsAcceptor;
|
||||
use crate::rtr::cache::SharedRtrCache;
|
||||
use crate::rtr::session::RtrSession;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum RtrTransportKind {
|
||||
Tcp,
|
||||
Tls,
|
||||
Ssh,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||||
pub struct RtrTransportConnectionCounts {
|
||||
pub tcp: usize,
|
||||
pub tls: usize,
|
||||
pub ssh: usize,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct RtrTransportConnectionCounters {
|
||||
tcp: AtomicUsize,
|
||||
tls: AtomicUsize,
|
||||
ssh: AtomicUsize,
|
||||
}
|
||||
|
||||
impl RtrTransportConnectionCounters {
|
||||
pub fn increment(&self, transport: RtrTransportKind) {
|
||||
self.counter(transport).fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn decrement(&self, transport: RtrTransportKind) {
|
||||
self.counter(transport).fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn snapshot(&self) -> RtrTransportConnectionCounts {
|
||||
RtrTransportConnectionCounts {
|
||||
tcp: self.tcp.load(Ordering::Relaxed),
|
||||
tls: self.tls.load(Ordering::Relaxed),
|
||||
ssh: self.ssh.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
fn counter(&self, transport: RtrTransportKind) -> &AtomicUsize {
|
||||
match transport {
|
||||
RtrTransportKind::Tcp => &self.tcp,
|
||||
RtrTransportKind::Tls => &self.tls,
|
||||
RtrTransportKind::Ssh => &self.ssh,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ConnectionGuard {
|
||||
active_connections: Arc<AtomicUsize>,
|
||||
transport_connections: Arc<RtrTransportConnectionCounters>,
|
||||
transport: RtrTransportKind,
|
||||
_permit: OwnedSemaphorePermit,
|
||||
}
|
||||
|
||||
impl ConnectionGuard {
|
||||
pub fn new(active_connections: Arc<AtomicUsize>, permit: OwnedSemaphorePermit) -> Self {
|
||||
pub fn new(
|
||||
active_connections: Arc<AtomicUsize>,
|
||||
transport_connections: Arc<RtrTransportConnectionCounters>,
|
||||
transport: RtrTransportKind,
|
||||
permit: OwnedSemaphorePermit,
|
||||
) -> Self {
|
||||
active_connections.fetch_add(1, Ordering::Relaxed);
|
||||
transport_connections.increment(transport);
|
||||
Self {
|
||||
active_connections,
|
||||
transport_connections,
|
||||
transport,
|
||||
_permit: permit,
|
||||
}
|
||||
}
|
||||
@@ -38,6 +95,7 @@ impl ConnectionGuard {
|
||||
impl Drop for ConnectionGuard {
|
||||
fn drop(&mut self) {
|
||||
self.active_connections.fetch_sub(1, Ordering::Relaxed);
|
||||
self.transport_connections.decrement(self.transport);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,7 +130,10 @@ pub async fn handle_tls_connection(
|
||||
.with_context(|| format!("TLS handshake failed for {}", peer_addr))?;
|
||||
info!("RTR TLS handshake completed for {}", peer_addr);
|
||||
match verify_peer_certificate_ip(&tls_stream, peer_addr.ip()) {
|
||||
Ok(()) => info!("RTR TLS client certificate SAN IP validated for {}", peer_addr),
|
||||
Ok(()) => info!(
|
||||
"RTR TLS client certificate SAN IP validated for {}",
|
||||
peer_addr
|
||||
),
|
||||
Err(err) => {
|
||||
if enforce_client_san_ip_match {
|
||||
return Err(err).with_context(|| {
|
||||
|
||||
@@ -20,7 +20,8 @@ use rustls::ServerConfig;
|
||||
use crate::rtr::cache::SharedRtrCache;
|
||||
use crate::rtr::server::config::RtrServiceConfig;
|
||||
use crate::rtr::server::connection::{
|
||||
ConnectionGuard, handle_tcp_connection, handle_tls_connection, is_expected_disconnect,
|
||||
ConnectionGuard, RtrTransportConnectionCounters, RtrTransportKind, handle_tcp_connection,
|
||||
handle_tls_connection, is_expected_disconnect,
|
||||
};
|
||||
use crate::rtr::server::ssh::RtrSshRuntimeConfig;
|
||||
use crate::rtr::server::tls::load_rustls_server_config_with_options;
|
||||
@@ -30,6 +31,7 @@ type TransportFuture = Pin<Box<dyn Future<Output = Result<()>> + Send>>;
|
||||
|
||||
pub trait TransportAcceptor: Clone + Send + Sync + 'static {
|
||||
fn name(&self) -> &'static str;
|
||||
fn kind(&self) -> RtrTransportKind;
|
||||
fn requires_handshake_limit(&self) -> bool {
|
||||
false
|
||||
}
|
||||
@@ -52,6 +54,9 @@ impl TransportAcceptor for TcpTransport {
|
||||
fn name(&self) -> &'static str {
|
||||
"TCP"
|
||||
}
|
||||
fn kind(&self) -> RtrTransportKind {
|
||||
RtrTransportKind::Tcp
|
||||
}
|
||||
|
||||
fn handle_connection(
|
||||
&self,
|
||||
@@ -85,6 +90,9 @@ impl TransportAcceptor for TlsTransport {
|
||||
fn name(&self) -> &'static str {
|
||||
"TLS"
|
||||
}
|
||||
fn kind(&self) -> RtrTransportKind {
|
||||
RtrTransportKind::Tls
|
||||
}
|
||||
fn requires_handshake_limit(&self) -> bool {
|
||||
true
|
||||
}
|
||||
@@ -125,6 +133,9 @@ impl TransportAcceptor for SshTransport {
|
||||
fn name(&self) -> &'static str {
|
||||
"SSH"
|
||||
}
|
||||
fn kind(&self) -> RtrTransportKind {
|
||||
RtrTransportKind::Ssh
|
||||
}
|
||||
|
||||
fn handle_connection(
|
||||
&self,
|
||||
@@ -187,6 +198,7 @@ pub struct RtrServer {
|
||||
connection_limiter: Arc<Semaphore>,
|
||||
handshake_limiter: Arc<Semaphore>,
|
||||
active_connections: Arc<AtomicUsize>,
|
||||
transport_connections: Arc<RtrTransportConnectionCounters>,
|
||||
config: RtrServiceConfig,
|
||||
}
|
||||
|
||||
@@ -199,6 +211,7 @@ impl RtrServer {
|
||||
connection_limiter: Arc<Semaphore>,
|
||||
handshake_limiter: Arc<Semaphore>,
|
||||
active_connections: Arc<AtomicUsize>,
|
||||
transport_connections: Arc<RtrTransportConnectionCounters>,
|
||||
config: RtrServiceConfig,
|
||||
) -> Self {
|
||||
Self {
|
||||
@@ -209,6 +222,7 @@ impl RtrServer {
|
||||
connection_limiter,
|
||||
handshake_limiter,
|
||||
active_connections,
|
||||
transport_connections,
|
||||
config,
|
||||
}
|
||||
}
|
||||
@@ -358,7 +372,9 @@ impl RtrServer {
|
||||
let notify_tx = self.notify_tx.clone();
|
||||
let shutdown_tx = self.shutdown_tx.clone();
|
||||
let active_connections = self.active_connections.clone();
|
||||
let transport_connections = self.transport_connections.clone();
|
||||
let transport_instance = transport.clone();
|
||||
let transport_kind = transport_instance.kind();
|
||||
|
||||
debug!(
|
||||
"RTR {} client connected: peer_addr={}, active_connections(before_spawn)={}",
|
||||
@@ -368,7 +384,12 @@ impl RtrServer {
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let guard = ConnectionGuard::new(active_connections, permit);
|
||||
let guard = ConnectionGuard::new(
|
||||
active_connections,
|
||||
transport_connections,
|
||||
transport_kind,
|
||||
permit,
|
||||
);
|
||||
info!(
|
||||
"RTR {} connection established: peer_addr={}, active_connections={}",
|
||||
transport_instance.name(),
|
||||
|
||||
@@ -7,7 +7,8 @@ pub mod ssh;
|
||||
pub mod tls;
|
||||
|
||||
pub use config::RtrServiceConfig;
|
||||
pub use connection::RtrTransportConnectionCounts;
|
||||
pub use listener::RtrServer;
|
||||
pub use notifier::RtrNotifier;
|
||||
pub use service::{RtrService, RunningRtrService};
|
||||
pub use service::{RtrService, RtrServiceStats, RunningRtrService};
|
||||
pub use tls::load_rustls_server_config;
|
||||
|
||||
@@ -11,6 +11,9 @@ use tracing::{error, warn};
|
||||
|
||||
use crate::rtr::cache::SharedRtrCache;
|
||||
use crate::rtr::server::config::RtrServiceConfig;
|
||||
use crate::rtr::server::connection::{
|
||||
RtrTransportConnectionCounters, RtrTransportConnectionCounts,
|
||||
};
|
||||
use crate::rtr::server::listener::RtrServer;
|
||||
use crate::rtr::server::notifier::RtrNotifier;
|
||||
use crate::rtr::server::ssh::{SshAuthMode, load_rtr_ssh_runtime_config};
|
||||
@@ -22,9 +25,31 @@ pub struct RtrService {
|
||||
connection_limiter: Arc<Semaphore>,
|
||||
handshake_limiter: Arc<Semaphore>,
|
||||
active_connections: Arc<AtomicUsize>,
|
||||
transport_connections: Arc<RtrTransportConnectionCounters>,
|
||||
config: RtrServiceConfig,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct RtrServiceStats {
|
||||
active_connections: Arc<AtomicUsize>,
|
||||
transport_connections: Arc<RtrTransportConnectionCounters>,
|
||||
max_connections: usize,
|
||||
}
|
||||
|
||||
impl RtrServiceStats {
|
||||
pub fn active_connections(&self) -> usize {
|
||||
self.active_connections.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn transport_connections(&self) -> RtrTransportConnectionCounts {
|
||||
self.transport_connections.snapshot()
|
||||
}
|
||||
|
||||
pub fn max_connections(&self) -> usize {
|
||||
self.max_connections
|
||||
}
|
||||
}
|
||||
|
||||
impl RtrService {
|
||||
pub fn new(cache: SharedRtrCache) -> Self {
|
||||
Self::with_config(cache, RtrServiceConfig::default())
|
||||
@@ -41,6 +66,7 @@ impl RtrService {
|
||||
connection_limiter: Arc::new(Semaphore::new(config.max_connections)),
|
||||
handshake_limiter: Arc::new(Semaphore::new(config.max_concurrent_handshakes)),
|
||||
active_connections: Arc::new(AtomicUsize::new(0)),
|
||||
transport_connections: Arc::new(RtrTransportConnectionCounters::default()),
|
||||
config,
|
||||
}
|
||||
}
|
||||
@@ -61,10 +87,22 @@ impl RtrService {
|
||||
self.active_connections.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn transport_connections(&self) -> RtrTransportConnectionCounts {
|
||||
self.transport_connections.snapshot()
|
||||
}
|
||||
|
||||
pub fn max_connections(&self) -> usize {
|
||||
self.config.max_connections
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> RtrServiceStats {
|
||||
RtrServiceStats {
|
||||
active_connections: self.active_connections.clone(),
|
||||
transport_connections: self.transport_connections.clone(),
|
||||
max_connections: self.config.max_connections,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn tcp_server(&self, bind_addr: SocketAddr) -> RtrServer {
|
||||
RtrServer::new(
|
||||
bind_addr,
|
||||
@@ -74,6 +112,7 @@ impl RtrService {
|
||||
self.connection_limiter.clone(),
|
||||
self.handshake_limiter.clone(),
|
||||
self.active_connections.clone(),
|
||||
self.transport_connections.clone(),
|
||||
self.config.clone(),
|
||||
)
|
||||
}
|
||||
@@ -87,6 +126,7 @@ impl RtrService {
|
||||
self.connection_limiter.clone(),
|
||||
self.handshake_limiter.clone(),
|
||||
self.active_connections.clone(),
|
||||
self.transport_connections.clone(),
|
||||
self.config.clone(),
|
||||
)
|
||||
}
|
||||
@@ -100,6 +140,7 @@ impl RtrService {
|
||||
self.connection_limiter.clone(),
|
||||
self.handshake_limiter.clone(),
|
||||
self.active_connections.clone(),
|
||||
self.transport_connections.clone(),
|
||||
self.config.clone(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -69,7 +69,10 @@ pub fn load_rtr_ssh_runtime_config(
|
||||
};
|
||||
let password = password.map(str::trim).filter(|value| !value.is_empty());
|
||||
if matches!(auth_mode, SshAuthMode::Password | SshAuthMode::Both) && password.is_none() {
|
||||
bail!("SSH auth mode '{}' requires non-empty password", auth_mode.as_str());
|
||||
bail!(
|
||||
"SSH auth mode '{}' requires non-empty password",
|
||||
auth_mode.as_str()
|
||||
);
|
||||
}
|
||||
|
||||
let mut methods = MethodSet::empty();
|
||||
|
||||
Reference in New Issue
Block a user