NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260803 死repo传输层黑名单+daemon健康检查摘除(#141)

Browse Source
This commit is contained in:
yuyr
2026-08-03 21:17:19 +08:00
parent f37fe8c4cb
commit e98eb6d68f
16 changed files with 1990 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

432
src/tools/rpki_daemon.rs
View File

@@ -6,6 +6,9 @@ use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::Duration;
use crate::parallel::dead_repo_blacklist::DeadRepoBlacklist;
use crate::parallel::types::RepoTransportMode;
#[derive(Clone, Debug, PartialEq, Eq)]
struct Args {
state_root: PathBuf,
@@ -21,6 +24,12 @@ struct Args {
db_stats_bin: Option<PathBuf>,
db_stats_exact_every: Option<u64>,
time_bin: Option<PathBuf>,
/// Dead-repo blacklist (#141): enables daemon-side health checks and
/// auto-injects the child `--dead-repo-blacklist` flag when absent.
dead_repo_blacklist: Option<PathBuf>,
dead_repo_blacklist_fail_threshold: Option<u32>,
dead_repo_health_check_interval_secs: u64,
dead_repo_probe_rsync_bin: PathBuf,
child_args: Vec<String>,
}
@@ -52,6 +61,23 @@ struct DaemonStatus {
current_run_seq: Option<u64>,
current_run_id: Option<String>,
last_run_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
dead_repo_blacklist: Option<DeadRepoBlacklistStatus>,
}
#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
struct DeadRepoBlacklistStatus {
path: String,
entries: usize,
blacklisted: usize,
last_health_check_at_rfc3339_utc: Option<String>,
}
/// Mutable daemon-side health check bookkeeping (#141).
#[derive(Clone, Debug, Default, PartialEq, Eq)]
struct DeadRepoHealthRuntime {
last_health_check_at: Option<time::OffsetDateTime>,
}
#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
@@ -145,6 +171,16 @@ struct RunSummary {
db_stats: Vec<DbStatsSummary>,
retention_deleted_runs: Vec<String>,
artifacts: Vec<ArtifactInfo>,
#[serde(skip_serializing_if = "Option::is_none")]
dead_repo_blacklist: Option<DeadRepoBlacklistRunSummary>,
}
#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
struct DeadRepoBlacklistRunSummary {
path: String,
entries: usize,
blacklisted: usize,
}
fn usage() -> String {
@@ -170,6 +206,17 @@ Options:
Run db_stats --exact every n runs (default: disabled)
--time-bin <path> GNU time binary for child process metrics (default: /usr/bin/time when present)
--no-time-wrapper Disable GNU time wrapper
--dead-repo-blacklist <path>
Enable the dead-repo transport blacklist (#141): daemon probes
blacklisted entries between runs and removes revived repos;
auto-injects the child flag when not already present
--dead-repo-blacklist-fail-threshold <n>
Admission threshold forwarded to the child when its args do not
already set it (default: child default 3)
--dead-repo-health-check-interval-secs <n>
Minimum seconds between health check sweeps (default: 600)
--dead-repo-probe-rsync-bin <path>
rsync binary used for health probes (default: rsync)
--help Show this help
Child argument placeholders:
@@ -211,12 +258,16 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
let mut db_stats_exact_every = None;
let mut time_bin = default_time_bin();
let mut no_time_wrapper = false;
let mut dead_repo_blacklist: Option<PathBuf> = None;
let mut dead_repo_blacklist_fail_threshold: Option<u32> = None;
let mut dead_repo_health_check_interval_secs = 600u64;
let mut dead_repo_probe_rsync_bin = PathBuf::from("rsync");
let mut i = 1usize;
while i < argv.len() {
match argv[i].as_str() {
"--" => {
let child_args = argv[i + 1..].to_vec();
let mut child_args = argv[i + 1..].to_vec();
let state_root =
state_root.ok_or_else(|| format!("--state-root is required\n\n{}", usage()))?;
let work_db = work_db.unwrap_or_else(|| state_root.join("state").join("work-db"));
@@ -225,6 +276,23 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
if no_time_wrapper {
time_bin = None;
}
// Dead-repo blacklist (#141): forward enablement to the child
// unless its args already carry the flag explicitly.
if let Some(path) = dead_repo_blacklist.as_ref() {
if !child_args.iter().any(|arg| arg == "--dead-repo-blacklist") {
child_args.push("--dead-repo-blacklist".to_string());
child_args.push(path_string(path));
}
if let Some(threshold) = dead_repo_blacklist_fail_threshold {
if !child_args
.iter()
.any(|arg| arg == "--dead-repo-blacklist-fail-threshold")
{
child_args.push("--dead-repo-blacklist-fail-threshold".to_string());
child_args.push(threshold.to_string());
}
}
}
let args = Args {
state_root,
rpki_bin: rpki_bin
@@ -240,6 +308,10 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
db_stats_bin,
db_stats_exact_every,
time_bin,
dead_repo_blacklist,
dead_repo_blacklist_fail_threshold,
dead_repo_health_check_interval_secs,
dead_repo_probe_rsync_bin,
child_args,
};
return validate_args(args);
@@ -315,6 +387,33 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
"--no-time-wrapper" => {
no_time_wrapper = true;
}
"--dead-repo-blacklist" => {
i += 1;
dead_repo_blacklist = Some(PathBuf::from(value_at(argv, i, "--dead-repo-blacklist")?));
}
"--dead-repo-blacklist-fail-threshold" => {
i += 1;
let parsed = parse_u64(
value_at(argv, i, "--dead-repo-blacklist-fail-threshold")?,
"--dead-repo-blacklist-fail-threshold",
)?;
if parsed == 0 || parsed > u32::MAX as u64 {
return Err("--dead-repo-blacklist-fail-threshold must be in 1..=u32::MAX".to_string());
}
dead_repo_blacklist_fail_threshold = Some(parsed as u32);
}
"--dead-repo-health-check-interval-secs" => {
i += 1;
dead_repo_health_check_interval_secs = parse_u64(
value_at(argv, i, "--dead-repo-health-check-interval-secs")?,
"--dead-repo-health-check-interval-secs",
)?;
}
"--dead-repo-probe-rsync-bin" => {
i += 1;
dead_repo_probe_rsync_bin =
PathBuf::from(value_at(argv, i, "--dead-repo-probe-rsync-bin")?);
}
other => return Err(format!("unknown argument: {other}\n\n{}", usage())),
}
i += 1;
@@ -330,6 +429,11 @@ fn validate_args(args: Args) -> Result<Args, String> {
usage()
));
}
if args.dead_repo_blacklist_fail_threshold.is_some() && args.dead_repo_blacklist.is_none() {
return Err(
"--dead-repo-blacklist-fail-threshold requires --dead-repo-blacklist".to_string(),
);
}
Ok(args)
}
@@ -387,10 +491,19 @@ fn write_json_pretty<T: Serialize>(path: &Path, value: &T) -> Result<(), String>
fs::create_dir_all(parent)
.map_err(|e| format!("create parent dir failed: {}: {e}", parent.display()))?;
}
let file =
File::create(path).map_err(|e| format!("create json failed: {}: {e}", path.display()))?;
serde_json::to_writer_pretty(file, value)
.map_err(|e| format!("write json failed: {}: {e}", path.display()))
let bytes = serde_json::to_vec_pretty(value)
.map_err(|e| format!("serialize json failed: {}: {e}", path.display()))?;
// Atomic write (tmp + rename) so concurrent readers never see a torn file.
let mut tmp_name = path
.file_name()
.map(|name| name.to_os_string())
.unwrap_or_default();
tmp_name.push(".tmp");
let tmp_path = path.with_file_name(tmp_name);
fs::write(&tmp_path, &bytes)
.map_err(|e| format!("write json tmp failed: {}: {e}", tmp_path.display()))?;
fs::rename(&tmp_path, path)
.map_err(|e| format!("rename json tmp failed: {} -> {}: {e}", tmp_path.display(), path.display()))
}
fn append_json_line<T: Serialize>(path: &Path, value: &T) -> Result<(), String> {
@@ -409,12 +522,30 @@ fn append_json_line<T: Serialize>(path: &Path, value: &T) -> Result<(), String>
.map_err(|e| format!("flush jsonl failed: {}: {e}", path.display()))
}
fn dead_repo_blacklist_status(
args: &Args,
health: &DeadRepoHealthRuntime,
) -> Option<DeadRepoBlacklistStatus> {
let path = args.dead_repo_blacklist.as_ref()?;
let (blacklist, _) = DeadRepoBlacklist::load(path);
let last_health_check_at_rfc3339_utc = health
.last_health_check_at
.and_then(|t| format_rfc3339(t).ok());
Some(DeadRepoBlacklistStatus {
path: path_string(path),
entries: blacklist.len(),
blacklisted: blacklist.blacklisted_len(),
last_health_check_at_rfc3339_utc,
})
}
fn write_status(
args: &Args,
state: DaemonState,
runs_completed: u64,
current: Option<&RunContext>,
last_run_id: Option<String>,
health: &DeadRepoHealthRuntime,
) -> Result<(), String> {
let updated_at_rfc3339_utc = format_rfc3339(utc_now())?;
let status = DaemonStatus {
@@ -425,10 +556,131 @@ fn write_status(
current_run_seq: current.map(|ctx| ctx.seq),
current_run_id: current.map(|ctx| ctx.run_id.clone()),
last_run_id,
dead_repo_blacklist: dead_repo_blacklist_status(args, health),
};
write_json_pretty(&status_path(args), &status)
}
/// Short-timeout HTTP GET of the RRDP notification file. Any HTTP status
/// (even 4xx) proves the transport is alive; only transport errors keep the
/// entry blacklisted.
fn probe_rrdp_transport(uri: &str) -> bool {
let config = crate::fetch::http::HttpFetcherConfig {
connect_timeout: Duration::from_secs(3),
timeout: Duration::from_secs(6),
large_body_timeout: Duration::from_secs(6),
..Default::default()
};
let fetcher = match crate::fetch::http::BlockingHttpFetcher::new(config) {
Ok(fetcher) => fetcher,
Err(_) => return false,
};
match fetcher.fetch_bytes(uri) {
Ok(_) => true,
Err(err) => err.starts_with("http status"),
}
}
/// rsync list-only probe: connecting to the daemon and listing the module
/// root is enough to prove the transport is alive.
fn probe_rsync_transport(rsync_bin: &Path, base_uri: &str) -> bool {
Command::new(rsync_bin)
.arg("--contimeout=5")
.arg("--timeout=8")
.arg(base_uri)
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.map(|status| status.success())
.unwrap_or(false)
}
/// Probe due blacklist entries between runs (#141). Runs in the supervisor
/// loop while no child is active, preserving the single-writer assumption.
fn maybe_run_dead_repo_health_check(args: &Args, health: &mut DeadRepoHealthRuntime) {
let Some(path) = args.dead_repo_blacklist.clone() else {
return;
};
let now = utc_now();
if let Some(last) = health.last_health_check_at {
let elapsed = (now - last).whole_seconds();
if elapsed >= 0 && (elapsed as u64) < args.dead_repo_health_check_interval_secs {
return;
}
}
let (mut blacklist, warning) = DeadRepoBlacklist::load(&path);
if let Some(warning) = warning {
eprintln!("[dead-repo-health] {warning}");
}
let now_unix = now.unix_timestamp().max(0) as u64;
let due = blacklist.probe_due_entries(now_unix, args.dead_repo_health_check_interval_secs);
if due.is_empty() {
health.last_health_check_at = Some(now);
return;
}
// Probe in parallel: in the docker soak the daemon runs once per run
// (--max-runs 1), so a serial sweep of ~12 dead entries would add up to a
// minute of latency to every run.
let rsync_bin = args.dead_repo_probe_rsync_bin.clone();
let probe_results: Vec<(RepoTransportMode, String, bool)> = due
.iter()
.map(|entry| (entry.transport, entry.uri.clone()))
.collect::<Vec<_>>()
.into_iter()
.map(|(transport, uri)| {
let rsync_bin = rsync_bin.clone();
std::thread::spawn(move || {
let alive = match transport {
RepoTransportMode::Rrdp => probe_rrdp_transport(&uri),
RepoTransportMode::Rsync => probe_rsync_transport(&rsync_bin, &uri),
};
(transport, uri, alive)
})
})
.filter_map(|handle| handle.join().ok())
.collect();
let mut removed = 0usize;
let mut still_dead = 0usize;
for (transport, uri, alive) in &probe_results {
if *alive {
blacklist.record_probe_success(*transport, uri);
removed += 1;
eprintln!("[dead-repo-health] removed {} {} (probe ok)", transport.as_str(), uri);
crate::progress_log::emit(
"dead_repo_blacklist_remove",
serde_json::json!({
"transport": transport.as_str(),
"uri": uri,
"reason": "health_probe_ok",
}),
);
} else {
blacklist.record_probe_failure(*transport, uri, now_unix);
still_dead += 1;
eprintln!(
"[dead-repo-health] still dead {} {} (probe failed)",
transport.as_str(),
uri
);
}
}
if let Err(err) = blacklist.store_atomic(&path, now_unix) {
eprintln!(
"[dead-repo-health] persist blacklist failed: {}: {err}",
path.display()
);
}
crate::progress_log::emit(
"dead_repo_health_check",
serde_json::json!({
"probed": probe_results.len(),
"removed": removed,
"still_dead": still_dead,
}),
);
health.last_health_check_at = Some(now);
}
fn render_child_args(args: &[String], daemon_args: &Args, ctx: &RunContext) -> Vec<String> {
args.iter()
.map(|arg| {
@@ -634,6 +886,7 @@ fn run_child_once(args: &Args, ctx: &RunContext) -> Result<RunSummary, String> {
db_stats: Vec::new(),
retention_deleted_runs: Vec::new(),
artifacts,
dead_repo_blacklist: None,
};
Ok(summary)
}
@@ -1031,6 +1284,14 @@ fn collect_post_run_metrics(args: &Args, ctx: &RunContext, summary: &mut RunSumm
summary.path_stats = collect_state_path_stats(args, ctx);
summary.db_stats = collect_db_stats(args, ctx);
summary.artifacts = collect_artifacts(&ctx.run_dir).unwrap_or_default();
if let Some(path) = args.dead_repo_blacklist.as_ref() {
let (blacklist, _) = DeadRepoBlacklist::load(path);
summary.dead_repo_blacklist = Some(DeadRepoBlacklistRunSummary {
path: path_string(path),
entries: blacklist.len(),
blacklisted: blacklist.blacklisted_len(),
});
}
}
fn run_daemon(args: &Args) -> Result<(), String> {
@@ -1050,13 +1311,17 @@ fn run_daemon(args: &Args) -> Result<(), String> {
let mut runs_completed = 0u64;
let mut next_seq = 1u64;
let mut last_run_id = None;
let mut health = DeadRepoHealthRuntime::default();
write_status(
args,
DaemonState::Starting,
runs_completed,
None,
last_run_id.clone(),
&health,
)?;
// Probe once at startup so revived repos are removed before the first run.
maybe_run_dead_repo_health_check(args, &mut health);
loop {
if args.max_runs.is_some_and(|max| runs_completed >= max) {
@@ -1069,6 +1334,7 @@ fn run_daemon(args: &Args) -> Result<(), String> {
runs_completed,
None,
last_run_id.clone(),
&health,
)?;
let ctx = make_run_context(args, next_seq, utc_now());
write_status(
@@ -1077,6 +1343,7 @@ fn run_daemon(args: &Args) -> Result<(), String> {
runs_completed,
Some(&ctx),
last_run_id.clone(),
&health,
)?;
let mut summary = run_child_once(args, &ctx)?;
write_status(
@@ -1085,6 +1352,7 @@ fn run_daemon(args: &Args) -> Result<(), String> {
runs_completed,
Some(&ctx),
last_run_id.clone(),
&health,
)?;
collect_post_run_metrics(args, &ctx, &mut summary);
let removed = apply_retention(&args.state_root.join("runs"), args.retain_runs)?;
@@ -1099,19 +1367,29 @@ fn run_daemon(args: &Args) -> Result<(), String> {
if args.max_runs.is_some_and(|max| runs_completed >= max) {
break;
}
// Health probes run only while no child is active (single-writer).
maybe_run_dead_repo_health_check(args, &mut health);
write_status(
args,
DaemonState::Sleeping,
runs_completed,
None,
last_run_id.clone(),
&health,
)?;
if args.interval_secs > 0 {
std::thread::sleep(Duration::from_secs(args.interval_secs));
}
}
write_status(args, DaemonState::Exited, runs_completed, None, last_run_id)
write_status(
args,
DaemonState::Exited,
runs_completed,
None,
last_run_id,
&health,
)
}
pub fn main_entry() -> i32 {
@@ -1154,6 +1432,10 @@ mod tests {
db_stats_bin: None,
db_stats_exact_every: None,
time_bin: None,
dead_repo_blacklist: None,
dead_repo_blacklist_fail_threshold: None,
dead_repo_health_check_interval_secs: 600,
dead_repo_probe_rsync_bin: PathBuf::from("rsync"),
child_args: vec!["--version".to_string()],
}
}
@@ -1330,6 +1612,10 @@ mod tests {
db_stats_bin: None,
db_stats_exact_every: None,
time_bin: None,
dead_repo_blacklist: None,
dead_repo_blacklist_fail_threshold: None,
dead_repo_health_check_interval_secs: 600,
dead_repo_probe_rsync_bin: PathBuf::from("rsync"),
child_args: vec![
"{state_root}/state/work-db".to_string(),
"{run_out}/result.ccr".to_string(),
@@ -1652,6 +1938,10 @@ mod tests {
db_stats_bin: None,
db_stats_exact_every: None,
time_bin: None,
dead_repo_blacklist: None,
dead_repo_blacklist_fail_threshold: None,
dead_repo_health_check_interval_secs: 600,
dead_repo_probe_rsync_bin: PathBuf::from("rsync"),
child_args: vec![
"-c".to_string(),
"echo stdout-{run_seq}; echo stderr-{run_seq} >&2; echo marker > {run_out}/marker.txt".to_string(),
@@ -1743,6 +2033,10 @@ mod tests {
db_stats_bin: Some(db_stats_bin),
db_stats_exact_every: Some(1),
time_bin: None,
dead_repo_blacklist: None,
dead_repo_blacklist_fail_threshold: None,
dead_repo_health_check_interval_secs: 600,
dead_repo_probe_rsync_bin: PathBuf::from("rsync"),
child_args: vec![
"-c".to_string(),
"mkdir -p {state_root}/state/work-db {state_root}/state/repo-bytes.db; \
@@ -1781,4 +2075,130 @@ mod tests {
.any(|item| item["label"] == "work_db" && item["exists"] == true)
);
}
#[test]
fn dead_repo_blacklist_flag_is_injected_into_child_args() {
let argv = vec![
"rpki_daemon".to_string(),
"--state-root".to_string(),
"/tmp/daemon".to_string(),
"--rpki-bin".to_string(),
"/bin/echo".to_string(),
"--dead-repo-blacklist".to_string(),
"/tmp/daemon/state/dead-repo-blacklist.json".to_string(),
"--dead-repo-blacklist-fail-threshold".to_string(),
"2".to_string(),
"--".to_string(),
"--db".to_string(),
"{state_root}/state/work-db".to_string(),
];
let args = parse_args(&argv).expect("parse");
assert_eq!(
args.child_args,
vec![
"--db".to_string(),
"{state_root}/state/work-db".to_string(),
"--dead-repo-blacklist".to_string(),
"/tmp/daemon/state/dead-repo-blacklist.json".to_string(),
"--dead-repo-blacklist-fail-threshold".to_string(),
"2".to_string(),
]
);
}
#[test]
fn dead_repo_blacklist_injection_respects_existing_child_flags() {
let argv = vec![
"rpki_daemon".to_string(),
"--state-root".to_string(),
"/tmp/daemon".to_string(),
"--rpki-bin".to_string(),
"/bin/echo".to_string(),
"--dead-repo-blacklist".to_string(),
"/tmp/daemon/state/dead-repo-blacklist.json".to_string(),
"--".to_string(),
"--dead-repo-blacklist".to_string(),
"/custom/path.json".to_string(),
];
let args = parse_args(&argv).expect("parse");
assert_eq!(
args.child_args,
vec![
"--dead-repo-blacklist".to_string(),
"/custom/path.json".to_string(),
]
);
}
#[test]
fn dead_repo_threshold_without_blacklist_is_rejected() {
let argv = vec![
"rpki_daemon".to_string(),
"--state-root".to_string(),
"/tmp/daemon".to_string(),
"--rpki-bin".to_string(),
"/bin/echo".to_string(),
"--dead-repo-blacklist-fail-threshold".to_string(),
"2".to_string(),
"--".to_string(),
"--version".to_string(),
];
let err = parse_args(&argv).expect_err("parse should fail");
assert!(err.contains("requires --dead-repo-blacklist"), "{err}");
}
#[test]
fn health_check_keeps_dead_entry_and_removes_revived_entry() {
use std::io::{Read as _, Write as _};
use std::net::TcpListener;
// Revived RRDP host: one-shot HTTP server answering 200.
let listener = TcpListener::bind("127.0.0.1:0").expect("bind");
let port = listener.local_addr().expect("local addr").port();
std::thread::spawn(move || {
if let Ok((mut stream, _)) = listener.accept() {
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf);
let body = b"<notification/>";
let _ = stream.write_all(
format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
)
.as_bytes(),
);
let _ = stream.write_all(body);
}
});
let td = tempfile::tempdir().expect("tempdir");
let blacklist_path = td.path().join("dead-repo-blacklist.json");
let revived_uri = format!("http://127.0.0.1:{port}/notification.xml");
let dead_uri = "http://127.0.0.1:1/notification.xml".to_string();
let mut blacklist = DeadRepoBlacklist::new();
for uri in [&revived_uri, &dead_uri] {
blacklist.record_transport_failure(RepoTransportMode::Rrdp, uri, 100, 1, 256);
}
blacklist
.store_atomic(&blacklist_path, 100)
.expect("store blacklist");
let mut args = test_args(td.path().join("daemon"));
args.dead_repo_blacklist = Some(blacklist_path.clone());
args.dead_repo_health_check_interval_secs = 0;
let mut health = DeadRepoHealthRuntime::default();
maybe_run_dead_repo_health_check(&args, &mut health);
let (after, warning) = DeadRepoBlacklist::load(&blacklist_path);
assert!(warning.is_none());
assert!(!after.is_blacklisted(RepoTransportMode::Rrdp, &revived_uri));
assert!(after.is_blacklisted(RepoTransportMode::Rrdp, &dead_uri));
assert!(health.last_health_check_at.is_some());
// Status JSON exposes the blacklist section.
let status = dead_repo_blacklist_status(&args, &health).expect("status");
assert_eq!(status.entries, 1);
assert_eq!(status.blacklisted, 1);
assert!(status.last_health_check_at_rfc3339_utc.is_some());
}
}