20260803 死repo传输层黑名单+daemon健康检查摘除(#141)
This commit is contained in:
parent
f37fe8c4cb
commit
e98eb6d68f
@ -66,6 +66,14 @@ PERIODIC_SNAPSHOT_MAX_DELTAS=100
|
|||||||
# 每隔多少轮执行一次 db_stats --exact。设置为空或 0 表示关闭 exact 统计。
|
# 每隔多少轮执行一次 db_stats --exact。设置为空或 0 表示关闭 exact 统计。
|
||||||
DB_STATS_EXACT_EVERY=3
|
DB_STATS_EXACT_EVERY=3
|
||||||
|
|
||||||
|
# 死 repo 传输层黑名单(#141,默认关)。设置状态文件路径即开启:
|
||||||
|
# 连续 N 轮传输层抓取失败的 repo 会被拉黑并在后续轮次跳过其传输尝试;
|
||||||
|
# daemon 会按间隔对黑名单条目做健康探针,恢复后自动摘除。
|
||||||
|
# 留空表示完全关闭(行为与旧版本一致)。
|
||||||
|
#DEAD_REPO_BLACKLIST_PATH=/var/lib/ours-rp/state/dead-repo-blacklist.json
|
||||||
|
#DEAD_REPO_BLACKLIST_FAIL_THRESHOLD=3
|
||||||
|
#DEAD_REPO_HEALTH_CHECK_INTERVAL_SECS=600
|
||||||
|
|
||||||
# 是否开启 ours RP progress log。1 表示开启。
|
# 是否开启 ours RP progress log。1 表示开启。
|
||||||
RPKI_PROGRESS_LOG=1
|
RPKI_PROGRESS_LOG=1
|
||||||
|
|
||||||
|
|||||||
@ -1331,6 +1331,17 @@ run_one_round() {
|
|||||||
daemon_args+=(--db-stats-exact-every "$DB_STATS_EXACT_EVERY")
|
daemon_args+=(--db-stats-exact-every "$DB_STATS_EXACT_EVERY")
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
# Dead-repo transport blacklist (#141): opt-in via env; the daemon forwards
|
||||||
|
# the enable flag and threshold to the child unless already present there.
|
||||||
|
if [[ -n "${DEAD_REPO_BLACKLIST_PATH:-}" ]]; then
|
||||||
|
daemon_args+=(--dead-repo-blacklist "$DEAD_REPO_BLACKLIST_PATH")
|
||||||
|
if [[ -n "${DEAD_REPO_BLACKLIST_FAIL_THRESHOLD:-}" ]]; then
|
||||||
|
daemon_args+=(--dead-repo-blacklist-fail-threshold "$DEAD_REPO_BLACKLIST_FAIL_THRESHOLD")
|
||||||
|
fi
|
||||||
|
if [[ -n "${DEAD_REPO_HEALTH_CHECK_INTERVAL_SECS:-}" ]]; then
|
||||||
|
daemon_args+=(--dead-repo-health-check-interval-secs "$DEAD_REPO_HEALTH_CHECK_INTERVAL_SECS")
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
set +e
|
set +e
|
||||||
env \
|
env \
|
||||||
|
|||||||
45
src/cli.rs
45
src/cli.rs
@ -279,6 +279,13 @@ Options:
|
|||||||
--control-plane-stage-workers <n>
|
--control-plane-stage-workers <n>
|
||||||
Experimental: Phase 2 ready publication point stage worker count;
|
Experimental: Phase 2 ready publication point stage worker count;
|
||||||
0 disables the stage pool and keeps inline staging (default: 0)
|
0 disables the stage pool and keeps inline staging (default: 0)
|
||||||
|
--dead-repo-blacklist <path>
|
||||||
|
Enable the dead-repo transport blacklist persisted at this JSON
|
||||||
|
path (default: disabled). Blacklisted rrdp repos skip straight to
|
||||||
|
rsync; dual-blacklisted repos terminate instantly.
|
||||||
|
--dead-repo-blacklist-fail-threshold <n>
|
||||||
|
Consecutive runs with transport-class fetch failure before a
|
||||||
|
(repo, transport) entry is blacklisted (default: 3)
|
||||||
|
|
||||||
--rsync-local-dir <path> Use LocalDirRsyncFetcher rooted at this directory (offline tests)
|
--rsync-local-dir <path> Use LocalDirRsyncFetcher rooted at this directory (offline tests)
|
||||||
--disable-rrdp Disable RRDP and synchronize only via rsync
|
--disable-rrdp Disable RRDP and synchronize only via rsync
|
||||||
@ -313,6 +320,8 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
|||||||
let mut ta_paths: Vec<PathBuf> = Vec::new();
|
let mut ta_paths: Vec<PathBuf> = Vec::new();
|
||||||
let mut parallel_phase1_cfg = ParallelPhase1Config::default();
|
let mut parallel_phase1_cfg = ParallelPhase1Config::default();
|
||||||
let mut parallel_phase2_cfg = ParallelPhase2Config::default();
|
let mut parallel_phase2_cfg = ParallelPhase2Config::default();
|
||||||
|
let mut dead_repo_blacklist_path: Option<PathBuf> = None;
|
||||||
|
let mut dead_repo_blacklist_fail_threshold: Option<u32> = None;
|
||||||
|
|
||||||
let mut db_path: Option<PathBuf> = None;
|
let mut db_path: Option<PathBuf> = None;
|
||||||
let mut raw_store_db: Option<PathBuf> = None;
|
let mut raw_store_db: Option<PathBuf> = None;
|
||||||
@ -531,6 +540,23 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
|||||||
.parse::<usize>()
|
.parse::<usize>()
|
||||||
.map_err(|_| format!("invalid --control-plane-stage-workers: {v}"))?;
|
.map_err(|_| format!("invalid --control-plane-stage-workers: {v}"))?;
|
||||||
}
|
}
|
||||||
|
"--dead-repo-blacklist" => {
|
||||||
|
i += 1;
|
||||||
|
let v = argv
|
||||||
|
.get(i)
|
||||||
|
.ok_or("--dead-repo-blacklist requires a value")?;
|
||||||
|
dead_repo_blacklist_path = Some(PathBuf::from(v));
|
||||||
|
}
|
||||||
|
"--dead-repo-blacklist-fail-threshold" => {
|
||||||
|
i += 1;
|
||||||
|
let v = argv
|
||||||
|
.get(i)
|
||||||
|
.ok_or("--dead-repo-blacklist-fail-threshold requires a value")?;
|
||||||
|
dead_repo_blacklist_fail_threshold = Some(
|
||||||
|
v.parse::<u32>()
|
||||||
|
.map_err(|_| format!("invalid --dead-repo-blacklist-fail-threshold: {v}"))?,
|
||||||
|
);
|
||||||
|
}
|
||||||
"--db" => {
|
"--db" => {
|
||||||
i += 1;
|
i += 1;
|
||||||
let v = argv.get(i).ok_or("--db requires a value")?;
|
let v = argv.get(i).ok_or("--db requires a value")?;
|
||||||
@ -1122,6 +1148,25 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if dead_repo_blacklist_fail_threshold.is_some() && dead_repo_blacklist_path.is_none() {
|
||||||
|
return Err(
|
||||||
|
"--dead-repo-blacklist-fail-threshold requires --dead-repo-blacklist".to_string()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if let Some(path) = dead_repo_blacklist_path {
|
||||||
|
let fail_threshold = dead_repo_blacklist_fail_threshold.unwrap_or(3);
|
||||||
|
if fail_threshold == 0 {
|
||||||
|
return Err("--dead-repo-blacklist-fail-threshold must be >= 1".to_string());
|
||||||
|
}
|
||||||
|
parallel_phase1_cfg.dead_repo_blacklist =
|
||||||
|
Some(crate::parallel::dead_repo_blacklist::DeadRepoBlacklistConfig {
|
||||||
|
path,
|
||||||
|
fail_threshold,
|
||||||
|
capacity:
|
||||||
|
crate::parallel::dead_repo_blacklist::DEAD_REPO_BLACKLIST_DEFAULT_CAPACITY,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
Ok(CliArgs {
|
Ok(CliArgs {
|
||||||
verification_only,
|
verification_only,
|
||||||
verification_source_run_dir,
|
verification_source_run_dir,
|
||||||
|
|||||||
@ -1,8 +1,13 @@
|
|||||||
|
use crate::parallel::dead_repo_blacklist::DeadRepoBlacklistConfig;
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
pub struct ParallelPhase1Config {
|
pub struct ParallelPhase1Config {
|
||||||
pub max_repo_sync_workers_global: usize,
|
pub max_repo_sync_workers_global: usize,
|
||||||
pub max_inflight_snapshot_bytes_global: usize,
|
pub max_inflight_snapshot_bytes_global: usize,
|
||||||
pub max_pending_repo_results: usize,
|
pub max_pending_repo_results: usize,
|
||||||
|
/// Dead-repo transport blacklist (#141). `None` disables the feature
|
||||||
|
/// entirely (default, behavior unchanged).
|
||||||
|
pub dead_repo_blacklist: Option<DeadRepoBlacklistConfig>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
@ -42,6 +47,7 @@ impl Default for ParallelPhase1Config {
|
|||||||
max_repo_sync_workers_global: 4,
|
max_repo_sync_workers_global: 4,
|
||||||
max_inflight_snapshot_bytes_global: 512 * 1024 * 1024,
|
max_inflight_snapshot_bytes_global: 512 * 1024 * 1024,
|
||||||
max_pending_repo_results: 1024,
|
max_pending_repo_results: 1024,
|
||||||
|
dead_repo_blacklist: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
376
src/parallel/dead_repo_blacklist.rs
Normal file
376
src/parallel/dead_repo_blacklist.rs
Normal file
@ -0,0 +1,376 @@
|
|||||||
|
//! Persistent dead-repo transport blacklist (#141).
|
||||||
|
//!
|
||||||
|
//! Tracks per-(repo, transport) consecutive transport-layer fetch failures
|
||||||
|
//! across runs. Once an entry reaches the configured failure threshold it is
|
||||||
|
//! admitted to the blacklist; the live transport scheduler then skips the
|
||||||
|
//! dead transport (rrdp-blacklisted -> straight to rsync, dual-blacklisted ->
|
||||||
|
//! immediate terminal failure). Entries are removed either by a successful
|
||||||
|
//! fetch (self-heal) or by the daemon-side health probe.
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::fs;
|
||||||
|
use std::io;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use crate::parallel::types::{RepoTransportErrorClass, RepoTransportMode};
|
||||||
|
|
||||||
|
pub const DEAD_REPO_BLACKLIST_SCHEMA_VERSION: u32 = 1;
|
||||||
|
pub const DEAD_REPO_BLACKLIST_DEFAULT_CAPACITY: usize = 256;
|
||||||
|
/// Probe backoff is capped at interval * 2^PROBE_BACKOFF_CAP_SHIFT.
|
||||||
|
pub const PROBE_BACKOFF_CAP_SHIFT: u32 = 4;
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct DeadRepoBlacklistConfig {
|
||||||
|
pub path: PathBuf,
|
||||||
|
pub fail_threshold: u32,
|
||||||
|
pub capacity: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct DeadRepoBlacklistEntry {
|
||||||
|
pub transport: RepoTransportMode,
|
||||||
|
pub uri: String,
|
||||||
|
/// Consecutive runs with transport-class fetch failure. Reset on success.
|
||||||
|
pub consecutive_failures: u32,
|
||||||
|
/// True once `consecutive_failures` reached the admission threshold.
|
||||||
|
#[serde(default)]
|
||||||
|
pub blacklisted: bool,
|
||||||
|
#[serde(default)]
|
||||||
|
pub first_failure_at_unix: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub last_failure_at_unix: u64,
|
||||||
|
/// When the entry was admitted to the blacklist.
|
||||||
|
#[serde(default)]
|
||||||
|
pub added_at_unix: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub last_probe_at_unix: Option<u64>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub probe_failures: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum DeadRepoFailureOutcome {
|
||||||
|
/// Failure counted, entry not (or not yet) blacklisted.
|
||||||
|
Counted { consecutive_failures: u32 },
|
||||||
|
/// Entry just reached the threshold and was admitted.
|
||||||
|
Admitted,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum DeadRepoSuccessOutcome {
|
||||||
|
/// A counting (not yet admitted) entry was reset.
|
||||||
|
CounterReset,
|
||||||
|
/// A blacklisted entry was removed (repo revived).
|
||||||
|
BlacklistRemoved,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize)]
|
||||||
|
struct DeadRepoBlacklistFile {
|
||||||
|
schema_version: u32,
|
||||||
|
updated_at_unix: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
entries: Vec<DeadRepoBlacklistEntry>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// In-memory blacklist state for one writer at a time (child run, or daemon
|
||||||
|
/// supervisor between runs). Lookups go through a HashMap keyed by
|
||||||
|
/// (transport, uri).
|
||||||
|
#[derive(Clone, Debug, Default)]
|
||||||
|
pub struct DeadRepoBlacklist {
|
||||||
|
entries: HashMap<(RepoTransportMode, String), DeadRepoBlacklistEntry>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DeadRepoBlacklist {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Load from disk. A missing file or an unreadable/incompatible file
|
||||||
|
/// degrades to an empty blacklist (logged by the caller via `warning`).
|
||||||
|
pub fn load(path: &Path) -> (Self, Option<String>) {
|
||||||
|
let bytes = match fs::read(path) {
|
||||||
|
Ok(bytes) => bytes,
|
||||||
|
Err(err) if err.kind() == io::ErrorKind::NotFound => return (Self::new(), None),
|
||||||
|
Err(err) => {
|
||||||
|
return (
|
||||||
|
Self::new(),
|
||||||
|
Some(format!(
|
||||||
|
"dead repo blacklist {} unreadable ({err}); starting empty",
|
||||||
|
path.display()
|
||||||
|
)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
match serde_json::from_slice::<DeadRepoBlacklistFile>(&bytes) {
|
||||||
|
Ok(file) if file.schema_version == DEAD_REPO_BLACKLIST_SCHEMA_VERSION => {
|
||||||
|
let entries = file
|
||||||
|
.entries
|
||||||
|
.into_iter()
|
||||||
|
.map(|entry| ((entry.transport, entry.uri.clone()), entry))
|
||||||
|
.collect();
|
||||||
|
(Self { entries }, None)
|
||||||
|
}
|
||||||
|
Ok(file) => (
|
||||||
|
Self::new(),
|
||||||
|
Some(format!(
|
||||||
|
"dead repo blacklist {} schema_version {} != {}; starting empty",
|
||||||
|
path.display(),
|
||||||
|
file.schema_version,
|
||||||
|
DEAD_REPO_BLACKLIST_SCHEMA_VERSION
|
||||||
|
)),
|
||||||
|
),
|
||||||
|
Err(err) => (
|
||||||
|
Self::new(),
|
||||||
|
Some(format!(
|
||||||
|
"dead repo blacklist {} unparsable ({err}); starting empty",
|
||||||
|
path.display()
|
||||||
|
)),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persist atomically (tmp file + rename).
|
||||||
|
pub fn store_atomic(&self, path: &Path, now_unix: u64) -> io::Result<()> {
|
||||||
|
if let Some(parent) = path.parent() {
|
||||||
|
fs::create_dir_all(parent)?;
|
||||||
|
}
|
||||||
|
let mut entries: Vec<&DeadRepoBlacklistEntry> = self.entries.values().collect();
|
||||||
|
entries.sort_by(|a, b| a.uri.cmp(&b.uri).then(a.transport.cmp(&b.transport)));
|
||||||
|
let file = DeadRepoBlacklistFile {
|
||||||
|
schema_version: DEAD_REPO_BLACKLIST_SCHEMA_VERSION,
|
||||||
|
updated_at_unix: now_unix,
|
||||||
|
entries: entries.into_iter().cloned().collect(),
|
||||||
|
};
|
||||||
|
let bytes = serde_json::to_vec_pretty(&file)
|
||||||
|
.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
|
||||||
|
let mut tmp_name = path.file_name().map(|name| name.to_os_string()).unwrap_or_default();
|
||||||
|
tmp_name.push(".tmp");
|
||||||
|
let tmp_path = path.with_file_name(tmp_name);
|
||||||
|
fs::write(&tmp_path, &bytes)?;
|
||||||
|
// Best-effort durability before the rename.
|
||||||
|
if let Ok(file) = fs::File::open(&tmp_path) {
|
||||||
|
let _ = file.sync_all();
|
||||||
|
}
|
||||||
|
fs::rename(&tmp_path, path)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_blacklisted(&self, transport: RepoTransportMode, uri: &str) -> bool {
|
||||||
|
self.entries
|
||||||
|
.get(&(transport, uri.to_string()))
|
||||||
|
.map(|entry| entry.blacklisted)
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record a transport-class fetch failure. Returns whether the failure
|
||||||
|
/// newly admitted the entry to the blacklist.
|
||||||
|
pub fn record_transport_failure(
|
||||||
|
&mut self,
|
||||||
|
transport: RepoTransportMode,
|
||||||
|
uri: &str,
|
||||||
|
now_unix: u64,
|
||||||
|
fail_threshold: u32,
|
||||||
|
capacity: usize,
|
||||||
|
) -> DeadRepoFailureOutcome {
|
||||||
|
let key = (transport, uri.to_string());
|
||||||
|
if !self.entries.contains_key(&key) {
|
||||||
|
self.evict_if_full(capacity);
|
||||||
|
}
|
||||||
|
let threshold = fail_threshold.max(1);
|
||||||
|
let entry = self.entries.entry(key).or_insert_with(|| DeadRepoBlacklistEntry {
|
||||||
|
transport,
|
||||||
|
uri: uri.to_string(),
|
||||||
|
consecutive_failures: 0,
|
||||||
|
blacklisted: false,
|
||||||
|
first_failure_at_unix: now_unix,
|
||||||
|
last_failure_at_unix: now_unix,
|
||||||
|
added_at_unix: 0,
|
||||||
|
last_probe_at_unix: None,
|
||||||
|
probe_failures: 0,
|
||||||
|
});
|
||||||
|
entry.consecutive_failures = entry.consecutive_failures.saturating_add(1);
|
||||||
|
entry.last_failure_at_unix = now_unix;
|
||||||
|
if entry.first_failure_at_unix == 0 {
|
||||||
|
entry.first_failure_at_unix = now_unix;
|
||||||
|
}
|
||||||
|
if !entry.blacklisted && entry.consecutive_failures >= threshold {
|
||||||
|
entry.blacklisted = true;
|
||||||
|
entry.added_at_unix = now_unix;
|
||||||
|
// A fresh admission restarts probe bookkeeping.
|
||||||
|
entry.last_probe_at_unix = None;
|
||||||
|
entry.probe_failures = 0;
|
||||||
|
return DeadRepoFailureOutcome::Admitted;
|
||||||
|
}
|
||||||
|
DeadRepoFailureOutcome::Counted {
|
||||||
|
consecutive_failures: entry.consecutive_failures,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record a successful fetch: reset a counting entry, or remove a
|
||||||
|
/// blacklisted entry entirely (self-heal).
|
||||||
|
pub fn record_transport_success(
|
||||||
|
&mut self,
|
||||||
|
transport: RepoTransportMode,
|
||||||
|
uri: &str,
|
||||||
|
) -> Option<DeadRepoSuccessOutcome> {
|
||||||
|
match self.entries.remove(&(transport, uri.to_string())) {
|
||||||
|
Some(entry) if entry.blacklisted => Some(DeadRepoSuccessOutcome::BlacklistRemoved),
|
||||||
|
Some(_) => Some(DeadRepoSuccessOutcome::CounterReset),
|
||||||
|
None => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Blacklisted entries whose next probe is due at `now_unix`.
|
||||||
|
pub fn probe_due_entries(
|
||||||
|
&self,
|
||||||
|
now_unix: u64,
|
||||||
|
probe_interval_secs: u64,
|
||||||
|
) -> Vec<DeadRepoBlacklistEntry> {
|
||||||
|
let mut due: Vec<DeadRepoBlacklistEntry> = self
|
||||||
|
.entries
|
||||||
|
.values()
|
||||||
|
.filter(|entry| entry.blacklisted)
|
||||||
|
.filter(|entry| {
|
||||||
|
let interval = probe_backoff_secs(probe_interval_secs, entry.probe_failures);
|
||||||
|
match entry.last_probe_at_unix {
|
||||||
|
None => true,
|
||||||
|
Some(last) => now_unix.saturating_sub(last) >= interval,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.cloned()
|
||||||
|
.collect();
|
||||||
|
due.sort_by(|a, b| a.uri.cmp(&b.uri));
|
||||||
|
due
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Probe succeeded: remove the entry. Returns true if it was present.
|
||||||
|
pub fn record_probe_success(&mut self, transport: RepoTransportMode, uri: &str) -> bool {
|
||||||
|
self.entries.remove(&(transport, uri.to_string())).is_some()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn record_probe_failure(&mut self, transport: RepoTransportMode, uri: &str, now_unix: u64) {
|
||||||
|
if let Some(entry) = self.entries.get_mut(&(transport, uri.to_string())) {
|
||||||
|
entry.last_probe_at_unix = Some(now_unix);
|
||||||
|
entry.probe_failures = entry.probe_failures.saturating_add(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn len(&self) -> usize {
|
||||||
|
self.entries.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_empty(&self) -> bool {
|
||||||
|
self.entries.is_empty()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn blacklisted_len(&self) -> usize {
|
||||||
|
self.entries.values().filter(|entry| entry.blacklisted).count()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Deterministic snapshot of all entries (for status/observability).
|
||||||
|
pub fn entries_sorted(&self) -> Vec<DeadRepoBlacklistEntry> {
|
||||||
|
let mut entries: Vec<DeadRepoBlacklistEntry> = self.entries.values().cloned().collect();
|
||||||
|
entries.sort_by(|a, b| a.uri.cmp(&b.uri).then(a.transport.cmp(&b.transport)));
|
||||||
|
entries
|
||||||
|
}
|
||||||
|
|
||||||
|
fn evict_if_full(&mut self, capacity: usize) {
|
||||||
|
let capacity = capacity.max(1);
|
||||||
|
if self.entries.len() < capacity {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Prefer evicting the stalest counting entry; fall back to the oldest
|
||||||
|
// blacklisted entry.
|
||||||
|
let victim = self
|
||||||
|
.entries
|
||||||
|
.values()
|
||||||
|
.filter(|entry| !entry.blacklisted)
|
||||||
|
.min_by_key(|entry| entry.last_failure_at_unix)
|
||||||
|
.map(|entry| (entry.transport, entry.uri.clone()))
|
||||||
|
.or_else(|| {
|
||||||
|
self.entries
|
||||||
|
.values()
|
||||||
|
.min_by_key(|entry| entry.added_at_unix)
|
||||||
|
.map(|entry| (entry.transport, entry.uri.clone()))
|
||||||
|
});
|
||||||
|
if let Some(key) = victim {
|
||||||
|
self.entries.remove(&key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn probe_backoff_secs(base_interval_secs: u64, probe_failures: u32) -> u64 {
|
||||||
|
// First retry after a probe failure still uses the base interval;
|
||||||
|
// exponential backoff kicks in from the second consecutive failure.
|
||||||
|
let shift = probe_failures.saturating_sub(1).min(PROBE_BACKOFF_CAP_SHIFT);
|
||||||
|
base_interval_secs.saturating_mul(1u64 << shift)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Classify an HTTP fetch error string (formats from
|
||||||
|
/// `fetch/http.rs::BlockingHttpFetcher::fetch_bytes`).
|
||||||
|
pub fn classify_http_fetch_error(detail: &str) -> RepoTransportErrorClass {
|
||||||
|
if detail.starts_with("http status") {
|
||||||
|
// The peer answered: transport is fine, the failure is protocol-level.
|
||||||
|
RepoTransportErrorClass::Protocol
|
||||||
|
} else if detail.starts_with("http request failed") || detail.starts_with("http read body failed") {
|
||||||
|
RepoTransportErrorClass::TransportFetch
|
||||||
|
} else {
|
||||||
|
RepoTransportErrorClass::Unknown
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Classify an rsync fetch error string using the same host-level substring
|
||||||
|
/// set as the run-local rsync failure scope short-circuit.
|
||||||
|
pub fn classify_rsync_fetch_error(detail: &str) -> RepoTransportErrorClass {
|
||||||
|
if crate::parallel::repo_scheduler::is_host_level_rsync_failure(detail)
|
||||||
|
// Stall failures are transport death too: the fail-fast mechanism
|
||||||
|
// gives up when wall-clock windows pass with no (additional) progress
|
||||||
|
// (#141 M9). Content-level errors fail immediately and never carry
|
||||||
|
// this marker, so they still do not count.
|
||||||
|
|| detail.contains("rsync fail-fast gave up")
|
||||||
|
{
|
||||||
|
RepoTransportErrorClass::TransportFetch
|
||||||
|
} else {
|
||||||
|
// Content/protocol-level rsync failures (digest mismatch, vanished
|
||||||
|
// files, partial transfers) never count toward the blacklist.
|
||||||
|
RepoTransportErrorClass::Unknown
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Map a structured RRDP sync error to a transport error class.
|
||||||
|
pub fn classify_rrdp_sync_error(err: &crate::sync::rrdp::RrdpSyncError) -> RepoTransportErrorClass {
|
||||||
|
match err {
|
||||||
|
crate::sync::rrdp::RrdpSyncError::Fetch(detail) => classify_http_fetch_error(detail),
|
||||||
|
crate::sync::rrdp::RrdpSyncError::Rrdp(_) => RepoTransportErrorClass::Protocol,
|
||||||
|
crate::sync::rrdp::RrdpSyncError::Storage(_) => RepoTransportErrorClass::Storage,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Map a structured rsync-side repo sync error to a transport error class.
|
||||||
|
pub fn classify_rsync_repo_sync_error(err: &crate::sync::repo::RepoSyncError) -> RepoTransportErrorClass {
|
||||||
|
match err {
|
||||||
|
crate::sync::repo::RepoSyncError::Rsync(crate::fetch::rsync::RsyncFetchError::Fetch(detail)) => {
|
||||||
|
classify_rsync_fetch_error(detail)
|
||||||
|
}
|
||||||
|
crate::sync::repo::RepoSyncError::Storage(_) => RepoTransportErrorClass::Storage,
|
||||||
|
_ => RepoTransportErrorClass::Protocol,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RepoTransportMode {
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
RepoTransportMode::Rrdp => "rrdp",
|
||||||
|
RepoTransportMode::Rsync => "rsync",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep the public surface free of internal key types; the (transport, uri)
|
||||||
|
// tuple key is an implementation detail of the lookup map.
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "dead_repo_blacklist_tests.rs"]
|
||||||
|
mod tests;
|
||||||
360
src/parallel/dead_repo_blacklist_tests.rs
Normal file
360
src/parallel/dead_repo_blacklist_tests.rs
Normal file
@ -0,0 +1,360 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::parallel::types::RepoTransportMode;
|
||||||
|
|
||||||
|
const RRDP_URI: &str = "https://dead.example.com/notification.xml";
|
||||||
|
const RSYNC_URI: &str = "rsync://dead.example.com/repo/";
|
||||||
|
|
||||||
|
fn now() -> u64 {
|
||||||
|
1_700_000_000
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn admission_requires_threshold_consecutive_failures() {
|
||||||
|
let mut bl = DeadRepoBlacklist::new();
|
||||||
|
assert!(!bl.is_blacklisted(RepoTransportMode::Rrdp, RRDP_URI));
|
||||||
|
|
||||||
|
let out1 = bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now(), 3, 256);
|
||||||
|
assert_eq!(
|
||||||
|
out1,
|
||||||
|
DeadRepoFailureOutcome::Counted {
|
||||||
|
consecutive_failures: 1
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert!(!bl.is_blacklisted(RepoTransportMode::Rrdp, RRDP_URI));
|
||||||
|
|
||||||
|
let out2 = bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + 1, 3, 256);
|
||||||
|
assert_eq!(
|
||||||
|
out2,
|
||||||
|
DeadRepoFailureOutcome::Counted {
|
||||||
|
consecutive_failures: 2
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert!(!bl.is_blacklisted(RepoTransportMode::Rrdp, RRDP_URI));
|
||||||
|
|
||||||
|
let out3 = bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + 2, 3, 256);
|
||||||
|
assert_eq!(out3, DeadRepoFailureOutcome::Admitted);
|
||||||
|
assert!(bl.is_blacklisted(RepoTransportMode::Rrdp, RRDP_URI));
|
||||||
|
|
||||||
|
// Further failures keep counting but do not re-admit.
|
||||||
|
let out4 = bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + 3, 3, 256);
|
||||||
|
assert!(matches!(
|
||||||
|
out4,
|
||||||
|
DeadRepoFailureOutcome::Counted {
|
||||||
|
consecutive_failures: 4
|
||||||
|
}
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn threshold_of_one_admits_immediately() {
|
||||||
|
let mut bl = DeadRepoBlacklist::new();
|
||||||
|
let out = bl.record_transport_failure(RepoTransportMode::Rsync, RSYNC_URI, now(), 1, 256);
|
||||||
|
assert_eq!(out, DeadRepoFailureOutcome::Admitted);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn success_resets_counting_entry() {
|
||||||
|
let mut bl = DeadRepoBlacklist::new();
|
||||||
|
bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now(), 3, 256);
|
||||||
|
assert_eq!(bl.len(), 1);
|
||||||
|
let out = bl.record_transport_success(RepoTransportMode::Rrdp, RRDP_URI);
|
||||||
|
assert_eq!(out, Some(DeadRepoSuccessOutcome::CounterReset));
|
||||||
|
assert!(bl.is_empty());
|
||||||
|
// Next failure starts from zero again.
|
||||||
|
let out = bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now(), 3, 256);
|
||||||
|
assert_eq!(
|
||||||
|
out,
|
||||||
|
DeadRepoFailureOutcome::Counted {
|
||||||
|
consecutive_failures: 1
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn success_on_blacklisted_entry_self_heals() {
|
||||||
|
let mut bl = DeadRepoBlacklist::new();
|
||||||
|
for i in 0..3 {
|
||||||
|
bl.record_transport_failure(RepoTransportMode::Rsync, RSYNC_URI, now() + i, 3, 256);
|
||||||
|
}
|
||||||
|
assert!(bl.is_blacklisted(RepoTransportMode::Rsync, RSYNC_URI));
|
||||||
|
let out = bl.record_transport_success(RepoTransportMode::Rsync, RSYNC_URI);
|
||||||
|
assert_eq!(out, Some(DeadRepoSuccessOutcome::BlacklistRemoved));
|
||||||
|
assert!(!bl.is_blacklisted(RepoTransportMode::Rsync, RSYNC_URI));
|
||||||
|
assert!(bl.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn success_on_unknown_entry_is_none() {
|
||||||
|
let mut bl = DeadRepoBlacklist::new();
|
||||||
|
assert_eq!(
|
||||||
|
bl.record_transport_success(RepoTransportMode::Rrdp, RRDP_URI),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn entries_are_per_transport() {
|
||||||
|
let mut bl = DeadRepoBlacklist::new();
|
||||||
|
for i in 0..3 {
|
||||||
|
bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + i, 3, 256);
|
||||||
|
}
|
||||||
|
assert!(bl.is_blacklisted(RepoTransportMode::Rrdp, RRDP_URI));
|
||||||
|
assert!(!bl.is_blacklisted(RepoTransportMode::Rsync, RRDP_URI));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn capacity_evicts_stale_counting_entries_first() {
|
||||||
|
let mut bl = DeadRepoBlacklist::new();
|
||||||
|
// One blacklisted entry (old) and two counting entries.
|
||||||
|
for i in 0..3 {
|
||||||
|
bl.record_transport_failure(RepoTransportMode::Rrdp, "https://a.example/n.xml", 100 + i, 1, 3);
|
||||||
|
}
|
||||||
|
bl.record_transport_failure(RepoTransportMode::Rrdp, "https://b.example/n.xml", 200, 3, 3);
|
||||||
|
bl.record_transport_failure(RepoTransportMode::Rrdp, "https://c.example/n.xml", 300, 3, 3);
|
||||||
|
assert_eq!(bl.len(), 3);
|
||||||
|
assert_eq!(bl.blacklisted_len(), 1);
|
||||||
|
|
||||||
|
// Inserting a fourth entry evicts the stalest counting entry (b).
|
||||||
|
bl.record_transport_failure(RepoTransportMode::Rrdp, "https://d.example/n.xml", 400, 3, 3);
|
||||||
|
assert_eq!(bl.len(), 3);
|
||||||
|
assert!(bl.entries_sorted().iter().any(|e| e.uri == "https://a.example/n.xml"));
|
||||||
|
assert!(!bl.entries_sorted().iter().any(|e| e.uri == "https://b.example/n.xml"));
|
||||||
|
assert!(bl.entries_sorted().iter().any(|e| e.uri == "https://c.example/n.xml"));
|
||||||
|
assert!(bl.entries_sorted().iter().any(|e| e.uri == "https://d.example/n.xml"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn capacity_evicts_oldest_blacklisted_when_no_counting_entries() {
|
||||||
|
let mut bl = DeadRepoBlacklist::new();
|
||||||
|
bl.record_transport_failure(RepoTransportMode::Rrdp, "https://a.example/n.xml", 100, 1, 2);
|
||||||
|
bl.record_transport_failure(RepoTransportMode::Rrdp, "https://b.example/n.xml", 200, 1, 2);
|
||||||
|
assert_eq!(bl.blacklisted_len(), 2);
|
||||||
|
bl.record_transport_failure(RepoTransportMode::Rrdp, "https://c.example/n.xml", 300, 1, 2);
|
||||||
|
assert_eq!(bl.len(), 2);
|
||||||
|
assert!(!bl.entries_sorted().iter().any(|e| e.uri == "https://a.example/n.xml"));
|
||||||
|
assert!(bl.entries_sorted().iter().any(|e| e.uri == "https://b.example/n.xml"));
|
||||||
|
assert!(bl.entries_sorted().iter().any(|e| e.uri == "https://c.example/n.xml"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn store_load_roundtrip() {
|
||||||
|
let dir = std::env::temp_dir().join(format!(
|
||||||
|
"dead_repo_blacklist_test_roundtrip_{}_{}",
|
||||||
|
std::process::id(),
|
||||||
|
now()
|
||||||
|
));
|
||||||
|
let path = dir.join("state").join("dead-repo-blacklist.json");
|
||||||
|
let mut bl = DeadRepoBlacklist::new();
|
||||||
|
for i in 0..3 {
|
||||||
|
bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + i, 3, 256);
|
||||||
|
}
|
||||||
|
bl.record_transport_failure(RepoTransportMode::Rsync, RSYNC_URI, now(), 3, 256);
|
||||||
|
bl.store_atomic(&path, now() + 10).unwrap();
|
||||||
|
|
||||||
|
let (loaded, warning) = DeadRepoBlacklist::load(&path);
|
||||||
|
assert!(warning.is_none());
|
||||||
|
assert!(loaded.is_blacklisted(RepoTransportMode::Rrdp, RRDP_URI));
|
||||||
|
assert!(!loaded.is_blacklisted(RepoTransportMode::Rsync, RSYNC_URI));
|
||||||
|
assert_eq!(loaded.len(), 2);
|
||||||
|
let entry = loaded
|
||||||
|
.entries_sorted()
|
||||||
|
.into_iter()
|
||||||
|
.find(|e| e.uri == RRDP_URI)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(entry.consecutive_failures, 3);
|
||||||
|
assert!(entry.blacklisted);
|
||||||
|
assert_eq!(entry.first_failure_at_unix, now());
|
||||||
|
|
||||||
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn load_missing_file_is_empty_without_warning() {
|
||||||
|
let path = std::env::temp_dir().join(format!(
|
||||||
|
"dead_repo_blacklist_test_missing_{}_{}.json",
|
||||||
|
std::process::id(),
|
||||||
|
now()
|
||||||
|
));
|
||||||
|
let (bl, warning) = DeadRepoBlacklist::load(&path);
|
||||||
|
assert!(bl.is_empty());
|
||||||
|
assert!(warning.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn load_corrupted_file_degrades_to_empty_with_warning() {
|
||||||
|
let path = std::env::temp_dir().join(format!(
|
||||||
|
"dead_repo_blacklist_test_corrupt_{}_{}.json",
|
||||||
|
std::process::id(),
|
||||||
|
now()
|
||||||
|
));
|
||||||
|
std::fs::write(&path, b"{not json").unwrap();
|
||||||
|
let (bl, warning) = DeadRepoBlacklist::load(&path);
|
||||||
|
assert!(bl.is_empty());
|
||||||
|
assert!(warning.is_some());
|
||||||
|
let _ = std::fs::remove_file(&path);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn load_wrong_schema_version_degrades_to_empty() {
|
||||||
|
let path = std::env::temp_dir().join(format!(
|
||||||
|
"dead_repo_blacklist_test_schema_{}_{}.json",
|
||||||
|
std::process::id(),
|
||||||
|
now()
|
||||||
|
));
|
||||||
|
std::fs::write(
|
||||||
|
&path,
|
||||||
|
r#"{"schema_version": 99, "updated_at_unix": 0, "entries": []}"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let (bl, warning) = DeadRepoBlacklist::load(&path);
|
||||||
|
assert!(bl.is_empty());
|
||||||
|
assert!(warning.unwrap().contains("schema_version"));
|
||||||
|
let _ = std::fs::remove_file(&path);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn probe_due_uses_exponential_backoff() {
|
||||||
|
let mut bl = DeadRepoBlacklist::new();
|
||||||
|
for i in 0..3 {
|
||||||
|
bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + i, 3, 256);
|
||||||
|
}
|
||||||
|
// Never probed: due immediately.
|
||||||
|
let due = bl.probe_due_entries(now() + 100, 600);
|
||||||
|
assert_eq!(due.len(), 1);
|
||||||
|
|
||||||
|
// Probe failure: not due until base interval has passed.
|
||||||
|
bl.record_probe_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + 100);
|
||||||
|
assert!(bl.probe_due_entries(now() + 100 + 599, 600).is_empty());
|
||||||
|
assert_eq!(bl.probe_due_entries(now() + 100 + 600, 600).len(), 1);
|
||||||
|
|
||||||
|
// Second probe failure: backoff doubles to 1200s.
|
||||||
|
bl.record_probe_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + 700);
|
||||||
|
assert!(bl.probe_due_entries(now() + 700 + 1199, 600).is_empty());
|
||||||
|
assert_eq!(bl.probe_due_entries(now() + 700 + 1200, 600).len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn probe_success_removes_entry() {
|
||||||
|
let mut bl = DeadRepoBlacklist::new();
|
||||||
|
for i in 0..3 {
|
||||||
|
bl.record_transport_failure(RepoTransportMode::Rsync, RSYNC_URI, now() + i, 3, 256);
|
||||||
|
}
|
||||||
|
assert!(bl.record_probe_success(RepoTransportMode::Rsync, RSYNC_URI));
|
||||||
|
assert!(bl.is_empty());
|
||||||
|
assert!(!bl.record_probe_success(RepoTransportMode::Rsync, RSYNC_URI));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn counting_entries_are_not_probed() {
|
||||||
|
let mut bl = DeadRepoBlacklist::new();
|
||||||
|
bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now(), 3, 256);
|
||||||
|
assert!(bl.probe_due_entries(now() + 10_000, 600).is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn probe_backoff_caps_at_max_shift() {
|
||||||
|
assert_eq!(probe_backoff_secs(600, 0), 600);
|
||||||
|
assert_eq!(probe_backoff_secs(600, 1), 600);
|
||||||
|
assert_eq!(probe_backoff_secs(600, 2), 1200);
|
||||||
|
assert_eq!(probe_backoff_secs(600, 5), 9600);
|
||||||
|
assert_eq!(probe_backoff_secs(600, 100), 9600);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn http_fetch_error_classification() {
|
||||||
|
use crate::parallel::types::RepoTransportErrorClass;
|
||||||
|
assert_eq!(
|
||||||
|
classify_http_fetch_error("http request failed: reqwest::Error { kind: Connect, .. }"),
|
||||||
|
RepoTransportErrorClass::TransportFetch
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
classify_http_fetch_error("http read body failed: connection closed; status=200"),
|
||||||
|
RepoTransportErrorClass::TransportFetch
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
classify_http_fetch_error("http status 404 Not Found; content_type=; .."),
|
||||||
|
RepoTransportErrorClass::Protocol
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
classify_http_fetch_error("something unexpected"),
|
||||||
|
RepoTransportErrorClass::Unknown
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rsync_fetch_error_classification() {
|
||||||
|
use crate::parallel::types::RepoTransportErrorClass;
|
||||||
|
assert_eq!(
|
||||||
|
classify_rsync_fetch_error("rsync error: timeout waiting for daemon connection"),
|
||||||
|
RepoTransportErrorClass::TransportFetch
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
classify_rsync_fetch_error("rsync: failed to connect to host: Connection refused"),
|
||||||
|
RepoTransportErrorClass::TransportFetch
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
classify_rsync_fetch_error("temporary failure in name resolution"),
|
||||||
|
RepoTransportErrorClass::TransportFetch
|
||||||
|
);
|
||||||
|
// Content-level failures must not count.
|
||||||
|
assert_eq!(
|
||||||
|
classify_rsync_fetch_error("rsync file digest mismatch after download"),
|
||||||
|
RepoTransportErrorClass::Unknown
|
||||||
|
);
|
||||||
|
// Stall failures (#141 M9): fail-fast give-up means the host delivered no
|
||||||
|
// bytes inside the wall-clock window — transport death, must count.
|
||||||
|
assert_eq!(
|
||||||
|
classify_rsync_fetch_error(
|
||||||
|
"rsync fail-fast gave up after 1 attempts with no progress: rsync wall-clock window expired"
|
||||||
|
),
|
||||||
|
RepoTransportErrorClass::TransportFetch
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
classify_rsync_fetch_error(
|
||||||
|
"rsync fail-fast gave up after 3 attempts with no additional progress: rsync wall-clock window expired"
|
||||||
|
),
|
||||||
|
RepoTransportErrorClass::TransportFetch
|
||||||
|
);
|
||||||
|
// Content-level give-up text without the fail-fast marker stays unknown.
|
||||||
|
assert_eq!(
|
||||||
|
classify_rsync_fetch_error("rsync error: some files vanished before transfer"),
|
||||||
|
RepoTransportErrorClass::Unknown
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rrdp_sync_error_classification() {
|
||||||
|
use crate::parallel::types::RepoTransportErrorClass;
|
||||||
|
use crate::sync::rrdp::RrdpSyncError;
|
||||||
|
assert_eq!(
|
||||||
|
classify_rrdp_sync_error(&RrdpSyncError::Fetch(
|
||||||
|
"http request failed: connect timeout".to_string()
|
||||||
|
)),
|
||||||
|
RepoTransportErrorClass::TransportFetch
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
classify_rrdp_sync_error(&RrdpSyncError::Fetch("http status 500".to_string())),
|
||||||
|
RepoTransportErrorClass::Protocol
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
classify_rrdp_sync_error(&RrdpSyncError::Storage("db full".to_string())),
|
||||||
|
RepoTransportErrorClass::Storage
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn readmission_after_probe_removal_starts_fresh() {
|
||||||
|
let mut bl = DeadRepoBlacklist::new();
|
||||||
|
for i in 0..3 {
|
||||||
|
bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + i, 3, 256);
|
||||||
|
}
|
||||||
|
assert!(bl.record_probe_success(RepoTransportMode::Rrdp, RRDP_URI));
|
||||||
|
// No half-open: needs the full threshold again.
|
||||||
|
let out = bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + 10, 3, 256);
|
||||||
|
assert_eq!(
|
||||||
|
out,
|
||||||
|
DeadRepoFailureOutcome::Counted {
|
||||||
|
consecutive_failures: 1
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
@ -1,4 +1,5 @@
|
|||||||
pub mod config;
|
pub mod config;
|
||||||
|
pub mod dead_repo_blacklist;
|
||||||
pub mod object_worker;
|
pub mod object_worker;
|
||||||
pub mod phase2_scheduler;
|
pub mod phase2_scheduler;
|
||||||
pub mod repo_runtime;
|
pub mod repo_runtime;
|
||||||
|
|||||||
@ -103,6 +103,17 @@ pub trait RepoSyncRuntime: Send + Sync {
|
|||||||
) -> Result<TransportPrefetchDispatchStats, String>;
|
) -> Result<TransportPrefetchDispatchStats, String>;
|
||||||
|
|
||||||
fn transport_prefetch_snapshot(&self) -> TransportPrefetchSnapshot;
|
fn transport_prefetch_snapshot(&self) -> TransportPrefetchSnapshot;
|
||||||
|
|
||||||
|
/// Dead-repo blacklist working copy (#141) for run-end persistence.
|
||||||
|
/// `None` when the feature is disabled.
|
||||||
|
fn dead_repo_blacklist_state(
|
||||||
|
&self,
|
||||||
|
) -> Option<(
|
||||||
|
crate::parallel::dead_repo_blacklist::DeadRepoBlacklistConfig,
|
||||||
|
crate::parallel::dead_repo_blacklist::DeadRepoBlacklist,
|
||||||
|
)> {
|
||||||
|
None
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct Phase1RepoSyncRuntime<E: RepoTransportExecutor> {
|
pub struct Phase1RepoSyncRuntime<E: RepoTransportExecutor> {
|
||||||
@ -634,6 +645,18 @@ impl<E: RepoTransportExecutor> RepoSyncRuntime for Phase1RepoSyncRuntime<E> {
|
|||||||
})
|
})
|
||||||
.unwrap_or_else(|| TransportPrefetchSnapshot::new(self.sync_preference, Vec::new()))
|
.unwrap_or_else(|| TransportPrefetchSnapshot::new(self.sync_preference, Vec::new()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn dead_repo_blacklist_state(
|
||||||
|
&self,
|
||||||
|
) -> Option<(
|
||||||
|
crate::parallel::dead_repo_blacklist::DeadRepoBlacklistConfig,
|
||||||
|
crate::parallel::dead_repo_blacklist::DeadRepoBlacklist,
|
||||||
|
)> {
|
||||||
|
let coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
|
||||||
|
let config = coordinator.config.dead_repo_blacklist.clone()?;
|
||||||
|
let blacklist = coordinator.dead_repo_blacklist().cloned()?;
|
||||||
|
Some((config, blacklist))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn outcome_from_transport_result(
|
fn outcome_from_transport_result(
|
||||||
@ -666,7 +689,9 @@ fn outcome_from_transport_result(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
(
|
(
|
||||||
RepoTransportResultKind::Failed { detail, warnings },
|
RepoTransportResultKind::Failed {
|
||||||
|
detail, warnings, ..
|
||||||
|
},
|
||||||
RepoRuntimeState::FailedTerminal,
|
RepoRuntimeState::FailedTerminal,
|
||||||
) => RepoSyncRuntimeOutcome {
|
) => RepoSyncRuntimeOutcome {
|
||||||
repo_sync_ok: false,
|
repo_sync_ok: false,
|
||||||
@ -798,6 +823,7 @@ mod tests {
|
|||||||
result: RepoTransportResultKind::Failed {
|
result: RepoTransportResultKind::Failed {
|
||||||
detail: "rrdp failed".to_string(),
|
detail: "rrdp failed".to_string(),
|
||||||
warnings: vec![Warning::new("rrdp failed")],
|
warnings: vec![Warning::new("rrdp failed")],
|
||||||
|
error_class: crate::parallel::types::RepoTransportErrorClass::Unknown,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -842,6 +868,7 @@ mod tests {
|
|||||||
result: RepoTransportResultKind::Failed {
|
result: RepoTransportResultKind::Failed {
|
||||||
detail: "rrdp failed".to_string(),
|
detail: "rrdp failed".to_string(),
|
||||||
warnings: vec![Warning::new("rrdp failed")],
|
warnings: vec![Warning::new("rrdp failed")],
|
||||||
|
error_class: crate::parallel::types::RepoTransportErrorClass::Unknown,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -858,6 +885,7 @@ mod tests {
|
|||||||
result: RepoTransportResultKind::Failed {
|
result: RepoTransportResultKind::Failed {
|
||||||
detail: "rsync failed".to_string(),
|
detail: "rsync failed".to_string(),
|
||||||
warnings: vec![Warning::new("rsync failed")],
|
warnings: vec![Warning::new("rsync failed")],
|
||||||
|
error_class: crate::parallel::types::RepoTransportErrorClass::Unknown,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,11 +1,14 @@
|
|||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
|
|
||||||
|
use crate::parallel::dead_repo_blacklist::DeadRepoBlacklist;
|
||||||
use crate::parallel::types::{
|
use crate::parallel::types::{
|
||||||
InFlightRepoEntry, RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoRuntimeState,
|
InFlightRepoEntry, RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoRuntimeState,
|
||||||
RepoSyncResultEnvelope, RepoSyncResultKind, RepoSyncResultRef, RepoSyncTask, RepoTaskState,
|
RepoSyncResultEnvelope, RepoSyncResultKind, RepoSyncResultRef, RepoSyncTask, RepoTaskState,
|
||||||
RepoTransportMode, RepoTransportResultEnvelope, RepoTransportResultKind, RepoTransportTask,
|
RepoTransportErrorClass, RepoTransportMode, RepoTransportResultEnvelope,
|
||||||
|
RepoTransportResultKind, RepoTransportTask,
|
||||||
};
|
};
|
||||||
use crate::policy::SyncPreference;
|
use crate::policy::SyncPreference;
|
||||||
|
use crate::report::Warning;
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
pub enum RepoRequestAction {
|
pub enum RepoRequestAction {
|
||||||
@ -74,6 +77,9 @@ pub struct TransportStateTables {
|
|||||||
rsync_failure_probe_inflight: HashMap<String, String>,
|
rsync_failure_probe_inflight: HashMap<String, String>,
|
||||||
rsync_failure_scope_reachable: HashSet<String>,
|
rsync_failure_scope_reachable: HashSet<String>,
|
||||||
runtime_records: HashMap<RepoIdentity, RepoRuntimeRecord>,
|
runtime_records: HashMap<RepoIdentity, RepoRuntimeRecord>,
|
||||||
|
/// Frozen run-start snapshot of the dead-repo blacklist (#141). Entries
|
||||||
|
/// admitted during this run only take effect from the next run.
|
||||||
|
dead_repo_blacklist: Option<DeadRepoBlacklist>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TransportStateTables {
|
impl TransportStateTables {
|
||||||
@ -81,6 +87,44 @@ impl TransportStateTables {
|
|||||||
Self::default()
|
Self::default()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn set_dead_repo_blacklist(&mut self, blacklist: DeadRepoBlacklist) {
|
||||||
|
self.dead_repo_blacklist = Some(blacklist);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dead_repo_terminal_envelope(
|
||||||
|
identity: &RepoIdentity,
|
||||||
|
requesters: &[RepoRequester],
|
||||||
|
rsync_scope_uri: &str,
|
||||||
|
rsync_failure_scope_uri: Option<String>,
|
||||||
|
) -> RepoTransportResultEnvelope {
|
||||||
|
let first_requester = requesters
|
||||||
|
.first()
|
||||||
|
.expect("blacklist terminal record must keep at least one requester");
|
||||||
|
RepoTransportResultEnvelope {
|
||||||
|
dedup_key: RepoDedupKey::RsyncScope {
|
||||||
|
rsync_scope_uri: rsync_scope_uri.to_string(),
|
||||||
|
},
|
||||||
|
rsync_failure_scope_uri,
|
||||||
|
repo_identity: identity.clone(),
|
||||||
|
mode: RepoTransportMode::Rsync,
|
||||||
|
tal_id: first_requester.tal_id.clone(),
|
||||||
|
rir_id: first_requester.rir_id.clone(),
|
||||||
|
timing_ms: 0,
|
||||||
|
result: RepoTransportResultKind::Failed {
|
||||||
|
detail: format!(
|
||||||
|
"dead repo blacklist: rsync transport persistently unreachable: {}",
|
||||||
|
identity.rsync_base_uri
|
||||||
|
),
|
||||||
|
warnings: vec![Warning::new(format!(
|
||||||
|
"dead_repo_blacklist_skip_all: repository {} skipped after repeated transport failures",
|
||||||
|
identity.rsync_base_uri
|
||||||
|
))
|
||||||
|
.with_context(identity.rsync_base_uri.clone())],
|
||||||
|
error_class: RepoTransportErrorClass::Unknown,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn runtime_record(&self, identity: &RepoIdentity) -> Option<&RepoRuntimeRecord> {
|
pub fn runtime_record(&self, identity: &RepoIdentity) -> Option<&RepoRuntimeRecord> {
|
||||||
self.runtime_records.get(identity)
|
self.runtime_records.get(identity)
|
||||||
}
|
}
|
||||||
@ -205,6 +249,82 @@ impl TransportStateTables {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Dead-repo blacklist fast paths (#141), consulted only for fresh
|
||||||
|
// identities; existing records already encode their terminal state.
|
||||||
|
if let Some(blacklist) = self.dead_repo_blacklist.as_ref() {
|
||||||
|
let rrdp_wanted = sync_preference == SyncPreference::RrdpThenRsync
|
||||||
|
&& identity.notification_uri.is_some();
|
||||||
|
let rrdp_dead = rrdp_wanted
|
||||||
|
&& identity
|
||||||
|
.notification_uri
|
||||||
|
.as_deref()
|
||||||
|
.map(|uri| blacklist.is_blacklisted(RepoTransportMode::Rrdp, uri))
|
||||||
|
.unwrap_or(false);
|
||||||
|
let rsync_dead = blacklist.is_blacklisted(
|
||||||
|
RepoTransportMode::Rsync,
|
||||||
|
identity.rsync_base_uri.as_str(),
|
||||||
|
);
|
||||||
|
if rsync_dead && (!rrdp_wanted || rrdp_dead) {
|
||||||
|
crate::progress_log::emit(
|
||||||
|
"dead_repo_blacklist_skip_all",
|
||||||
|
serde_json::json!({
|
||||||
|
"repo_key_notification_uri": identity.notification_uri,
|
||||||
|
"repo_key_rsync_base_uri": identity.rsync_base_uri,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
let envelope = Self::dead_repo_terminal_envelope(
|
||||||
|
&identity,
|
||||||
|
std::slice::from_ref(&requester),
|
||||||
|
&rsync_scope_uri,
|
||||||
|
rsync_failure_scope_uri.clone(),
|
||||||
|
);
|
||||||
|
self.runtime_records.insert(
|
||||||
|
identity.clone(),
|
||||||
|
RepoRuntimeRecord {
|
||||||
|
identity,
|
||||||
|
state: RepoRuntimeState::FailedTerminal,
|
||||||
|
rrdp_notification_key: None,
|
||||||
|
rsync_scope_key: rsync_scope_uri,
|
||||||
|
rsync_failure_scope_key: rsync_failure_scope_uri,
|
||||||
|
requesters: vec![requester],
|
||||||
|
validation_time,
|
||||||
|
priority,
|
||||||
|
last_success: None,
|
||||||
|
terminal_failure: Some(envelope.clone()),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
return TransportRequestAction::ReusedTerminalFailure(envelope);
|
||||||
|
}
|
||||||
|
if rrdp_dead {
|
||||||
|
crate::progress_log::emit(
|
||||||
|
"dead_repo_blacklist_skip_rrdp",
|
||||||
|
serde_json::json!({
|
||||||
|
"repo_key_notification_uri": identity.notification_uri,
|
||||||
|
"repo_key_rsync_base_uri": identity.rsync_base_uri,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
let mut action = self.register_rsync_request(
|
||||||
|
identity,
|
||||||
|
requester,
|
||||||
|
validation_time,
|
||||||
|
priority,
|
||||||
|
rsync_scope_uri,
|
||||||
|
rsync_failure_scope_uri,
|
||||||
|
);
|
||||||
|
// A blacklisted rrdp transport is a known-persistent failure,
|
||||||
|
// so the rsync fallback must keep the short retry profile
|
||||||
|
// (#141): the prefetch layer records no real rrdp failure in
|
||||||
|
// skipped runs, which would otherwise silently drop the task
|
||||||
|
// back to the default 15s window from the second skipped run
|
||||||
|
// on. The coordinator wrapper only ever sets the flag to true,
|
||||||
|
// so setting it here survives registration.
|
||||||
|
if let TransportRequestAction::Enqueue(task) = &mut action {
|
||||||
|
task.retry_short_timeout = true;
|
||||||
|
}
|
||||||
|
return action;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if sync_preference == SyncPreference::RrdpThenRsync {
|
if sync_preference == SyncPreference::RrdpThenRsync {
|
||||||
if let Some(notification_uri) = identity.notification_uri.clone() {
|
if let Some(notification_uri) = identity.notification_uri.clone() {
|
||||||
if let Some(entry) = self.rrdp_inflight.get_mut(¬ification_uri) {
|
if let Some(entry) = self.rrdp_inflight.get_mut(¬ification_uri) {
|
||||||
@ -512,6 +632,7 @@ impl TransportStateTables {
|
|||||||
rsync_failure_by_scope: &HashMap<String, RepoTransportResultEnvelope>,
|
rsync_failure_by_scope: &HashMap<String, RepoTransportResultEnvelope>,
|
||||||
rsync_failure_probe_inflight: &mut HashMap<String, String>,
|
rsync_failure_probe_inflight: &mut HashMap<String, String>,
|
||||||
rsync_failure_scope_reachable: &HashSet<String>,
|
rsync_failure_scope_reachable: &HashSet<String>,
|
||||||
|
dead_repo_blacklist: Option<&DeadRepoBlacklist>,
|
||||||
follow_up_tasks: &mut Vec<RepoTransportTask>,
|
follow_up_tasks: &mut Vec<RepoTransportTask>,
|
||||||
) {
|
) {
|
||||||
let rsync_scope_uri = record.rsync_scope_key.clone();
|
let rsync_scope_uri = record.rsync_scope_key.clone();
|
||||||
@ -548,6 +669,34 @@ impl TransportStateTables {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Dead-repo blacklist (#141): the rsync transport of this repo is
|
||||||
|
// persistently unreachable; terminate instantly instead of enqueueing
|
||||||
|
// another doomed fetch (rrdp was already attempted or skipped).
|
||||||
|
if let Some(blacklist) = dead_repo_blacklist {
|
||||||
|
if blacklist.is_blacklisted(
|
||||||
|
RepoTransportMode::Rsync,
|
||||||
|
record.identity.rsync_base_uri.as_str(),
|
||||||
|
) {
|
||||||
|
crate::progress_log::emit(
|
||||||
|
"dead_repo_blacklist_skip_all",
|
||||||
|
serde_json::json!({
|
||||||
|
"repo_key_notification_uri": record.identity.notification_uri,
|
||||||
|
"repo_key_rsync_base_uri": record.identity.rsync_base_uri,
|
||||||
|
"after_rrdp_failure": true,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
let envelope = Self::dead_repo_terminal_envelope(
|
||||||
|
&record.identity,
|
||||||
|
&record.requesters,
|
||||||
|
&rsync_scope_uri,
|
||||||
|
record.rsync_failure_scope_key.clone(),
|
||||||
|
);
|
||||||
|
record.state = RepoRuntimeState::FailedTerminal;
|
||||||
|
record.terminal_failure = Some(envelope);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let first_requester = record
|
let first_requester = record
|
||||||
.requesters
|
.requesters
|
||||||
.first()
|
.first()
|
||||||
@ -666,6 +815,7 @@ impl TransportStateTables {
|
|||||||
&self.rsync_failure_by_scope,
|
&self.rsync_failure_by_scope,
|
||||||
&mut self.rsync_failure_probe_inflight,
|
&mut self.rsync_failure_probe_inflight,
|
||||||
&self.rsync_failure_scope_reachable,
|
&self.rsync_failure_scope_reachable,
|
||||||
|
self.dead_repo_blacklist.as_ref(),
|
||||||
&mut follow_up_tasks,
|
&mut follow_up_tasks,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@ -718,6 +868,7 @@ impl TransportStateTables {
|
|||||||
&self.rsync_failure_by_scope,
|
&self.rsync_failure_by_scope,
|
||||||
&mut self.rsync_failure_probe_inflight,
|
&mut self.rsync_failure_probe_inflight,
|
||||||
&self.rsync_failure_scope_reachable,
|
&self.rsync_failure_scope_reachable,
|
||||||
|
self.dead_repo_blacklist.as_ref(),
|
||||||
&mut follow_up_tasks,
|
&mut follow_up_tasks,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@ -783,6 +934,7 @@ impl TransportStateTables {
|
|||||||
&self.rsync_failure_by_scope,
|
&self.rsync_failure_by_scope,
|
||||||
&mut self.rsync_failure_probe_inflight,
|
&mut self.rsync_failure_probe_inflight,
|
||||||
&self.rsync_failure_scope_reachable,
|
&self.rsync_failure_scope_reachable,
|
||||||
|
self.dead_repo_blacklist.as_ref(),
|
||||||
&mut follow_up_tasks,
|
&mut follow_up_tasks,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@ -814,7 +966,7 @@ fn reusable_rsync_failure_scope(result: &RepoTransportResultEnvelope) -> Option<
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_host_level_rsync_failure(detail: &str) -> bool {
|
pub(crate) fn is_host_level_rsync_failure(detail: &str) -> bool {
|
||||||
let lower = detail.to_ascii_lowercase();
|
let lower = detail.to_ascii_lowercase();
|
||||||
lower.contains("timeout waiting for daemon connection")
|
lower.contains("timeout waiting for daemon connection")
|
||||||
|| lower.contains("failed to connect")
|
|| lower.contains("failed to connect")
|
||||||
@ -1462,6 +1614,7 @@ mod transport_tests {
|
|||||||
result: RepoTransportResultKind::Failed {
|
result: RepoTransportResultKind::Failed {
|
||||||
detail: "rrdp timeout".to_string(),
|
detail: "rrdp timeout".to_string(),
|
||||||
warnings: Vec::new(),
|
warnings: Vec::new(),
|
||||||
|
error_class: crate::parallel::types::RepoTransportErrorClass::Unknown,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
time::OffsetDateTime::UNIX_EPOCH,
|
time::OffsetDateTime::UNIX_EPOCH,
|
||||||
@ -1567,6 +1720,7 @@ mod transport_tests {
|
|||||||
result: RepoTransportResultKind::Failed {
|
result: RepoTransportResultKind::Failed {
|
||||||
detail: "rrdp timeout".to_string(),
|
detail: "rrdp timeout".to_string(),
|
||||||
warnings: Vec::new(),
|
warnings: Vec::new(),
|
||||||
|
error_class: crate::parallel::types::RepoTransportErrorClass::Unknown,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
time::OffsetDateTime::UNIX_EPOCH,
|
time::OffsetDateTime::UNIX_EPOCH,
|
||||||
@ -1597,6 +1751,7 @@ mod transport_tests {
|
|||||||
result: RepoTransportResultKind::Failed {
|
result: RepoTransportResultKind::Failed {
|
||||||
detail: "rsync error: timeout waiting for daemon connection".to_string(),
|
detail: "rsync error: timeout waiting for daemon connection".to_string(),
|
||||||
warnings: Vec::new(),
|
warnings: Vec::new(),
|
||||||
|
error_class: crate::parallel::types::RepoTransportErrorClass::Unknown,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
time::OffsetDateTime::UNIX_EPOCH,
|
time::OffsetDateTime::UNIX_EPOCH,
|
||||||
@ -1659,6 +1814,7 @@ mod transport_tests {
|
|||||||
result: RepoTransportResultKind::Failed {
|
result: RepoTransportResultKind::Failed {
|
||||||
detail: "rsync timeout".to_string(),
|
detail: "rsync timeout".to_string(),
|
||||||
warnings: Vec::new(),
|
warnings: Vec::new(),
|
||||||
|
error_class: crate::parallel::types::RepoTransportErrorClass::Unknown,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
time::OffsetDateTime::UNIX_EPOCH,
|
time::OffsetDateTime::UNIX_EPOCH,
|
||||||
@ -1920,6 +2076,7 @@ mod transport_tests {
|
|||||||
result: RepoTransportResultKind::Failed {
|
result: RepoTransportResultKind::Failed {
|
||||||
detail: "rsync file digest mismatch after download".to_string(),
|
detail: "rsync file digest mismatch after download".to_string(),
|
||||||
warnings: Vec::new(),
|
warnings: Vec::new(),
|
||||||
|
error_class: crate::parallel::types::RepoTransportErrorClass::Unknown,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
time::OffsetDateTime::UNIX_EPOCH,
|
time::OffsetDateTime::UNIX_EPOCH,
|
||||||
@ -1972,6 +2129,7 @@ mod transport_tests {
|
|||||||
result: RepoTransportResultKind::Failed {
|
result: RepoTransportResultKind::Failed {
|
||||||
detail: "rsync error: failed to connect to daemon".to_string(),
|
detail: "rsync error: failed to connect to daemon".to_string(),
|
||||||
warnings: Vec::new(),
|
warnings: Vec::new(),
|
||||||
|
error_class: crate::parallel::types::RepoTransportErrorClass::Unknown,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
time::OffsetDateTime::UNIX_EPOCH,
|
time::OffsetDateTime::UNIX_EPOCH,
|
||||||
@ -2031,6 +2189,7 @@ mod transport_tests {
|
|||||||
result: RepoTransportResultKind::Failed {
|
result: RepoTransportResultKind::Failed {
|
||||||
detail: "temporary failure in name resolution".to_string(),
|
detail: "temporary failure in name resolution".to_string(),
|
||||||
warnings: Vec::new(),
|
warnings: Vec::new(),
|
||||||
|
error_class: crate::parallel::types::RepoTransportErrorClass::Unknown,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
time::OffsetDateTime::UNIX_EPOCH,
|
time::OffsetDateTime::UNIX_EPOCH,
|
||||||
|
|||||||
@ -115,19 +115,24 @@ impl<H: Fetcher + 'static> RepoTransportExecutor for LiveRrdpTransportExecutor<H
|
|||||||
warnings: Vec::new(),
|
warnings: Vec::new(),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Err(err) => RepoTransportResultEnvelope {
|
Err(err) => {
|
||||||
dedup_key: task.dedup_key,
|
let error_class =
|
||||||
rsync_failure_scope_uri: task.rsync_failure_scope_uri,
|
crate::parallel::dead_repo_blacklist::classify_rrdp_sync_error(&err);
|
||||||
repo_identity: task.repo_identity,
|
RepoTransportResultEnvelope {
|
||||||
mode: RepoTransportMode::Rrdp,
|
dedup_key: task.dedup_key,
|
||||||
tal_id: task.tal_id,
|
rsync_failure_scope_uri: task.rsync_failure_scope_uri,
|
||||||
rir_id: task.rir_id,
|
repo_identity: task.repo_identity,
|
||||||
timing_ms: started.elapsed().as_millis() as u64,
|
mode: RepoTransportMode::Rrdp,
|
||||||
result: RepoTransportResultKind::Failed {
|
tal_id: task.tal_id,
|
||||||
detail: err.to_string(),
|
rir_id: task.rir_id,
|
||||||
warnings: Vec::new(),
|
timing_ms: started.elapsed().as_millis() as u64,
|
||||||
},
|
result: RepoTransportResultKind::Failed {
|
||||||
},
|
detail: err.to_string(),
|
||||||
|
warnings: Vec::new(),
|
||||||
|
error_class,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -206,19 +211,24 @@ impl<R: RsyncFetcher + 'static> RepoTransportExecutor for LiveRsyncTransportExec
|
|||||||
warnings: Vec::new(),
|
warnings: Vec::new(),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Err(err) => RepoTransportResultEnvelope {
|
Err(err) => {
|
||||||
dedup_key: task.dedup_key,
|
let error_class =
|
||||||
rsync_failure_scope_uri: task.rsync_failure_scope_uri,
|
crate::parallel::dead_repo_blacklist::classify_rsync_repo_sync_error(&err);
|
||||||
repo_identity: task.repo_identity,
|
RepoTransportResultEnvelope {
|
||||||
mode: RepoTransportMode::Rsync,
|
dedup_key: task.dedup_key,
|
||||||
tal_id: task.tal_id,
|
rsync_failure_scope_uri: task.rsync_failure_scope_uri,
|
||||||
rir_id: task.rir_id,
|
repo_identity: task.repo_identity,
|
||||||
timing_ms: started.elapsed().as_millis() as u64,
|
mode: RepoTransportMode::Rsync,
|
||||||
result: RepoTransportResultKind::Failed {
|
tal_id: task.tal_id,
|
||||||
detail: err.to_string(),
|
rir_id: task.rir_id,
|
||||||
warnings: Vec::new(),
|
timing_ms: started.elapsed().as_millis() as u64,
|
||||||
},
|
result: RepoTransportResultKind::Failed {
|
||||||
},
|
detail: err.to_string(),
|
||||||
|
warnings: Vec::new(),
|
||||||
|
error_class,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,6 +2,9 @@ use std::collections::VecDeque;
|
|||||||
|
|
||||||
use crate::current_repo_index::{CurrentRepoIndex, CurrentRepoIndexHandle};
|
use crate::current_repo_index::{CurrentRepoIndex, CurrentRepoIndexHandle};
|
||||||
use crate::parallel::config::ParallelPhase1Config;
|
use crate::parallel::config::ParallelPhase1Config;
|
||||||
|
use crate::parallel::dead_repo_blacklist::{
|
||||||
|
DeadRepoBlacklist, DeadRepoFailureOutcome, DeadRepoSuccessOutcome,
|
||||||
|
};
|
||||||
use crate::parallel::repo_scheduler::{
|
use crate::parallel::repo_scheduler::{
|
||||||
InFlightRepoTable, RepoCompletion, RepoRequestAction, TransportCompletion,
|
InFlightRepoTable, RepoCompletion, RepoRequestAction, TransportCompletion,
|
||||||
TransportRequestAction, TransportStateTables,
|
TransportRequestAction, TransportStateTables,
|
||||||
@ -9,7 +12,8 @@ use crate::parallel::repo_scheduler::{
|
|||||||
use crate::parallel::stats::ParallelRunStats;
|
use crate::parallel::stats::ParallelRunStats;
|
||||||
use crate::parallel::types::{
|
use crate::parallel::types::{
|
||||||
RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoSyncResultEnvelope, RepoSyncTask,
|
RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoSyncResultEnvelope, RepoSyncTask,
|
||||||
RepoTransportResultEnvelope, RepoTransportTask, TalInputSpec,
|
RepoTransportErrorClass, RepoTransportMode, RepoTransportResultEnvelope,
|
||||||
|
RepoTransportResultKind, RepoTransportTask, TalInputSpec,
|
||||||
};
|
};
|
||||||
use crate::policy::SyncPreference;
|
use crate::policy::SyncPreference;
|
||||||
|
|
||||||
@ -22,19 +26,42 @@ pub struct GlobalRunCoordinator {
|
|||||||
pub pending_repo_tasks: VecDeque<RepoSyncTask>,
|
pub pending_repo_tasks: VecDeque<RepoSyncTask>,
|
||||||
pub pending_transport_tasks: VecDeque<RepoTransportTask>,
|
pub pending_transport_tasks: VecDeque<RepoTransportTask>,
|
||||||
pub stats: ParallelRunStats,
|
pub stats: ParallelRunStats,
|
||||||
|
/// Mutable working copy of the dead-repo blacklist (#141): failure/success
|
||||||
|
/// counters update during the run and persist at run end. Skip decisions
|
||||||
|
/// inside `transport_tables` use the frozen run-start snapshot.
|
||||||
|
pub dead_repo_blacklist: Option<DeadRepoBlacklist>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl GlobalRunCoordinator {
|
impl GlobalRunCoordinator {
|
||||||
pub fn new(config: ParallelPhase1Config, tal_inputs: Vec<TalInputSpec>) -> Self {
|
pub fn new(config: ParallelPhase1Config, tal_inputs: Vec<TalInputSpec>) -> Self {
|
||||||
|
let dead_repo_blacklist = config
|
||||||
|
.dead_repo_blacklist
|
||||||
|
.as_ref()
|
||||||
|
.map(|blacklist_config| {
|
||||||
|
let (blacklist, warning) = DeadRepoBlacklist::load(&blacklist_config.path);
|
||||||
|
if let Some(warning) = warning {
|
||||||
|
crate::progress_log::emit(
|
||||||
|
"dead_repo_blacklist_load_warning",
|
||||||
|
serde_json::json!({ "warning": warning }),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
blacklist
|
||||||
|
});
|
||||||
|
let mut transport_tables = TransportStateTables::new();
|
||||||
|
if let Some(blacklist) = dead_repo_blacklist.as_ref() {
|
||||||
|
// Frozen run-start snapshot for skip decisions.
|
||||||
|
transport_tables.set_dead_repo_blacklist(blacklist.clone());
|
||||||
|
}
|
||||||
Self {
|
Self {
|
||||||
config,
|
config,
|
||||||
tal_inputs,
|
tal_inputs,
|
||||||
current_repo_index: CurrentRepoIndex::shared(),
|
current_repo_index: CurrentRepoIndex::shared(),
|
||||||
in_flight_repos: InFlightRepoTable::new(),
|
in_flight_repos: InFlightRepoTable::new(),
|
||||||
transport_tables: TransportStateTables::new(),
|
transport_tables,
|
||||||
pending_repo_tasks: VecDeque::new(),
|
pending_repo_tasks: VecDeque::new(),
|
||||||
pending_transport_tasks: VecDeque::new(),
|
pending_transport_tasks: VecDeque::new(),
|
||||||
stats: ParallelRunStats::default(),
|
stats: ParallelRunStats::default(),
|
||||||
|
dead_repo_blacklist,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -193,9 +220,92 @@ impl GlobalRunCoordinator {
|
|||||||
{
|
{
|
||||||
self.stats.repo_tasks_failed += 1;
|
self.stats.repo_tasks_failed += 1;
|
||||||
}
|
}
|
||||||
|
self.update_dead_repo_blacklist(&result, finished_at);
|
||||||
Ok(completion)
|
Ok(completion)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Update dead-repo blacklist counters (#141) from a real worker result.
|
||||||
|
/// Runs on the single pump thread; synthesized terminal envelopes never
|
||||||
|
/// pass through here, so no double counting is possible.
|
||||||
|
fn update_dead_repo_blacklist(
|
||||||
|
&mut self,
|
||||||
|
result: &RepoTransportResultEnvelope,
|
||||||
|
finished_at: time::OffsetDateTime,
|
||||||
|
) {
|
||||||
|
let Some(blacklist_config) = self.config.dead_repo_blacklist.clone() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let Some(blacklist) = self.dead_repo_blacklist.as_mut() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let now_unix = finished_at.unix_timestamp().max(0) as u64;
|
||||||
|
let key: Option<(RepoTransportMode, String)> = match (&result.dedup_key, &result.result) {
|
||||||
|
(RepoDedupKey::RrdpNotify { notification_uri }, RepoTransportResultKind::Success { .. }) => {
|
||||||
|
Some((RepoTransportMode::Rrdp, notification_uri.clone()))
|
||||||
|
}
|
||||||
|
(RepoDedupKey::RsyncScope { .. }, RepoTransportResultKind::Success { .. }) => {
|
||||||
|
Some((RepoTransportMode::Rsync, result.repo_identity.rsync_base_uri.clone()))
|
||||||
|
}
|
||||||
|
(
|
||||||
|
RepoDedupKey::RrdpNotify { notification_uri },
|
||||||
|
RepoTransportResultKind::Failed {
|
||||||
|
error_class: RepoTransportErrorClass::TransportFetch,
|
||||||
|
..
|
||||||
|
},
|
||||||
|
) => Some((RepoTransportMode::Rrdp, notification_uri.clone())),
|
||||||
|
(
|
||||||
|
RepoDedupKey::RsyncScope { .. },
|
||||||
|
RepoTransportResultKind::Failed {
|
||||||
|
error_class: RepoTransportErrorClass::TransportFetch,
|
||||||
|
..
|
||||||
|
},
|
||||||
|
) => Some((RepoTransportMode::Rsync, result.repo_identity.rsync_base_uri.clone())),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
let Some((transport, uri)) = key else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
match &result.result {
|
||||||
|
RepoTransportResultKind::Success { .. } => {
|
||||||
|
if let Some(outcome) = blacklist.record_transport_success(transport, &uri) {
|
||||||
|
crate::progress_log::emit(
|
||||||
|
"dead_repo_blacklist_remove",
|
||||||
|
serde_json::json!({
|
||||||
|
"transport": transport.as_str(),
|
||||||
|
"uri": uri,
|
||||||
|
"reason": match outcome {
|
||||||
|
DeadRepoSuccessOutcome::CounterReset => "counter_reset",
|
||||||
|
DeadRepoSuccessOutcome::BlacklistRemoved => "fetch_success",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RepoTransportResultKind::Failed { .. } => {
|
||||||
|
if let DeadRepoFailureOutcome::Admitted = blacklist.record_transport_failure(
|
||||||
|
transport,
|
||||||
|
&uri,
|
||||||
|
now_unix,
|
||||||
|
blacklist_config.fail_threshold,
|
||||||
|
blacklist_config.capacity,
|
||||||
|
) {
|
||||||
|
crate::progress_log::emit(
|
||||||
|
"dead_repo_blacklist_admit",
|
||||||
|
serde_json::json!({
|
||||||
|
"transport": transport.as_str(),
|
||||||
|
"uri": uri,
|
||||||
|
"fail_threshold": blacklist_config.fail_threshold,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn dead_repo_blacklist(&self) -> Option<&DeadRepoBlacklist> {
|
||||||
|
self.dead_repo_blacklist.as_ref()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn runtime_record(
|
pub fn runtime_record(
|
||||||
&self,
|
&self,
|
||||||
identity: &RepoIdentity,
|
identity: &RepoIdentity,
|
||||||
@ -465,4 +575,288 @@ mod tests {
|
|||||||
assert_eq!(coordinator.stats.repo_tasks_total, 1);
|
assert_eq!(coordinator.stats.repo_tasks_total, 1);
|
||||||
assert_eq!(coordinator.stats.repo_tasks_reused, 1);
|
assert_eq!(coordinator.stats.repo_tasks_reused, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- Dead-repo blacklist (#141) integration tests ----
|
||||||
|
|
||||||
|
use crate::parallel::dead_repo_blacklist::{DeadRepoBlacklist, DeadRepoBlacklistConfig};
|
||||||
|
use crate::parallel::types::{
|
||||||
|
RepoDedupKey, RepoTransportErrorClass, RepoTransportMode, RepoTransportResultEnvelope,
|
||||||
|
RepoTransportResultKind,
|
||||||
|
};
|
||||||
|
|
||||||
|
const DEAD_NOTIFY: &str = "https://dead.example/notification.xml";
|
||||||
|
const DEAD_BASE: &str = "rsync://dead.example/repo/";
|
||||||
|
|
||||||
|
fn blacklist_config(fail_threshold: u32) -> DeadRepoBlacklistConfig {
|
||||||
|
DeadRepoBlacklistConfig {
|
||||||
|
path: std::env::temp_dir().join(format!(
|
||||||
|
"dead_repo_blacklist_coordinator_test_{}.json",
|
||||||
|
std::process::id()
|
||||||
|
)),
|
||||||
|
fail_threshold,
|
||||||
|
capacity: 256,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn coordinator_with_blacklist(
|
||||||
|
fail_threshold: u32,
|
||||||
|
seed: DeadRepoBlacklist,
|
||||||
|
) -> GlobalRunCoordinator {
|
||||||
|
let mut config = ParallelPhase1Config::default();
|
||||||
|
config.dead_repo_blacklist = Some(blacklist_config(fail_threshold));
|
||||||
|
let mut coordinator = GlobalRunCoordinator::new(
|
||||||
|
config,
|
||||||
|
vec![TalInputSpec::from_url("https://example.test/arin.tal")],
|
||||||
|
);
|
||||||
|
coordinator.dead_repo_blacklist = Some(seed.clone());
|
||||||
|
coordinator.transport_tables.set_dead_repo_blacklist(seed);
|
||||||
|
coordinator
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dead_identity() -> RepoIdentity {
|
||||||
|
RepoIdentity::new(Some(DEAD_NOTIFY.to_string()), DEAD_BASE)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dead_requester() -> RepoRequester {
|
||||||
|
requester("arin", "arin", "rsync://dead.example/repo/root.mft")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn transport_envelope(
|
||||||
|
mode: RepoTransportMode,
|
||||||
|
error_class: RepoTransportErrorClass,
|
||||||
|
) -> RepoTransportResultEnvelope {
|
||||||
|
let (dedup_key, detail) = match mode {
|
||||||
|
RepoTransportMode::Rrdp => (
|
||||||
|
RepoDedupKey::RrdpNotify {
|
||||||
|
notification_uri: DEAD_NOTIFY.to_string(),
|
||||||
|
},
|
||||||
|
"http request failed: connect timeout".to_string(),
|
||||||
|
),
|
||||||
|
RepoTransportMode::Rsync => (
|
||||||
|
RepoDedupKey::RsyncScope {
|
||||||
|
rsync_scope_uri: DEAD_BASE.to_string(),
|
||||||
|
},
|
||||||
|
"rsync error: timeout waiting for daemon connection".to_string(),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
RepoTransportResultEnvelope {
|
||||||
|
dedup_key,
|
||||||
|
rsync_failure_scope_uri: None,
|
||||||
|
repo_identity: dead_identity(),
|
||||||
|
mode,
|
||||||
|
tal_id: "arin".to_string(),
|
||||||
|
rir_id: "arin".to_string(),
|
||||||
|
timing_ms: 1,
|
||||||
|
result: RepoTransportResultKind::Failed {
|
||||||
|
detail,
|
||||||
|
warnings: Vec::new(),
|
||||||
|
error_class,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn register_dead(
|
||||||
|
coordinator: &mut GlobalRunCoordinator,
|
||||||
|
) -> crate::parallel::repo_scheduler::TransportRequestAction {
|
||||||
|
coordinator.register_transport_request(
|
||||||
|
dead_identity(),
|
||||||
|
dead_requester(),
|
||||||
|
time::OffsetDateTime::UNIX_EPOCH,
|
||||||
|
0,
|
||||||
|
DEAD_BASE.to_string(),
|
||||||
|
None,
|
||||||
|
SyncPreference::RrdpThenRsync,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn blacklist_admission_after_threshold_transport_failures() {
|
||||||
|
let mut coordinator = coordinator_with_blacklist(2, DeadRepoBlacklist::new());
|
||||||
|
for _ in 0..2 {
|
||||||
|
let action = register_dead(&mut coordinator);
|
||||||
|
assert!(matches!(
|
||||||
|
action,
|
||||||
|
crate::parallel::repo_scheduler::TransportRequestAction::Enqueue(_)
|
||||||
|
));
|
||||||
|
coordinator
|
||||||
|
.complete_transport_result(
|
||||||
|
transport_envelope(
|
||||||
|
RepoTransportMode::Rrdp,
|
||||||
|
RepoTransportErrorClass::TransportFetch,
|
||||||
|
),
|
||||||
|
time::OffsetDateTime::UNIX_EPOCH,
|
||||||
|
)
|
||||||
|
.expect("complete");
|
||||||
|
// Simulate the next run: run-local scheduler state resets, the
|
||||||
|
// persistent blacklist working copy carries over.
|
||||||
|
coordinator.reset_run_state();
|
||||||
|
}
|
||||||
|
let blacklist = coordinator.dead_repo_blacklist().expect("blacklist");
|
||||||
|
assert!(blacklist.is_blacklisted(RepoTransportMode::Rrdp, DEAD_NOTIFY));
|
||||||
|
// Admission mid-run does not affect this run's frozen snapshot: the
|
||||||
|
// scheduler still enqueues rrdp for a fresh identity registration.
|
||||||
|
coordinator.reset_run_state();
|
||||||
|
let action = register_dead(&mut coordinator);
|
||||||
|
assert!(matches!(
|
||||||
|
action,
|
||||||
|
crate::parallel::repo_scheduler::TransportRequestAction::Enqueue(_)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn blacklist_ignores_protocol_and_unknown_failures() {
|
||||||
|
let mut coordinator = coordinator_with_blacklist(1, DeadRepoBlacklist::new());
|
||||||
|
for error_class in [
|
||||||
|
RepoTransportErrorClass::Protocol,
|
||||||
|
RepoTransportErrorClass::Unknown,
|
||||||
|
RepoTransportErrorClass::Storage,
|
||||||
|
] {
|
||||||
|
let action = register_dead(&mut coordinator);
|
||||||
|
assert!(matches!(
|
||||||
|
action,
|
||||||
|
crate::parallel::repo_scheduler::TransportRequestAction::Enqueue(_)
|
||||||
|
));
|
||||||
|
coordinator
|
||||||
|
.complete_transport_result(
|
||||||
|
transport_envelope(RepoTransportMode::Rrdp, error_class),
|
||||||
|
time::OffsetDateTime::UNIX_EPOCH,
|
||||||
|
)
|
||||||
|
.expect("complete");
|
||||||
|
coordinator.reset_run_state();
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
coordinator
|
||||||
|
.dead_repo_blacklist()
|
||||||
|
.expect("blacklist")
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn blacklist_skip_rrdp_routes_straight_to_rsync() {
|
||||||
|
let mut seed = DeadRepoBlacklist::new();
|
||||||
|
seed.record_transport_failure(RepoTransportMode::Rrdp, DEAD_NOTIFY, 100, 1, 256);
|
||||||
|
let mut coordinator = coordinator_with_blacklist(3, seed);
|
||||||
|
let action = register_dead(&mut coordinator);
|
||||||
|
let task = match action {
|
||||||
|
crate::parallel::repo_scheduler::TransportRequestAction::Enqueue(task) => task,
|
||||||
|
other => panic!("expected rsync enqueue, got {other:?}"),
|
||||||
|
};
|
||||||
|
assert_eq!(task.mode, RepoTransportMode::Rsync);
|
||||||
|
// register_dead passes retry_short_timeout=false (mirroring a prefetch
|
||||||
|
// snapshot that recorded no real rrdp failure for skipped runs); the
|
||||||
|
// skip path must still keep the short retry profile (#141).
|
||||||
|
assert!(task.retry_short_timeout);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn blacklist_dual_dead_terminates_immediately() {
|
||||||
|
let mut seed = DeadRepoBlacklist::new();
|
||||||
|
seed.record_transport_failure(RepoTransportMode::Rrdp, DEAD_NOTIFY, 100, 1, 256);
|
||||||
|
seed.record_transport_failure(RepoTransportMode::Rsync, DEAD_BASE, 100, 1, 256);
|
||||||
|
let mut coordinator = coordinator_with_blacklist(3, seed);
|
||||||
|
let action = register_dead(&mut coordinator);
|
||||||
|
let envelope = match action {
|
||||||
|
crate::parallel::repo_scheduler::TransportRequestAction::ReusedTerminalFailure(
|
||||||
|
envelope,
|
||||||
|
) => envelope,
|
||||||
|
other => panic!("expected terminal failure, got {other:?}"),
|
||||||
|
};
|
||||||
|
match &envelope.result {
|
||||||
|
RepoTransportResultKind::Failed {
|
||||||
|
detail, warnings, ..
|
||||||
|
} => {
|
||||||
|
assert!(detail.contains("dead repo blacklist"));
|
||||||
|
assert!(warnings
|
||||||
|
.iter()
|
||||||
|
.any(|warning| warning.message.contains("dead_repo_blacklist_skip_all")));
|
||||||
|
}
|
||||||
|
other => panic!("expected failed result, got {other:?}"),
|
||||||
|
}
|
||||||
|
let record = coordinator
|
||||||
|
.runtime_record(&dead_identity())
|
||||||
|
.expect("record");
|
||||||
|
assert_eq!(
|
||||||
|
record.state,
|
||||||
|
crate::parallel::types::RepoRuntimeState::FailedTerminal
|
||||||
|
);
|
||||||
|
assert_eq!(coordinator.stats.repo_tasks_total, 0);
|
||||||
|
assert_eq!(coordinator.stats.repo_tasks_reused, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn blacklist_rsync_dead_terminates_after_rrdp_failure() {
|
||||||
|
// Only rsync blacklisted: rrdp is attempted; when it fails, the rsync
|
||||||
|
// fallback is short-circuited into an instant terminal failure.
|
||||||
|
let mut seed = DeadRepoBlacklist::new();
|
||||||
|
seed.record_transport_failure(RepoTransportMode::Rsync, DEAD_BASE, 100, 1, 256);
|
||||||
|
let mut coordinator = coordinator_with_blacklist(3, seed);
|
||||||
|
let action = register_dead(&mut coordinator);
|
||||||
|
assert!(matches!(
|
||||||
|
action,
|
||||||
|
crate::parallel::repo_scheduler::TransportRequestAction::Enqueue(_)
|
||||||
|
));
|
||||||
|
let completion = coordinator
|
||||||
|
.complete_transport_result(
|
||||||
|
transport_envelope(RepoTransportMode::Rrdp, RepoTransportErrorClass::TransportFetch),
|
||||||
|
time::OffsetDateTime::UNIX_EPOCH,
|
||||||
|
)
|
||||||
|
.expect("complete");
|
||||||
|
assert!(completion.follow_up_tasks.is_empty());
|
||||||
|
let record = coordinator
|
||||||
|
.runtime_record(&dead_identity())
|
||||||
|
.expect("record");
|
||||||
|
assert_eq!(
|
||||||
|
record.state,
|
||||||
|
crate::parallel::types::RepoRuntimeState::FailedTerminal
|
||||||
|
);
|
||||||
|
let terminal = record.terminal_failure.as_ref().expect("terminal failure");
|
||||||
|
match &terminal.result {
|
||||||
|
RepoTransportResultKind::Failed { detail, .. } => {
|
||||||
|
assert!(detail.contains("dead repo blacklist"));
|
||||||
|
}
|
||||||
|
other => panic!("expected failed result, got {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn blacklist_success_resets_failure_counter() {
|
||||||
|
let mut coordinator = coordinator_with_blacklist(3, DeadRepoBlacklist::new());
|
||||||
|
// One transport failure starts counting.
|
||||||
|
let action = register_dead(&mut coordinator);
|
||||||
|
assert!(matches!(
|
||||||
|
action,
|
||||||
|
crate::parallel::repo_scheduler::TransportRequestAction::Enqueue(_)
|
||||||
|
));
|
||||||
|
coordinator
|
||||||
|
.complete_transport_result(
|
||||||
|
transport_envelope(RepoTransportMode::Rrdp, RepoTransportErrorClass::TransportFetch),
|
||||||
|
time::OffsetDateTime::UNIX_EPOCH,
|
||||||
|
)
|
||||||
|
.expect("complete");
|
||||||
|
assert_eq!(coordinator.dead_repo_blacklist().expect("blacklist").len(), 1);
|
||||||
|
|
||||||
|
// A later success clears the entry.
|
||||||
|
coordinator.reset_run_state();
|
||||||
|
let action = register_dead(&mut coordinator);
|
||||||
|
assert!(matches!(
|
||||||
|
action,
|
||||||
|
crate::parallel::repo_scheduler::TransportRequestAction::Enqueue(_)
|
||||||
|
));
|
||||||
|
let mut success = transport_envelope(RepoTransportMode::Rrdp, RepoTransportErrorClass::Unknown);
|
||||||
|
success.result = RepoTransportResultKind::Success {
|
||||||
|
source: "rrdp".to_string(),
|
||||||
|
warnings: Vec::new(),
|
||||||
|
};
|
||||||
|
coordinator
|
||||||
|
.complete_transport_result(success, time::OffsetDateTime::UNIX_EPOCH)
|
||||||
|
.expect("complete");
|
||||||
|
assert!(
|
||||||
|
coordinator
|
||||||
|
.dead_repo_blacklist()
|
||||||
|
.expect("blacklist")
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -484,6 +484,7 @@ mod tests {
|
|||||||
result: crate::parallel::types::RepoTransportResultKind::Failed {
|
result: crate::parallel::types::RepoTransportResultKind::Failed {
|
||||||
detail: "timeout".to_string(),
|
detail: "timeout".to_string(),
|
||||||
warnings: Vec::new(),
|
warnings: Vec::new(),
|
||||||
|
error_class: crate::parallel::types::RepoTransportErrorClass::Unknown,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
recorder.record_result(&crate::parallel::types::RepoTransportResultEnvelope {
|
recorder.record_result(&crate::parallel::types::RepoTransportResultEnvelope {
|
||||||
@ -499,6 +500,7 @@ mod tests {
|
|||||||
result: crate::parallel::types::RepoTransportResultKind::Failed {
|
result: crate::parallel::types::RepoTransportResultKind::Failed {
|
||||||
detail: "timeout".to_string(),
|
detail: "timeout".to_string(),
|
||||||
warnings: Vec::new(),
|
warnings: Vec::new(),
|
||||||
|
error_class: crate::parallel::types::RepoTransportErrorClass::Unknown,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@ -1,5 +1,7 @@
|
|||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::policy::SyncPreference;
|
use crate::policy::SyncPreference;
|
||||||
use crate::report::Warning;
|
use crate::report::Warning;
|
||||||
|
|
||||||
@ -96,7 +98,8 @@ pub enum RepoDedupKey {
|
|||||||
RsyncScope { rsync_scope_uri: String },
|
RsyncScope { rsync_scope_uri: String },
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
pub enum RepoTransportMode {
|
pub enum RepoTransportMode {
|
||||||
Rrdp,
|
Rrdp,
|
||||||
Rsync,
|
Rsync,
|
||||||
@ -116,6 +119,26 @@ pub struct RepoTransportTask {
|
|||||||
pub requesters: Vec<RepoRequester>,
|
pub requesters: Vec<RepoRequester>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum RepoTransportErrorClass {
|
||||||
|
/// Transport-layer fetch failure (connect timeout/refused/TLS/DNS/broken
|
||||||
|
/// connection). Only this class counts toward the dead-repo blacklist.
|
||||||
|
TransportFetch,
|
||||||
|
/// Protocol-level failure (HTTP status, XML/parse, content mismatch).
|
||||||
|
Protocol,
|
||||||
|
/// Local storage failure.
|
||||||
|
Storage,
|
||||||
|
/// Unclassified (synthesized or legacy results); never counted.
|
||||||
|
Unknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for RepoTransportErrorClass {
|
||||||
|
fn default() -> Self {
|
||||||
|
RepoTransportErrorClass::Unknown
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
pub enum RepoTransportResultKind {
|
pub enum RepoTransportResultKind {
|
||||||
Success {
|
Success {
|
||||||
@ -125,6 +148,7 @@ pub enum RepoTransportResultKind {
|
|||||||
Failed {
|
Failed {
|
||||||
detail: String,
|
detail: String,
|
||||||
warnings: Vec<Warning>,
|
warnings: Vec<Warning>,
|
||||||
|
error_class: RepoTransportErrorClass,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -479,6 +503,7 @@ mod tests {
|
|||||||
result: RepoTransportResultKind::Failed {
|
result: RepoTransportResultKind::Failed {
|
||||||
detail: "timeout".to_string(),
|
detail: "timeout".to_string(),
|
||||||
warnings: vec![Warning::new("timeout")],
|
warnings: vec![Warning::new("timeout")],
|
||||||
|
error_class: super::RepoTransportErrorClass::Unknown,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
assert!(matches!(ok.result, RepoTransportResultKind::Success { .. }));
|
assert!(matches!(ok.result, RepoTransportResultKind::Success { .. }));
|
||||||
|
|||||||
@ -433,6 +433,8 @@ struct LatestRunMetrics {
|
|||||||
download_bytes: Option<u64>,
|
download_bytes: Option<u64>,
|
||||||
artifact_sizes: BTreeMap<String, u64>,
|
artifact_sizes: BTreeMap<String, u64>,
|
||||||
state_path_sizes: BTreeMap<String, PathSize>,
|
state_path_sizes: BTreeMap<String, PathSize>,
|
||||||
|
dead_repo_blacklist_entries: Option<u64>,
|
||||||
|
dead_repo_blacklist_blacklisted: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default, Serialize)]
|
#[derive(Clone, Debug, Default, Serialize)]
|
||||||
@ -1377,6 +1379,10 @@ fn build_latest_metrics(record: &RunRecord, snapshot: &mut MetricsSnapshot) {
|
|||||||
latest.download_bytes = json_u64(summary, &["stageTiming", "download_bytes_total"]);
|
latest.download_bytes = json_u64(summary, &["stageTiming", "download_bytes_total"]);
|
||||||
latest.artifact_sizes = extract_artifact_sizes(summary.get("artifacts"));
|
latest.artifact_sizes = extract_artifact_sizes(summary.get("artifacts"));
|
||||||
latest.state_path_sizes = extract_path_sizes(summary.get("pathStats"));
|
latest.state_path_sizes = extract_path_sizes(summary.get("pathStats"));
|
||||||
|
latest.dead_repo_blacklist_entries =
|
||||||
|
json_u64(summary, &["deadRepoBlacklist", "entries"]);
|
||||||
|
latest.dead_repo_blacklist_blacklisted =
|
||||||
|
json_u64(summary, &["deadRepoBlacklist", "blacklisted"]);
|
||||||
}
|
}
|
||||||
|
|
||||||
parse_report(&record.path.join("report.json"), snapshot, &mut latest);
|
parse_report(&record.path.join("report.json"), snapshot, &mut latest);
|
||||||
@ -2419,6 +2425,22 @@ fn render_latest_metrics(writer: &mut PromWriter<'_>, instance: &str, latest: &L
|
|||||||
value as f64,
|
value as f64,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if let Some(value) = latest.dead_repo_blacklist_entries {
|
||||||
|
writer.gauge(
|
||||||
|
"ours_rp_dead_repo_blacklist_entries",
|
||||||
|
"Dead-repo transport blacklist entries (latest run)",
|
||||||
|
&[label("instance", instance), label("state", "total")],
|
||||||
|
value as f64,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if let Some(value) = latest.dead_repo_blacklist_blacklisted {
|
||||||
|
writer.gauge(
|
||||||
|
"ours_rp_dead_repo_blacklist_entries",
|
||||||
|
"Dead-repo transport blacklist entries (latest run)",
|
||||||
|
&[label("instance", instance), label("state", "blacklisted")],
|
||||||
|
value as f64,
|
||||||
|
);
|
||||||
|
}
|
||||||
for (stage, value) in &latest.stage_seconds {
|
for (stage, value) in &latest.stage_seconds {
|
||||||
writer.gauge(
|
writer.gauge(
|
||||||
"ours_rp_run_stage_duration_seconds",
|
"ours_rp_run_stage_duration_seconds",
|
||||||
@ -3893,4 +3915,37 @@ mod tests {
|
|||||||
});
|
});
|
||||||
encode_content_info(&ci).expect("encode ccr")
|
encode_content_info(&ci).expect("encode ccr")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dead_repo_blacklist_gauges_render_from_latest_summary() {
|
||||||
|
let td = TempDir::new().expect("tempdir");
|
||||||
|
write_success_run(td.path(), "run_0001", 1);
|
||||||
|
// Rewrite the summary with the blacklist section present.
|
||||||
|
let run = td.path().join("runs").join("run_0001");
|
||||||
|
fs::write(
|
||||||
|
run.join("run-summary.json"),
|
||||||
|
r#"{"runSeq":1,"runId":"run_0001","startedAtRfc3339Utc":"2026-07-21T00:00:00Z","finishedAtRfc3339Utc":"2026-07-21T00:00:10Z","wallMs":10000,"status":"success","exitCode":0,"deadRepoBlacklist":{"path":"/state/dead-repo-blacklist.json","entries":5,"blacklisted":3}}"#,
|
||||||
|
)
|
||||||
|
.expect("summary");
|
||||||
|
let snapshot = scan_run_root(td.path(), "test").expect("scan");
|
||||||
|
let latest = snapshot.latest_run.as_ref().expect("latest run");
|
||||||
|
assert_eq!(latest.dead_repo_blacklist_entries, Some(5));
|
||||||
|
assert_eq!(latest.dead_repo_blacklist_blacklisted, Some(3));
|
||||||
|
let metrics = render_metrics(&snapshot);
|
||||||
|
assert!(metrics.contains(
|
||||||
|
r#"ours_rp_dead_repo_blacklist_entries{instance="test",state="total"} 5"#
|
||||||
|
));
|
||||||
|
assert!(metrics.contains(
|
||||||
|
r#"ours_rp_dead_repo_blacklist_entries{instance="test",state="blacklisted"} 3"#
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dead_repo_blacklist_gauges_absent_without_section() {
|
||||||
|
let td = TempDir::new().expect("tempdir");
|
||||||
|
write_success_run(td.path(), "run_0001", 1);
|
||||||
|
let snapshot = scan_run_root(td.path(), "test").expect("scan");
|
||||||
|
let metrics = render_metrics(&snapshot);
|
||||||
|
assert!(!metrics.contains("ours_rp_dead_repo_blacklist_entries"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -6,6 +6,9 @@ use std::path::{Path, PathBuf};
|
|||||||
use std::process::{Command, Stdio};
|
use std::process::{Command, Stdio};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use crate::parallel::dead_repo_blacklist::DeadRepoBlacklist;
|
||||||
|
use crate::parallel::types::RepoTransportMode;
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
struct Args {
|
struct Args {
|
||||||
state_root: PathBuf,
|
state_root: PathBuf,
|
||||||
@ -21,6 +24,12 @@ struct Args {
|
|||||||
db_stats_bin: Option<PathBuf>,
|
db_stats_bin: Option<PathBuf>,
|
||||||
db_stats_exact_every: Option<u64>,
|
db_stats_exact_every: Option<u64>,
|
||||||
time_bin: Option<PathBuf>,
|
time_bin: Option<PathBuf>,
|
||||||
|
/// Dead-repo blacklist (#141): enables daemon-side health checks and
|
||||||
|
/// auto-injects the child `--dead-repo-blacklist` flag when absent.
|
||||||
|
dead_repo_blacklist: Option<PathBuf>,
|
||||||
|
dead_repo_blacklist_fail_threshold: Option<u32>,
|
||||||
|
dead_repo_health_check_interval_secs: u64,
|
||||||
|
dead_repo_probe_rsync_bin: PathBuf,
|
||||||
child_args: Vec<String>,
|
child_args: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -52,6 +61,23 @@ struct DaemonStatus {
|
|||||||
current_run_seq: Option<u64>,
|
current_run_seq: Option<u64>,
|
||||||
current_run_id: Option<String>,
|
current_run_id: Option<String>,
|
||||||
last_run_id: Option<String>,
|
last_run_id: Option<String>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
dead_repo_blacklist: Option<DeadRepoBlacklistStatus>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
struct DeadRepoBlacklistStatus {
|
||||||
|
path: String,
|
||||||
|
entries: usize,
|
||||||
|
blacklisted: usize,
|
||||||
|
last_health_check_at_rfc3339_utc: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Mutable daemon-side health check bookkeeping (#141).
|
||||||
|
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||||
|
struct DeadRepoHealthRuntime {
|
||||||
|
last_health_check_at: Option<time::OffsetDateTime>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
|
#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
|
||||||
@ -145,6 +171,16 @@ struct RunSummary {
|
|||||||
db_stats: Vec<DbStatsSummary>,
|
db_stats: Vec<DbStatsSummary>,
|
||||||
retention_deleted_runs: Vec<String>,
|
retention_deleted_runs: Vec<String>,
|
||||||
artifacts: Vec<ArtifactInfo>,
|
artifacts: Vec<ArtifactInfo>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
dead_repo_blacklist: Option<DeadRepoBlacklistRunSummary>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
struct DeadRepoBlacklistRunSummary {
|
||||||
|
path: String,
|
||||||
|
entries: usize,
|
||||||
|
blacklisted: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn usage() -> String {
|
fn usage() -> String {
|
||||||
@ -170,6 +206,17 @@ Options:
|
|||||||
Run db_stats --exact every n runs (default: disabled)
|
Run db_stats --exact every n runs (default: disabled)
|
||||||
--time-bin <path> GNU time binary for child process metrics (default: /usr/bin/time when present)
|
--time-bin <path> GNU time binary for child process metrics (default: /usr/bin/time when present)
|
||||||
--no-time-wrapper Disable GNU time wrapper
|
--no-time-wrapper Disable GNU time wrapper
|
||||||
|
--dead-repo-blacklist <path>
|
||||||
|
Enable the dead-repo transport blacklist (#141): daemon probes
|
||||||
|
blacklisted entries between runs and removes revived repos;
|
||||||
|
auto-injects the child flag when not already present
|
||||||
|
--dead-repo-blacklist-fail-threshold <n>
|
||||||
|
Admission threshold forwarded to the child when its args do not
|
||||||
|
already set it (default: child default 3)
|
||||||
|
--dead-repo-health-check-interval-secs <n>
|
||||||
|
Minimum seconds between health check sweeps (default: 600)
|
||||||
|
--dead-repo-probe-rsync-bin <path>
|
||||||
|
rsync binary used for health probes (default: rsync)
|
||||||
--help Show this help
|
--help Show this help
|
||||||
|
|
||||||
Child argument placeholders:
|
Child argument placeholders:
|
||||||
@ -211,12 +258,16 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
|||||||
let mut db_stats_exact_every = None;
|
let mut db_stats_exact_every = None;
|
||||||
let mut time_bin = default_time_bin();
|
let mut time_bin = default_time_bin();
|
||||||
let mut no_time_wrapper = false;
|
let mut no_time_wrapper = false;
|
||||||
|
let mut dead_repo_blacklist: Option<PathBuf> = None;
|
||||||
|
let mut dead_repo_blacklist_fail_threshold: Option<u32> = None;
|
||||||
|
let mut dead_repo_health_check_interval_secs = 600u64;
|
||||||
|
let mut dead_repo_probe_rsync_bin = PathBuf::from("rsync");
|
||||||
|
|
||||||
let mut i = 1usize;
|
let mut i = 1usize;
|
||||||
while i < argv.len() {
|
while i < argv.len() {
|
||||||
match argv[i].as_str() {
|
match argv[i].as_str() {
|
||||||
"--" => {
|
"--" => {
|
||||||
let child_args = argv[i + 1..].to_vec();
|
let mut child_args = argv[i + 1..].to_vec();
|
||||||
let state_root =
|
let state_root =
|
||||||
state_root.ok_or_else(|| format!("--state-root is required\n\n{}", usage()))?;
|
state_root.ok_or_else(|| format!("--state-root is required\n\n{}", usage()))?;
|
||||||
let work_db = work_db.unwrap_or_else(|| state_root.join("state").join("work-db"));
|
let work_db = work_db.unwrap_or_else(|| state_root.join("state").join("work-db"));
|
||||||
@ -225,6 +276,23 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
|||||||
if no_time_wrapper {
|
if no_time_wrapper {
|
||||||
time_bin = None;
|
time_bin = None;
|
||||||
}
|
}
|
||||||
|
// Dead-repo blacklist (#141): forward enablement to the child
|
||||||
|
// unless its args already carry the flag explicitly.
|
||||||
|
if let Some(path) = dead_repo_blacklist.as_ref() {
|
||||||
|
if !child_args.iter().any(|arg| arg == "--dead-repo-blacklist") {
|
||||||
|
child_args.push("--dead-repo-blacklist".to_string());
|
||||||
|
child_args.push(path_string(path));
|
||||||
|
}
|
||||||
|
if let Some(threshold) = dead_repo_blacklist_fail_threshold {
|
||||||
|
if !child_args
|
||||||
|
.iter()
|
||||||
|
.any(|arg| arg == "--dead-repo-blacklist-fail-threshold")
|
||||||
|
{
|
||||||
|
child_args.push("--dead-repo-blacklist-fail-threshold".to_string());
|
||||||
|
child_args.push(threshold.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
let args = Args {
|
let args = Args {
|
||||||
state_root,
|
state_root,
|
||||||
rpki_bin: rpki_bin
|
rpki_bin: rpki_bin
|
||||||
@ -240,6 +308,10 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
|||||||
db_stats_bin,
|
db_stats_bin,
|
||||||
db_stats_exact_every,
|
db_stats_exact_every,
|
||||||
time_bin,
|
time_bin,
|
||||||
|
dead_repo_blacklist,
|
||||||
|
dead_repo_blacklist_fail_threshold,
|
||||||
|
dead_repo_health_check_interval_secs,
|
||||||
|
dead_repo_probe_rsync_bin,
|
||||||
child_args,
|
child_args,
|
||||||
};
|
};
|
||||||
return validate_args(args);
|
return validate_args(args);
|
||||||
@ -315,6 +387,33 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
|||||||
"--no-time-wrapper" => {
|
"--no-time-wrapper" => {
|
||||||
no_time_wrapper = true;
|
no_time_wrapper = true;
|
||||||
}
|
}
|
||||||
|
"--dead-repo-blacklist" => {
|
||||||
|
i += 1;
|
||||||
|
dead_repo_blacklist = Some(PathBuf::from(value_at(argv, i, "--dead-repo-blacklist")?));
|
||||||
|
}
|
||||||
|
"--dead-repo-blacklist-fail-threshold" => {
|
||||||
|
i += 1;
|
||||||
|
let parsed = parse_u64(
|
||||||
|
value_at(argv, i, "--dead-repo-blacklist-fail-threshold")?,
|
||||||
|
"--dead-repo-blacklist-fail-threshold",
|
||||||
|
)?;
|
||||||
|
if parsed == 0 || parsed > u32::MAX as u64 {
|
||||||
|
return Err("--dead-repo-blacklist-fail-threshold must be in 1..=u32::MAX".to_string());
|
||||||
|
}
|
||||||
|
dead_repo_blacklist_fail_threshold = Some(parsed as u32);
|
||||||
|
}
|
||||||
|
"--dead-repo-health-check-interval-secs" => {
|
||||||
|
i += 1;
|
||||||
|
dead_repo_health_check_interval_secs = parse_u64(
|
||||||
|
value_at(argv, i, "--dead-repo-health-check-interval-secs")?,
|
||||||
|
"--dead-repo-health-check-interval-secs",
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
"--dead-repo-probe-rsync-bin" => {
|
||||||
|
i += 1;
|
||||||
|
dead_repo_probe_rsync_bin =
|
||||||
|
PathBuf::from(value_at(argv, i, "--dead-repo-probe-rsync-bin")?);
|
||||||
|
}
|
||||||
other => return Err(format!("unknown argument: {other}\n\n{}", usage())),
|
other => return Err(format!("unknown argument: {other}\n\n{}", usage())),
|
||||||
}
|
}
|
||||||
i += 1;
|
i += 1;
|
||||||
@ -330,6 +429,11 @@ fn validate_args(args: Args) -> Result<Args, String> {
|
|||||||
usage()
|
usage()
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
if args.dead_repo_blacklist_fail_threshold.is_some() && args.dead_repo_blacklist.is_none() {
|
||||||
|
return Err(
|
||||||
|
"--dead-repo-blacklist-fail-threshold requires --dead-repo-blacklist".to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
Ok(args)
|
Ok(args)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -387,10 +491,19 @@ fn write_json_pretty<T: Serialize>(path: &Path, value: &T) -> Result<(), String>
|
|||||||
fs::create_dir_all(parent)
|
fs::create_dir_all(parent)
|
||||||
.map_err(|e| format!("create parent dir failed: {}: {e}", parent.display()))?;
|
.map_err(|e| format!("create parent dir failed: {}: {e}", parent.display()))?;
|
||||||
}
|
}
|
||||||
let file =
|
let bytes = serde_json::to_vec_pretty(value)
|
||||||
File::create(path).map_err(|e| format!("create json failed: {}: {e}", path.display()))?;
|
.map_err(|e| format!("serialize json failed: {}: {e}", path.display()))?;
|
||||||
serde_json::to_writer_pretty(file, value)
|
// Atomic write (tmp + rename) so concurrent readers never see a torn file.
|
||||||
.map_err(|e| format!("write json failed: {}: {e}", path.display()))
|
let mut tmp_name = path
|
||||||
|
.file_name()
|
||||||
|
.map(|name| name.to_os_string())
|
||||||
|
.unwrap_or_default();
|
||||||
|
tmp_name.push(".tmp");
|
||||||
|
let tmp_path = path.with_file_name(tmp_name);
|
||||||
|
fs::write(&tmp_path, &bytes)
|
||||||
|
.map_err(|e| format!("write json tmp failed: {}: {e}", tmp_path.display()))?;
|
||||||
|
fs::rename(&tmp_path, path)
|
||||||
|
.map_err(|e| format!("rename json tmp failed: {} -> {}: {e}", tmp_path.display(), path.display()))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn append_json_line<T: Serialize>(path: &Path, value: &T) -> Result<(), String> {
|
fn append_json_line<T: Serialize>(path: &Path, value: &T) -> Result<(), String> {
|
||||||
@ -409,12 +522,30 @@ fn append_json_line<T: Serialize>(path: &Path, value: &T) -> Result<(), String>
|
|||||||
.map_err(|e| format!("flush jsonl failed: {}: {e}", path.display()))
|
.map_err(|e| format!("flush jsonl failed: {}: {e}", path.display()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn dead_repo_blacklist_status(
|
||||||
|
args: &Args,
|
||||||
|
health: &DeadRepoHealthRuntime,
|
||||||
|
) -> Option<DeadRepoBlacklistStatus> {
|
||||||
|
let path = args.dead_repo_blacklist.as_ref()?;
|
||||||
|
let (blacklist, _) = DeadRepoBlacklist::load(path);
|
||||||
|
let last_health_check_at_rfc3339_utc = health
|
||||||
|
.last_health_check_at
|
||||||
|
.and_then(|t| format_rfc3339(t).ok());
|
||||||
|
Some(DeadRepoBlacklistStatus {
|
||||||
|
path: path_string(path),
|
||||||
|
entries: blacklist.len(),
|
||||||
|
blacklisted: blacklist.blacklisted_len(),
|
||||||
|
last_health_check_at_rfc3339_utc,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn write_status(
|
fn write_status(
|
||||||
args: &Args,
|
args: &Args,
|
||||||
state: DaemonState,
|
state: DaemonState,
|
||||||
runs_completed: u64,
|
runs_completed: u64,
|
||||||
current: Option<&RunContext>,
|
current: Option<&RunContext>,
|
||||||
last_run_id: Option<String>,
|
last_run_id: Option<String>,
|
||||||
|
health: &DeadRepoHealthRuntime,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
let updated_at_rfc3339_utc = format_rfc3339(utc_now())?;
|
let updated_at_rfc3339_utc = format_rfc3339(utc_now())?;
|
||||||
let status = DaemonStatus {
|
let status = DaemonStatus {
|
||||||
@ -425,10 +556,131 @@ fn write_status(
|
|||||||
current_run_seq: current.map(|ctx| ctx.seq),
|
current_run_seq: current.map(|ctx| ctx.seq),
|
||||||
current_run_id: current.map(|ctx| ctx.run_id.clone()),
|
current_run_id: current.map(|ctx| ctx.run_id.clone()),
|
||||||
last_run_id,
|
last_run_id,
|
||||||
|
dead_repo_blacklist: dead_repo_blacklist_status(args, health),
|
||||||
};
|
};
|
||||||
write_json_pretty(&status_path(args), &status)
|
write_json_pretty(&status_path(args), &status)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Short-timeout HTTP GET of the RRDP notification file. Any HTTP status
|
||||||
|
/// (even 4xx) proves the transport is alive; only transport errors keep the
|
||||||
|
/// entry blacklisted.
|
||||||
|
fn probe_rrdp_transport(uri: &str) -> bool {
|
||||||
|
let config = crate::fetch::http::HttpFetcherConfig {
|
||||||
|
connect_timeout: Duration::from_secs(3),
|
||||||
|
timeout: Duration::from_secs(6),
|
||||||
|
large_body_timeout: Duration::from_secs(6),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let fetcher = match crate::fetch::http::BlockingHttpFetcher::new(config) {
|
||||||
|
Ok(fetcher) => fetcher,
|
||||||
|
Err(_) => return false,
|
||||||
|
};
|
||||||
|
match fetcher.fetch_bytes(uri) {
|
||||||
|
Ok(_) => true,
|
||||||
|
Err(err) => err.starts_with("http status"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// rsync list-only probe: connecting to the daemon and listing the module
|
||||||
|
/// root is enough to prove the transport is alive.
|
||||||
|
fn probe_rsync_transport(rsync_bin: &Path, base_uri: &str) -> bool {
|
||||||
|
Command::new(rsync_bin)
|
||||||
|
.arg("--contimeout=5")
|
||||||
|
.arg("--timeout=8")
|
||||||
|
.arg(base_uri)
|
||||||
|
.stdout(Stdio::null())
|
||||||
|
.stderr(Stdio::null())
|
||||||
|
.status()
|
||||||
|
.map(|status| status.success())
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Probe due blacklist entries between runs (#141). Runs in the supervisor
|
||||||
|
/// loop while no child is active, preserving the single-writer assumption.
|
||||||
|
fn maybe_run_dead_repo_health_check(args: &Args, health: &mut DeadRepoHealthRuntime) {
|
||||||
|
let Some(path) = args.dead_repo_blacklist.clone() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let now = utc_now();
|
||||||
|
if let Some(last) = health.last_health_check_at {
|
||||||
|
let elapsed = (now - last).whole_seconds();
|
||||||
|
if elapsed >= 0 && (elapsed as u64) < args.dead_repo_health_check_interval_secs {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let (mut blacklist, warning) = DeadRepoBlacklist::load(&path);
|
||||||
|
if let Some(warning) = warning {
|
||||||
|
eprintln!("[dead-repo-health] {warning}");
|
||||||
|
}
|
||||||
|
let now_unix = now.unix_timestamp().max(0) as u64;
|
||||||
|
let due = blacklist.probe_due_entries(now_unix, args.dead_repo_health_check_interval_secs);
|
||||||
|
if due.is_empty() {
|
||||||
|
health.last_health_check_at = Some(now);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Probe in parallel: in the docker soak the daemon runs once per run
|
||||||
|
// (--max-runs 1), so a serial sweep of ~12 dead entries would add up to a
|
||||||
|
// minute of latency to every run.
|
||||||
|
let rsync_bin = args.dead_repo_probe_rsync_bin.clone();
|
||||||
|
let probe_results: Vec<(RepoTransportMode, String, bool)> = due
|
||||||
|
.iter()
|
||||||
|
.map(|entry| (entry.transport, entry.uri.clone()))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.into_iter()
|
||||||
|
.map(|(transport, uri)| {
|
||||||
|
let rsync_bin = rsync_bin.clone();
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
let alive = match transport {
|
||||||
|
RepoTransportMode::Rrdp => probe_rrdp_transport(&uri),
|
||||||
|
RepoTransportMode::Rsync => probe_rsync_transport(&rsync_bin, &uri),
|
||||||
|
};
|
||||||
|
(transport, uri, alive)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.filter_map(|handle| handle.join().ok())
|
||||||
|
.collect();
|
||||||
|
let mut removed = 0usize;
|
||||||
|
let mut still_dead = 0usize;
|
||||||
|
for (transport, uri, alive) in &probe_results {
|
||||||
|
if *alive {
|
||||||
|
blacklist.record_probe_success(*transport, uri);
|
||||||
|
removed += 1;
|
||||||
|
eprintln!("[dead-repo-health] removed {} {} (probe ok)", transport.as_str(), uri);
|
||||||
|
crate::progress_log::emit(
|
||||||
|
"dead_repo_blacklist_remove",
|
||||||
|
serde_json::json!({
|
||||||
|
"transport": transport.as_str(),
|
||||||
|
"uri": uri,
|
||||||
|
"reason": "health_probe_ok",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
blacklist.record_probe_failure(*transport, uri, now_unix);
|
||||||
|
still_dead += 1;
|
||||||
|
eprintln!(
|
||||||
|
"[dead-repo-health] still dead {} {} (probe failed)",
|
||||||
|
transport.as_str(),
|
||||||
|
uri
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Err(err) = blacklist.store_atomic(&path, now_unix) {
|
||||||
|
eprintln!(
|
||||||
|
"[dead-repo-health] persist blacklist failed: {}: {err}",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
crate::progress_log::emit(
|
||||||
|
"dead_repo_health_check",
|
||||||
|
serde_json::json!({
|
||||||
|
"probed": probe_results.len(),
|
||||||
|
"removed": removed,
|
||||||
|
"still_dead": still_dead,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
health.last_health_check_at = Some(now);
|
||||||
|
}
|
||||||
|
|
||||||
fn render_child_args(args: &[String], daemon_args: &Args, ctx: &RunContext) -> Vec<String> {
|
fn render_child_args(args: &[String], daemon_args: &Args, ctx: &RunContext) -> Vec<String> {
|
||||||
args.iter()
|
args.iter()
|
||||||
.map(|arg| {
|
.map(|arg| {
|
||||||
@ -634,6 +886,7 @@ fn run_child_once(args: &Args, ctx: &RunContext) -> Result<RunSummary, String> {
|
|||||||
db_stats: Vec::new(),
|
db_stats: Vec::new(),
|
||||||
retention_deleted_runs: Vec::new(),
|
retention_deleted_runs: Vec::new(),
|
||||||
artifacts,
|
artifacts,
|
||||||
|
dead_repo_blacklist: None,
|
||||||
};
|
};
|
||||||
Ok(summary)
|
Ok(summary)
|
||||||
}
|
}
|
||||||
@ -1031,6 +1284,14 @@ fn collect_post_run_metrics(args: &Args, ctx: &RunContext, summary: &mut RunSumm
|
|||||||
summary.path_stats = collect_state_path_stats(args, ctx);
|
summary.path_stats = collect_state_path_stats(args, ctx);
|
||||||
summary.db_stats = collect_db_stats(args, ctx);
|
summary.db_stats = collect_db_stats(args, ctx);
|
||||||
summary.artifacts = collect_artifacts(&ctx.run_dir).unwrap_or_default();
|
summary.artifacts = collect_artifacts(&ctx.run_dir).unwrap_or_default();
|
||||||
|
if let Some(path) = args.dead_repo_blacklist.as_ref() {
|
||||||
|
let (blacklist, _) = DeadRepoBlacklist::load(path);
|
||||||
|
summary.dead_repo_blacklist = Some(DeadRepoBlacklistRunSummary {
|
||||||
|
path: path_string(path),
|
||||||
|
entries: blacklist.len(),
|
||||||
|
blacklisted: blacklist.blacklisted_len(),
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn run_daemon(args: &Args) -> Result<(), String> {
|
fn run_daemon(args: &Args) -> Result<(), String> {
|
||||||
@ -1050,13 +1311,17 @@ fn run_daemon(args: &Args) -> Result<(), String> {
|
|||||||
let mut runs_completed = 0u64;
|
let mut runs_completed = 0u64;
|
||||||
let mut next_seq = 1u64;
|
let mut next_seq = 1u64;
|
||||||
let mut last_run_id = None;
|
let mut last_run_id = None;
|
||||||
|
let mut health = DeadRepoHealthRuntime::default();
|
||||||
write_status(
|
write_status(
|
||||||
args,
|
args,
|
||||||
DaemonState::Starting,
|
DaemonState::Starting,
|
||||||
runs_completed,
|
runs_completed,
|
||||||
None,
|
None,
|
||||||
last_run_id.clone(),
|
last_run_id.clone(),
|
||||||
|
&health,
|
||||||
)?;
|
)?;
|
||||||
|
// Probe once at startup so revived repos are removed before the first run.
|
||||||
|
maybe_run_dead_repo_health_check(args, &mut health);
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
if args.max_runs.is_some_and(|max| runs_completed >= max) {
|
if args.max_runs.is_some_and(|max| runs_completed >= max) {
|
||||||
@ -1069,6 +1334,7 @@ fn run_daemon(args: &Args) -> Result<(), String> {
|
|||||||
runs_completed,
|
runs_completed,
|
||||||
None,
|
None,
|
||||||
last_run_id.clone(),
|
last_run_id.clone(),
|
||||||
|
&health,
|
||||||
)?;
|
)?;
|
||||||
let ctx = make_run_context(args, next_seq, utc_now());
|
let ctx = make_run_context(args, next_seq, utc_now());
|
||||||
write_status(
|
write_status(
|
||||||
@ -1077,6 +1343,7 @@ fn run_daemon(args: &Args) -> Result<(), String> {
|
|||||||
runs_completed,
|
runs_completed,
|
||||||
Some(&ctx),
|
Some(&ctx),
|
||||||
last_run_id.clone(),
|
last_run_id.clone(),
|
||||||
|
&health,
|
||||||
)?;
|
)?;
|
||||||
let mut summary = run_child_once(args, &ctx)?;
|
let mut summary = run_child_once(args, &ctx)?;
|
||||||
write_status(
|
write_status(
|
||||||
@ -1085,6 +1352,7 @@ fn run_daemon(args: &Args) -> Result<(), String> {
|
|||||||
runs_completed,
|
runs_completed,
|
||||||
Some(&ctx),
|
Some(&ctx),
|
||||||
last_run_id.clone(),
|
last_run_id.clone(),
|
||||||
|
&health,
|
||||||
)?;
|
)?;
|
||||||
collect_post_run_metrics(args, &ctx, &mut summary);
|
collect_post_run_metrics(args, &ctx, &mut summary);
|
||||||
let removed = apply_retention(&args.state_root.join("runs"), args.retain_runs)?;
|
let removed = apply_retention(&args.state_root.join("runs"), args.retain_runs)?;
|
||||||
@ -1099,19 +1367,29 @@ fn run_daemon(args: &Args) -> Result<(), String> {
|
|||||||
if args.max_runs.is_some_and(|max| runs_completed >= max) {
|
if args.max_runs.is_some_and(|max| runs_completed >= max) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
// Health probes run only while no child is active (single-writer).
|
||||||
|
maybe_run_dead_repo_health_check(args, &mut health);
|
||||||
write_status(
|
write_status(
|
||||||
args,
|
args,
|
||||||
DaemonState::Sleeping,
|
DaemonState::Sleeping,
|
||||||
runs_completed,
|
runs_completed,
|
||||||
None,
|
None,
|
||||||
last_run_id.clone(),
|
last_run_id.clone(),
|
||||||
|
&health,
|
||||||
)?;
|
)?;
|
||||||
if args.interval_secs > 0 {
|
if args.interval_secs > 0 {
|
||||||
std::thread::sleep(Duration::from_secs(args.interval_secs));
|
std::thread::sleep(Duration::from_secs(args.interval_secs));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
write_status(args, DaemonState::Exited, runs_completed, None, last_run_id)
|
write_status(
|
||||||
|
args,
|
||||||
|
DaemonState::Exited,
|
||||||
|
runs_completed,
|
||||||
|
None,
|
||||||
|
last_run_id,
|
||||||
|
&health,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn main_entry() -> i32 {
|
pub fn main_entry() -> i32 {
|
||||||
@ -1154,6 +1432,10 @@ mod tests {
|
|||||||
db_stats_bin: None,
|
db_stats_bin: None,
|
||||||
db_stats_exact_every: None,
|
db_stats_exact_every: None,
|
||||||
time_bin: None,
|
time_bin: None,
|
||||||
|
dead_repo_blacklist: None,
|
||||||
|
dead_repo_blacklist_fail_threshold: None,
|
||||||
|
dead_repo_health_check_interval_secs: 600,
|
||||||
|
dead_repo_probe_rsync_bin: PathBuf::from("rsync"),
|
||||||
child_args: vec!["--version".to_string()],
|
child_args: vec!["--version".to_string()],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1330,6 +1612,10 @@ mod tests {
|
|||||||
db_stats_bin: None,
|
db_stats_bin: None,
|
||||||
db_stats_exact_every: None,
|
db_stats_exact_every: None,
|
||||||
time_bin: None,
|
time_bin: None,
|
||||||
|
dead_repo_blacklist: None,
|
||||||
|
dead_repo_blacklist_fail_threshold: None,
|
||||||
|
dead_repo_health_check_interval_secs: 600,
|
||||||
|
dead_repo_probe_rsync_bin: PathBuf::from("rsync"),
|
||||||
child_args: vec![
|
child_args: vec![
|
||||||
"{state_root}/state/work-db".to_string(),
|
"{state_root}/state/work-db".to_string(),
|
||||||
"{run_out}/result.ccr".to_string(),
|
"{run_out}/result.ccr".to_string(),
|
||||||
@ -1652,6 +1938,10 @@ mod tests {
|
|||||||
db_stats_bin: None,
|
db_stats_bin: None,
|
||||||
db_stats_exact_every: None,
|
db_stats_exact_every: None,
|
||||||
time_bin: None,
|
time_bin: None,
|
||||||
|
dead_repo_blacklist: None,
|
||||||
|
dead_repo_blacklist_fail_threshold: None,
|
||||||
|
dead_repo_health_check_interval_secs: 600,
|
||||||
|
dead_repo_probe_rsync_bin: PathBuf::from("rsync"),
|
||||||
child_args: vec![
|
child_args: vec![
|
||||||
"-c".to_string(),
|
"-c".to_string(),
|
||||||
"echo stdout-{run_seq}; echo stderr-{run_seq} >&2; echo marker > {run_out}/marker.txt".to_string(),
|
"echo stdout-{run_seq}; echo stderr-{run_seq} >&2; echo marker > {run_out}/marker.txt".to_string(),
|
||||||
@ -1743,6 +2033,10 @@ mod tests {
|
|||||||
db_stats_bin: Some(db_stats_bin),
|
db_stats_bin: Some(db_stats_bin),
|
||||||
db_stats_exact_every: Some(1),
|
db_stats_exact_every: Some(1),
|
||||||
time_bin: None,
|
time_bin: None,
|
||||||
|
dead_repo_blacklist: None,
|
||||||
|
dead_repo_blacklist_fail_threshold: None,
|
||||||
|
dead_repo_health_check_interval_secs: 600,
|
||||||
|
dead_repo_probe_rsync_bin: PathBuf::from("rsync"),
|
||||||
child_args: vec![
|
child_args: vec![
|
||||||
"-c".to_string(),
|
"-c".to_string(),
|
||||||
"mkdir -p {state_root}/state/work-db {state_root}/state/repo-bytes.db; \
|
"mkdir -p {state_root}/state/work-db {state_root}/state/repo-bytes.db; \
|
||||||
@ -1781,4 +2075,130 @@ mod tests {
|
|||||||
.any(|item| item["label"] == "work_db" && item["exists"] == true)
|
.any(|item| item["label"] == "work_db" && item["exists"] == true)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dead_repo_blacklist_flag_is_injected_into_child_args() {
|
||||||
|
let argv = vec![
|
||||||
|
"rpki_daemon".to_string(),
|
||||||
|
"--state-root".to_string(),
|
||||||
|
"/tmp/daemon".to_string(),
|
||||||
|
"--rpki-bin".to_string(),
|
||||||
|
"/bin/echo".to_string(),
|
||||||
|
"--dead-repo-blacklist".to_string(),
|
||||||
|
"/tmp/daemon/state/dead-repo-blacklist.json".to_string(),
|
||||||
|
"--dead-repo-blacklist-fail-threshold".to_string(),
|
||||||
|
"2".to_string(),
|
||||||
|
"--".to_string(),
|
||||||
|
"--db".to_string(),
|
||||||
|
"{state_root}/state/work-db".to_string(),
|
||||||
|
];
|
||||||
|
let args = parse_args(&argv).expect("parse");
|
||||||
|
assert_eq!(
|
||||||
|
args.child_args,
|
||||||
|
vec![
|
||||||
|
"--db".to_string(),
|
||||||
|
"{state_root}/state/work-db".to_string(),
|
||||||
|
"--dead-repo-blacklist".to_string(),
|
||||||
|
"/tmp/daemon/state/dead-repo-blacklist.json".to_string(),
|
||||||
|
"--dead-repo-blacklist-fail-threshold".to_string(),
|
||||||
|
"2".to_string(),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dead_repo_blacklist_injection_respects_existing_child_flags() {
|
||||||
|
let argv = vec![
|
||||||
|
"rpki_daemon".to_string(),
|
||||||
|
"--state-root".to_string(),
|
||||||
|
"/tmp/daemon".to_string(),
|
||||||
|
"--rpki-bin".to_string(),
|
||||||
|
"/bin/echo".to_string(),
|
||||||
|
"--dead-repo-blacklist".to_string(),
|
||||||
|
"/tmp/daemon/state/dead-repo-blacklist.json".to_string(),
|
||||||
|
"--".to_string(),
|
||||||
|
"--dead-repo-blacklist".to_string(),
|
||||||
|
"/custom/path.json".to_string(),
|
||||||
|
];
|
||||||
|
let args = parse_args(&argv).expect("parse");
|
||||||
|
assert_eq!(
|
||||||
|
args.child_args,
|
||||||
|
vec![
|
||||||
|
"--dead-repo-blacklist".to_string(),
|
||||||
|
"/custom/path.json".to_string(),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dead_repo_threshold_without_blacklist_is_rejected() {
|
||||||
|
let argv = vec![
|
||||||
|
"rpki_daemon".to_string(),
|
||||||
|
"--state-root".to_string(),
|
||||||
|
"/tmp/daemon".to_string(),
|
||||||
|
"--rpki-bin".to_string(),
|
||||||
|
"/bin/echo".to_string(),
|
||||||
|
"--dead-repo-blacklist-fail-threshold".to_string(),
|
||||||
|
"2".to_string(),
|
||||||
|
"--".to_string(),
|
||||||
|
"--version".to_string(),
|
||||||
|
];
|
||||||
|
let err = parse_args(&argv).expect_err("parse should fail");
|
||||||
|
assert!(err.contains("requires --dead-repo-blacklist"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn health_check_keeps_dead_entry_and_removes_revived_entry() {
|
||||||
|
use std::io::{Read as _, Write as _};
|
||||||
|
use std::net::TcpListener;
|
||||||
|
|
||||||
|
// Revived RRDP host: one-shot HTTP server answering 200.
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").expect("bind");
|
||||||
|
let port = listener.local_addr().expect("local addr").port();
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
if let Ok((mut stream, _)) = listener.accept() {
|
||||||
|
let mut buf = [0u8; 1024];
|
||||||
|
let _ = stream.read(&mut buf);
|
||||||
|
let body = b"<notification/>";
|
||||||
|
let _ = stream.write_all(
|
||||||
|
format!(
|
||||||
|
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||||
|
body.len()
|
||||||
|
)
|
||||||
|
.as_bytes(),
|
||||||
|
);
|
||||||
|
let _ = stream.write_all(body);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let td = tempfile::tempdir().expect("tempdir");
|
||||||
|
let blacklist_path = td.path().join("dead-repo-blacklist.json");
|
||||||
|
let revived_uri = format!("http://127.0.0.1:{port}/notification.xml");
|
||||||
|
let dead_uri = "http://127.0.0.1:1/notification.xml".to_string();
|
||||||
|
let mut blacklist = DeadRepoBlacklist::new();
|
||||||
|
for uri in [&revived_uri, &dead_uri] {
|
||||||
|
blacklist.record_transport_failure(RepoTransportMode::Rrdp, uri, 100, 1, 256);
|
||||||
|
}
|
||||||
|
blacklist
|
||||||
|
.store_atomic(&blacklist_path, 100)
|
||||||
|
.expect("store blacklist");
|
||||||
|
|
||||||
|
let mut args = test_args(td.path().join("daemon"));
|
||||||
|
args.dead_repo_blacklist = Some(blacklist_path.clone());
|
||||||
|
args.dead_repo_health_check_interval_secs = 0;
|
||||||
|
let mut health = DeadRepoHealthRuntime::default();
|
||||||
|
maybe_run_dead_repo_health_check(&args, &mut health);
|
||||||
|
|
||||||
|
let (after, warning) = DeadRepoBlacklist::load(&blacklist_path);
|
||||||
|
assert!(warning.is_none());
|
||||||
|
assert!(!after.is_blacklisted(RepoTransportMode::Rrdp, &revived_uri));
|
||||||
|
assert!(after.is_blacklisted(RepoTransportMode::Rrdp, &dead_uri));
|
||||||
|
assert!(health.last_health_check_at.is_some());
|
||||||
|
|
||||||
|
// Status JSON exposes the blacklist section.
|
||||||
|
let status = dead_repo_blacklist_status(&args, &health).expect("status");
|
||||||
|
assert_eq!(status.entries, 1);
|
||||||
|
assert_eq!(status.blacklisted, 1);
|
||||||
|
assert!(status.last_health_check_at_rfc3339_utc.is_some());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -345,6 +345,56 @@ fn persist_transport_request_prefetch_snapshot(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Persist the dead-repo blacklist working copy at run end (#141). Failures
|
||||||
|
/// degrade to a progress event; a broken blacklist file must never fail a run.
|
||||||
|
fn persist_dead_repo_blacklist(
|
||||||
|
runtime: &Arc<dyn RepoSyncRuntime>,
|
||||||
|
timing: Option<&TimingHandle>,
|
||||||
|
) -> Result<(), RunTreeFromTalError> {
|
||||||
|
let Some((blacklist_config, blacklist)) = runtime.dead_repo_blacklist_state() else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
let persist_started = std::time::Instant::now();
|
||||||
|
let now_unix = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|duration| duration.as_secs())
|
||||||
|
.unwrap_or(0);
|
||||||
|
match blacklist.store_atomic(&blacklist_config.path, now_unix) {
|
||||||
|
Ok(()) => {
|
||||||
|
let persist_ms = persist_started.elapsed().as_millis() as u64;
|
||||||
|
record_transport_prefetch_count(
|
||||||
|
timing,
|
||||||
|
"dead_repo_blacklist_entries",
|
||||||
|
blacklist.len() as u64,
|
||||||
|
);
|
||||||
|
record_transport_prefetch_count(
|
||||||
|
timing,
|
||||||
|
"dead_repo_blacklist_blacklisted",
|
||||||
|
blacklist.blacklisted_len() as u64,
|
||||||
|
);
|
||||||
|
crate::progress_log::emit(
|
||||||
|
"dead_repo_blacklist_persisted",
|
||||||
|
serde_json::json!({
|
||||||
|
"path": blacklist_config.path.display().to_string(),
|
||||||
|
"entries": blacklist.len(),
|
||||||
|
"blacklisted": blacklist.blacklisted_len(),
|
||||||
|
"persist_ms": persist_ms,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
crate::progress_log::emit(
|
||||||
|
"dead_repo_blacklist_persist_error",
|
||||||
|
serde_json::json!({
|
||||||
|
"path": blacklist_config.path.display().to_string(),
|
||||||
|
"error": err.to_string(),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn root_discovery_from_tal_input(
|
fn root_discovery_from_tal_input(
|
||||||
tal_input: &TalInputSpec,
|
tal_input: &TalInputSpec,
|
||||||
http_fetcher: &dyn Fetcher,
|
http_fetcher: &dyn Fetcher,
|
||||||
@ -813,6 +863,7 @@ where
|
|||||||
run_tree_serial_audit(root, &runner, config)?
|
run_tree_serial_audit(root, &runner, config)?
|
||||||
};
|
};
|
||||||
persist_transport_request_prefetch_snapshot(store.as_ref(), &runtime, config, timing.as_ref())?;
|
persist_transport_request_prefetch_snapshot(store.as_ref(), &runtime, config, timing.as_ref())?;
|
||||||
|
persist_dead_repo_blacklist(&runtime, timing.as_ref())?;
|
||||||
let downloads = download_log.snapshot_events();
|
let downloads = download_log.snapshot_events();
|
||||||
let download_stats = DownloadLogHandle::stats_from_events(&downloads);
|
let download_stats = DownloadLogHandle::stats_from_events(&downloads);
|
||||||
Ok(RunTreeFromTalAuditOutput {
|
Ok(RunTreeFromTalAuditOutput {
|
||||||
@ -936,6 +987,7 @@ where
|
|||||||
run_tree_serial_audit_multi_root(root_handles, &runner, config)?
|
run_tree_serial_audit_multi_root(root_handles, &runner, config)?
|
||||||
};
|
};
|
||||||
persist_transport_request_prefetch_snapshot(store.as_ref(), &runtime, config, timing.as_ref())?;
|
persist_transport_request_prefetch_snapshot(store.as_ref(), &runtime, config, timing.as_ref())?;
|
||||||
|
persist_dead_repo_blacklist(&runtime, timing.as_ref())?;
|
||||||
let downloads = download_log.snapshot_events();
|
let downloads = download_log.snapshot_events();
|
||||||
let download_stats = DownloadLogHandle::stats_from_events(&downloads);
|
let download_stats = DownloadLogHandle::stats_from_events(&downloads);
|
||||||
Ok(RunTreeFromTalAuditOutput {
|
Ok(RunTreeFromTalAuditOutput {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user