405 lines
14 KiB
Rust
405 lines
14 KiB
Rust
// Retention, report parsing, and post-run metric collection.
|
|
|
|
fn apply_retention(runs_root: &Path, retain_runs: usize) -> Result<Vec<PathBuf>, String> {
|
|
if !runs_root.exists() {
|
|
return Ok(Vec::new());
|
|
}
|
|
let mut dirs = Vec::new();
|
|
for entry in fs::read_dir(runs_root)
|
|
.map_err(|e| format!("read runs dir failed: {}: {e}", runs_root.display()))?
|
|
{
|
|
let entry = entry.map_err(|e| format!("read runs dir entry failed: {e}"))?;
|
|
if entry
|
|
.file_type()
|
|
.map_err(|e| format!("read file type failed: {}: {e}", entry.path().display()))?
|
|
.is_dir()
|
|
{
|
|
dirs.push(entry.path());
|
|
}
|
|
}
|
|
dirs.sort();
|
|
let remove_count = dirs.len().saturating_sub(retain_runs);
|
|
let mut removed = Vec::new();
|
|
for dir in dirs.into_iter().take(remove_count) {
|
|
fs::remove_dir_all(&dir)
|
|
.map_err(|e| format!("remove old run dir failed: {}: {e}", dir.display()))?;
|
|
removed.push(dir);
|
|
}
|
|
Ok(removed)
|
|
}
|
|
|
|
fn find_named_file(root: &Path, name: &str) -> Option<PathBuf> {
|
|
let mut stack = vec![root.to_path_buf()];
|
|
while let Some(dir) = stack.pop() {
|
|
let entries = fs::read_dir(&dir).ok()?;
|
|
for entry in entries.flatten() {
|
|
let path = entry.path();
|
|
let file_type = entry.file_type().ok()?;
|
|
if file_type.is_file() && entry.file_name().to_string_lossy() == name {
|
|
return Some(path);
|
|
}
|
|
if file_type.is_dir() {
|
|
stack.push(path);
|
|
}
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
fn read_json_value_if_exists(path: &Path) -> Option<serde_json::Value> {
|
|
let bytes = fs::read(path).ok()?;
|
|
serde_json::from_slice(&bytes).ok()
|
|
}
|
|
|
|
fn json_array_len(value: &serde_json::Value, key: &str) -> usize {
|
|
value
|
|
.get(key)
|
|
.and_then(serde_json::Value::as_array)
|
|
.map(Vec::len)
|
|
.unwrap_or(0)
|
|
}
|
|
|
|
fn parse_stdout_summary(run_dir: &Path) -> Option<ReportCounts> {
|
|
let stdout_path = run_dir.join("stdout.log");
|
|
let text = fs::read_to_string(stdout_path).ok()?;
|
|
let mut vrps = None;
|
|
let mut aspas = None;
|
|
let mut publication_points = None;
|
|
let mut rrdp_repos_unique = None;
|
|
let mut tree_instances_processed = None;
|
|
let mut tree_instances_failed = None;
|
|
let mut warnings = None;
|
|
|
|
for line in text.lines() {
|
|
if let Some(value) = line.strip_prefix("vrps=") {
|
|
vrps = value.trim().parse::<usize>().ok();
|
|
} else if let Some(value) = line.strip_prefix("aspas=") {
|
|
aspas = value.trim().parse::<usize>().ok();
|
|
} else if let Some(value) = line.strip_prefix("audit_publication_points=") {
|
|
publication_points = value.trim().parse::<usize>().ok();
|
|
} else if let Some(value) = line.strip_prefix("rrdp_repos_unique=") {
|
|
rrdp_repos_unique = value.trim().parse::<u64>().ok();
|
|
} else if let Some(value) = line.strip_prefix("warnings_total=") {
|
|
warnings = value.trim().parse::<usize>().ok();
|
|
} else if let Some(rest) = line.strip_prefix("publication_points_processed=") {
|
|
for token in rest.split_whitespace() {
|
|
if let Some(value) = token.strip_prefix("publication_points_failed=") {
|
|
tree_instances_failed = value.parse::<u64>().ok();
|
|
} else if tree_instances_processed.is_none() {
|
|
tree_instances_processed = token.parse::<u64>().ok();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Some(ReportCounts {
|
|
vrps: vrps?,
|
|
aspas: aspas?,
|
|
publication_points: publication_points?,
|
|
rrdp_repos_unique,
|
|
tree_instances_processed,
|
|
tree_instances_failed,
|
|
warnings: warnings.unwrap_or(0),
|
|
})
|
|
}
|
|
|
|
fn parse_report_counts_fallback(report: &serde_json::Value) -> ReportCounts {
|
|
let tree = report.get("tree");
|
|
let tree_warnings = tree
|
|
.and_then(|tree| tree.get("warnings"))
|
|
.and_then(serde_json::Value::as_array)
|
|
.map(Vec::len)
|
|
.unwrap_or(0);
|
|
let pp_warnings = report
|
|
.get("publication_points")
|
|
.and_then(serde_json::Value::as_array)
|
|
.map(|items| {
|
|
items
|
|
.iter()
|
|
.map(|pp| {
|
|
pp.get("warnings")
|
|
.and_then(serde_json::Value::as_array)
|
|
.map(Vec::len)
|
|
.unwrap_or(0)
|
|
})
|
|
.sum()
|
|
})
|
|
.unwrap_or(0);
|
|
ReportCounts {
|
|
vrps: json_array_len(report, "vrps"),
|
|
aspas: json_array_len(report, "aspas"),
|
|
publication_points: json_array_len(report, "publication_points"),
|
|
rrdp_repos_unique: None,
|
|
tree_instances_processed: tree
|
|
.and_then(|tree| tree.get("instances_processed"))
|
|
.and_then(serde_json::Value::as_u64),
|
|
tree_instances_failed: tree
|
|
.and_then(|tree| tree.get("instances_failed"))
|
|
.and_then(serde_json::Value::as_u64),
|
|
warnings: tree_warnings + pp_warnings,
|
|
}
|
|
}
|
|
|
|
fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
|
haystack
|
|
.windows(needle.len())
|
|
.position(|window| window == needle)
|
|
}
|
|
|
|
fn extract_json_object_field(path: &Path, field_name: &str) -> Option<serde_json::Value> {
|
|
let bytes = fs::read(path).ok()?;
|
|
let needle = format!("\"{field_name}\":");
|
|
let pos = find_subslice(&bytes, needle.as_bytes())?;
|
|
let mut i = pos + needle.len();
|
|
while i < bytes.len() && bytes[i].is_ascii_whitespace() {
|
|
i += 1;
|
|
}
|
|
if bytes.get(i).copied()? != b'{' {
|
|
return None;
|
|
}
|
|
let start = i;
|
|
let mut depth = 0u32;
|
|
let mut in_string = false;
|
|
let mut escaped = false;
|
|
for (offset, &b) in bytes[start..].iter().enumerate() {
|
|
if in_string {
|
|
if escaped {
|
|
escaped = false;
|
|
} else if b == b'\\' {
|
|
escaped = true;
|
|
} else if b == b'"' {
|
|
in_string = false;
|
|
}
|
|
continue;
|
|
}
|
|
match b {
|
|
b'"' => in_string = true,
|
|
b'{' => depth = depth.saturating_add(1),
|
|
b'}' => {
|
|
depth = depth.saturating_sub(1);
|
|
if depth == 0 {
|
|
let end = start + offset + 1;
|
|
return serde_json::from_slice(&bytes[start..end]).ok();
|
|
}
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
fn parse_report_metadata(run_dir: &Path) -> (Option<ReportCounts>, Option<serde_json::Value>) {
|
|
let Some(report_path) = find_named_file(run_dir, "report.json") else {
|
|
return (parse_stdout_summary(run_dir), None);
|
|
};
|
|
let counts = parse_stdout_summary(run_dir).or_else(|| {
|
|
read_json_value_if_exists(&report_path).map(|report| parse_report_counts_fallback(&report))
|
|
});
|
|
let repo_sync_stats = extract_json_object_field(&report_path, "repo_sync_stats");
|
|
(counts, repo_sync_stats)
|
|
}
|
|
|
|
fn collect_path_file_stats(label: &str, path: &Path) -> PathFileStats {
|
|
let mut stats = PathFileStats {
|
|
label: label.to_string(),
|
|
path: path_string(path),
|
|
exists: path.exists(),
|
|
is_dir: path.is_dir(),
|
|
total_size_bytes: 0,
|
|
file_count: 0,
|
|
dir_count: 0,
|
|
};
|
|
if !stats.exists {
|
|
return stats;
|
|
}
|
|
if path.is_file() {
|
|
if let Ok(metadata) = path.metadata() {
|
|
stats.total_size_bytes = metadata.len();
|
|
stats.file_count = 1;
|
|
}
|
|
return stats;
|
|
}
|
|
|
|
let mut stack = vec![path.to_path_buf()];
|
|
while let Some(dir) = stack.pop() {
|
|
let Ok(entries) = fs::read_dir(&dir) else {
|
|
continue;
|
|
};
|
|
for entry in entries.flatten() {
|
|
let Ok(file_type) = entry.file_type() else {
|
|
continue;
|
|
};
|
|
if file_type.is_dir() {
|
|
stats.dir_count = stats.dir_count.saturating_add(1);
|
|
stack.push(entry.path());
|
|
} else if file_type.is_file() {
|
|
stats.file_count = stats.file_count.saturating_add(1);
|
|
if let Ok(metadata) = entry.metadata() {
|
|
stats.total_size_bytes = stats.total_size_bytes.saturating_add(metadata.len());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
stats
|
|
}
|
|
|
|
fn collect_state_path_stats(args: &Args, ctx: &RunContext) -> Vec<PathFileStats> {
|
|
let mut stats = Vec::new();
|
|
stats.push(collect_path_file_stats(
|
|
"work_db",
|
|
&render_path_template(&args.work_db, args, ctx),
|
|
));
|
|
if let Some(path) = args.repo_bytes_db.as_ref() {
|
|
stats.push(collect_path_file_stats(
|
|
"repo_bytes_db",
|
|
&render_path_template(path, args, ctx),
|
|
));
|
|
}
|
|
if let Some(path) = args.raw_store_db.as_ref() {
|
|
stats.push(collect_path_file_stats(
|
|
"raw_store_db",
|
|
&render_path_template(path, args, ctx),
|
|
));
|
|
}
|
|
stats
|
|
}
|
|
|
|
fn parse_key_value_metrics(text: &str) -> BTreeMap<String, String> {
|
|
let mut metrics = BTreeMap::new();
|
|
for line in text.lines() {
|
|
let Some((key, value)) = line.split_once('=') else {
|
|
continue;
|
|
};
|
|
metrics.insert(key.trim().to_string(), value.trim().to_string());
|
|
}
|
|
metrics
|
|
}
|
|
|
|
fn run_db_stats_command(
|
|
db_stats_bin: &Path,
|
|
db_path: &Path,
|
|
run_dir: &Path,
|
|
mode: &str,
|
|
) -> DbStatsSummary {
|
|
let output_path = run_dir.join(format!("db-stats-{mode}.txt"));
|
|
let stderr_path = run_dir.join(format!("db-stats-{mode}.stderr.txt"));
|
|
let mut summary = DbStatsSummary {
|
|
mode: mode.to_string(),
|
|
db_path: path_string(db_path),
|
|
output_path: Some(path_string(&output_path)),
|
|
stderr_path: None,
|
|
status: "success".to_string(),
|
|
exit_code: None,
|
|
error: None,
|
|
metrics: BTreeMap::new(),
|
|
};
|
|
|
|
if !db_path.exists() {
|
|
summary.status = "skipped".to_string();
|
|
summary.error = Some(format!("db path does not exist: {}", db_path.display()));
|
|
summary.output_path = None;
|
|
return summary;
|
|
}
|
|
|
|
let mut command = Command::new(db_stats_bin);
|
|
command.arg("--db").arg(db_path);
|
|
if mode == "exact" {
|
|
command.arg("--exact");
|
|
}
|
|
match command.output() {
|
|
Ok(output) => {
|
|
summary.exit_code = output.status.code();
|
|
if !output.status.success() {
|
|
summary.status = "failed".to_string();
|
|
}
|
|
let stdout_text = String::from_utf8_lossy(&output.stdout).into_owned();
|
|
if let Err(err) = fs::write(&output_path, stdout_text.as_bytes()) {
|
|
summary.status = "failed".to_string();
|
|
summary.error = Some(format!(
|
|
"write db_stats output failed: {}: {err}",
|
|
output_path.display()
|
|
));
|
|
}
|
|
summary.metrics = parse_key_value_metrics(&stdout_text);
|
|
if !output.stderr.is_empty() {
|
|
if fs::write(&stderr_path, &output.stderr).is_ok() {
|
|
summary.stderr_path = Some(path_string(&stderr_path));
|
|
}
|
|
}
|
|
if !output.status.success() && summary.error.is_none() {
|
|
summary.error = Some(String::from_utf8_lossy(&output.stderr).into_owned());
|
|
}
|
|
}
|
|
Err(err) => {
|
|
summary.status = "spawn_failed".to_string();
|
|
summary.exit_code = None;
|
|
summary.output_path = None;
|
|
summary.error = Some(format!("spawn db_stats failed: {err}"));
|
|
}
|
|
}
|
|
summary
|
|
}
|
|
|
|
fn collect_db_stats(args: &Args, ctx: &RunContext) -> Vec<DbStatsSummary> {
|
|
let work_db = render_path_template(&args.work_db, args, ctx);
|
|
let Some(db_stats_bin) = args
|
|
.db_stats_bin
|
|
.as_ref()
|
|
.cloned()
|
|
.or_else(default_db_stats_bin)
|
|
else {
|
|
return vec![DbStatsSummary {
|
|
mode: "estimate".to_string(),
|
|
db_path: path_string(&work_db),
|
|
output_path: None,
|
|
stderr_path: None,
|
|
status: "skipped".to_string(),
|
|
exit_code: None,
|
|
error: Some(
|
|
"db_stats binary not configured and sibling db_stats was not found".to_string(),
|
|
),
|
|
metrics: BTreeMap::new(),
|
|
}];
|
|
};
|
|
|
|
let mut stats = Vec::new();
|
|
stats.push(run_db_stats_command(
|
|
&db_stats_bin,
|
|
&work_db,
|
|
&ctx.run_dir,
|
|
"estimate",
|
|
));
|
|
if args
|
|
.db_stats_exact_every
|
|
.is_some_and(|every| ctx.seq % every == 0)
|
|
{
|
|
stats.push(run_db_stats_command(
|
|
&db_stats_bin,
|
|
&work_db,
|
|
&ctx.run_dir,
|
|
"exact",
|
|
));
|
|
}
|
|
stats
|
|
}
|
|
|
|
fn collect_post_run_metrics(args: &Args, ctx: &RunContext, summary: &mut RunSummary) {
|
|
if let Some(path) = find_named_file(&ctx.run_dir, "stage-timing.json") {
|
|
summary.stage_timing = read_json_value_if_exists(&path);
|
|
}
|
|
let (report_counts, repo_sync_stats) = parse_report_metadata(&ctx.run_dir);
|
|
summary.report_counts = report_counts;
|
|
summary.repo_sync_stats = repo_sync_stats;
|
|
summary.path_stats = collect_state_path_stats(args, ctx);
|
|
summary.db_stats = collect_db_stats(args, ctx);
|
|
summary.artifacts = collect_artifacts(&ctx.run_dir).unwrap_or_default();
|
|
if let Some(path) = args.dead_repo_blacklist.as_ref() {
|
|
let (blacklist, _) = DeadRepoBlacklist::load(path);
|
|
summary.dead_repo_blacklist = Some(DeadRepoBlacklistRunSummary {
|
|
path: path_string(path),
|
|
entries: blacklist.len(),
|
|
blacklisted: blacklist.blacklisted_len(),
|
|
});
|
|
}
|
|
}
|