NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260609 增加三RP持续对比监控

Browse Source
This commit is contained in:
yuyr
2026-06-09 17:44:30 +08:00
parent d0224d53f6
commit 74012c686d
22 changed files with 3911 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/bin/rpki_inter_rp_metrics.rs Normal file
View File

@@ -0,0 +1,6 @@
fn main() {
if let Err(err) = rpki::tools::rpki_inter_rp_metrics::main_entry() {
eprintln!("{err}");
std::process::exit(1);
}
}

1
src/tools/mod.rs
View File

@@ -1,3 +1,4 @@
pub mod rpki_artifact_metrics;
pub mod rpki_daemon;
pub mod rpki_inter_rp_metrics;
pub mod sequence_triage_ccr_cir;

157
src/tools/rpki_artifact_metrics.rs
View File

@@ -60,19 +60,21 @@ fn real_main() -> Result<(), String> {
thread::spawn(move || {
loop {
thread::sleep(Duration::from_secs(poll_secs));
let next = match scan_run_root(&run_root, &instance) {
Ok(snapshot) => snapshot,
Err(err) => {
let mut previous = scanner.write().expect("metrics lock poisoned");
previous
.service
.parse_errors
.push(format!("scan failed: {err}"));
previous.service.last_reload_success = false;
previous.service.last_scan_timestamp_seconds = unix_now_seconds();
continue;
}
};
let previous_snapshot = scanner.read().expect("metrics lock poisoned").clone();
let next =
match scan_run_root_incremental(&run_root, &instance, Some(&previous_snapshot)) {
Ok(snapshot) => snapshot,
Err(err) => {
let mut previous = scanner.write().expect("metrics lock poisoned");
previous
.service
.parse_errors
.push(format!("scan failed: {err}"));
previous.service.last_reload_success = false;
previous.service.last_scan_timestamp_seconds = unix_now_seconds();
continue;
}
};
*scanner.write().expect("metrics lock poisoned") = next;
}
});
@@ -211,6 +213,7 @@ struct LatestRunMetrics {
max_rss_bytes: Option<u64>,
exit_code: Option<i64>,
vrps: u64,
vrps_unique: Option<u64>,
vaps: u64,
publication_points: u64,
warnings: u64,
@@ -374,6 +377,14 @@ struct RunRecord {
}
fn scan_run_root(input_root: &Path, instance: &str) -> Result<MetricsSnapshot, String> {
scan_run_root_incremental(input_root, instance, None)
}
fn scan_run_root_incremental(
input_root: &Path,
instance: &str,
previous: Option<&MetricsSnapshot>,
) -> Result<MetricsSnapshot, String> {
let started = Instant::now();
let runs_root = resolve_runs_root(input_root);
let mut snapshot = MetricsSnapshot {
@@ -418,7 +429,12 @@ fn scan_run_root(input_root: &Path, instance: &str) -> Result<MetricsSnapshot, S
.rev()
.find(|record| record.status == "success")
{
build_latest_metrics(latest, &mut snapshot);
if can_reuse_latest_metrics(latest, previous) {
reuse_latest_artifact_metrics(previous.expect("checked previous"), &mut snapshot);
} else {
build_latest_metrics(latest, &mut snapshot);
let _ = crate::memory_telemetry::malloc_trim_probe();
}
}
snapshot.service.last_scan_timestamp_seconds = unix_now_seconds();
@@ -427,6 +443,42 @@ fn scan_run_root(input_root: &Path, instance: &str) -> Result<MetricsSnapshot, S
Ok(snapshot)
}
fn can_reuse_latest_metrics(record: &RunRecord, previous: Option<&MetricsSnapshot>) -> bool {
let Some(previous) = previous else {
return false;
};
let Some(previous_latest) = previous.latest_run.as_ref() else {
return false;
};
if previous_latest.run_dir != record.path.display().to_string() {
return false;
}
if previous_latest.status != record.status {
return false;
}
let summary = record.summary.as_ref();
let meta = record.meta.as_ref();
let finished_at = summary
.and_then(|v| json_str(v, &["finishedAtRfc3339Utc"]))
.or_else(|| meta.and_then(|v| json_str(v, &["completed_at_rfc3339_utc"])));
let wall_seconds = summary.and_then(|v| json_u64(v, &["wallMs"])).unwrap_or(0) as f64 / 1000.0;
previous_latest.finished_at.as_deref() == finished_at
&& (previous_latest.wall_seconds - wall_seconds).abs() < f64::EPSILON
}
fn reuse_latest_artifact_metrics(previous: &MetricsSnapshot, snapshot: &mut MetricsSnapshot) {
snapshot.latest_run = previous.latest_run.clone();
snapshot.repo_stats = previous.repo_stats.clone();
snapshot.object_counts = previous.object_counts.clone();
snapshot.large_pp_counts = previous.large_pp_counts.clone();
snapshot.pp_sync_histograms = previous.pp_sync_histograms.clone();
snapshot.top_repos_by_sync_duration = previous.top_repos_by_sync_duration.clone();
snapshot.top_pp_by_object_count = previous.top_pp_by_object_count.clone();
snapshot.top_pp_by_sync_duration = previous.top_pp_by_sync_duration.clone();
snapshot.cir = previous.cir.clone();
snapshot.ccr = previous.ccr.clone();
}
fn resolve_runs_root(input_root: &Path) -> PathBuf {
let runs = input_root.join("runs");
if runs.is_dir() {
@@ -598,11 +650,65 @@ fn build_latest_metrics(record: &RunRecord, snapshot: &mut MetricsSnapshot) {
}
parse_report(&record.path.join("report.json"), snapshot, &mut latest);
latest.vrps_unique = count_vrp_csv_unique_keys_opt(
&record.path.join("vrps.csv"),
&mut snapshot.service.parse_errors,
);
parse_cir(&record.path.join("input.cir"), snapshot);
parse_ccr(&record.path.join("result.ccr"), snapshot);
snapshot.latest_run = Some(latest);
}
fn count_vrp_csv_unique_keys_opt(path: &Path, errors: &mut Vec<String>) -> Option<u64> {
if !path.exists() {
return None;
}
match count_vrp_csv_unique_keys(path) {
Ok(count) => Some(count),
Err(err) => {
errors.push(err);
None
}
}
}
fn count_vrp_csv_unique_keys(path: &Path) -> Result<u64, String> {
let content = fs::read_to_string(path)
.map_err(|e| format!("read VRP CSV failed: {}: {e}", path.display()))?;
let mut unique = BTreeSet::new();
for (index, line) in data_csv_lines(&content).enumerate() {
if index == 0 {
continue;
}
let columns = split_csv_simple(line);
if columns.len() < 3 {
return Err(format!(
"invalid VRP CSV row in {}: expected at least 3 columns, got {}",
path.display(),
columns.len()
));
}
unique.insert((
columns[0].to_string(),
columns[1].to_string(),
columns[2].to_string(),
));
}
Ok(unique.len() as u64)
}
fn data_csv_lines(content: &str) -> impl Iterator<Item = &str> {
content
.lines()
.map(str::trim)
.filter(|line| !line.is_empty())
.filter(|line| !line.starts_with('#'))
}
fn split_csv_simple(line: &str) -> Vec<&str> {
line.split(',').map(str::trim).collect()
}
fn parse_report(path: &Path, snapshot: &mut MetricsSnapshot, latest: &mut LatestRunMetrics) {
if !path.exists() {
return;
@@ -1164,6 +1270,14 @@ fn render_latest_metrics(writer: &mut PromWriter<'_>, instance: &str, latest: &L
&[label("instance", instance), label("kind", "total")],
latest.vrps as f64,
);
if let Some(value) = latest.vrps_unique {
writer.gauge(
"ours_rp_vrps",
"Latest run VRP count",
&[label("instance", instance), label("kind", "unique")],
value as f64,
);
}
writer.gauge(
"ours_rp_vaps",
"Latest run VAP/ASPA count",
@@ -2013,14 +2127,22 @@ mod tests {
.expect("meta");
fs::write(
run.join("run-summary.json"),
r#"{"runSeq":1,"runId":"run_0001","runDir":"RUN","startedAtRfc3339Utc":"2026-05-25T00:00:00Z","finishedAtRfc3339Utc":"2026-05-25T00:00:10Z","wallMs":10000,"status":"success","exitCode":0,"processMetrics":{"userSeconds":2.5,"systemSeconds":1.5,"cpuPercent":40,"maxRssKb":1000},"stageTiming":{"validation_ms":7000,"total_ms":9000,"download_event_count":2,"download_bytes_total":1234},"reportCounts":{"vrps":2,"aspas":1,"publicationPoints":2,"warnings":0},"repoSyncStats":{"by_phase":{"rrdp_delta":{"count":2,"duration_ms_total":3000}},"by_terminal_state":{"fresh":{"count":2,"duration_ms_total":3000}}},"pathStats":[{"label":"work-db","totalSizeBytes":99,"fileCount":2,"dirCount":1}],"artifacts":[{"path":"report.json","sizeBytes":10}]}"#,
r#"{"runSeq":1,"runId":"run_0001","runDir":"RUN","startedAtRfc3339Utc":"2026-05-25T00:00:00Z","finishedAtRfc3339Utc":"2026-05-25T00:00:10Z","wallMs":10000,"status":"success","exitCode":0,"processMetrics":{"userSeconds":2.5,"systemSeconds":1.5,"cpuPercent":40,"maxRssKb":1000},"stageTiming":{"validation_ms":7000,"total_ms":9000,"download_event_count":2,"download_bytes_total":1234},"reportCounts":{"vrps":3,"aspas":1,"publicationPoints":2,"warnings":0},"repoSyncStats":{"by_phase":{"rrdp_delta":{"count":2,"duration_ms_total":3000}},"by_terminal_state":{"fresh":{"count":2,"duration_ms_total":3000}}},"pathStats":[{"label":"work-db","totalSizeBytes":99,"fileCount":2,"dirCount":1}],"artifacts":[{"path":"report.json","sizeBytes":10}]}"#,
)
.expect("summary");
fs::write(run.join("process-time.txt"), "time").expect("time");
fs::write(run.join("stage-timing.json"), "{}").expect("stage");
fs::write(
run.join("vrps.csv"),
"ASN,IP Prefix,Max Length,Trust Anchor,Expires\n\
AS64496,192.0.2.0/24,24,ta,2026-05-25T01:00:00Z\n\
AS64496,192.0.2.0/24,24,ta,2026-05-25T01:00:00Z\n\
AS64497,2001:db8::/32,48,ta,2026-05-25T01:00:00Z\n",
)
.expect("vrps");
fs::write(
run.join("report.json"),
r#"{"tree":{"instances_processed":2,"instances_failed":0,"warnings":[]},"vrps":[{},{}],"aspas":[{}],"downloads":[{"kind":"rrdp_notification","uri":"https://repo.example/notify.xml","success":true,"duration_ms":100,"bytes":111},{"kind":"rrdp_delta","uri":"https://repo.example/session/1/delta.xml","success":true,"duration_ms":200,"bytes":222}],"publication_points":[{"rsync_base_uri":"rsync://repo.example/a/","manifest_rsync_uri":"rsync://repo.example/a/a.mft","publication_point_rsync_uri":"rsync://repo.example/a/","rrdp_notification_uri":"https://repo.example/notify.xml","repo_sync_source":"rrdp","repo_sync_phase":"rrdp_delta","repo_sync_duration_ms":1000,"repo_terminal_state":"fresh","objects":[{"kind":"roa","result":"ok"},{"kind":"manifest","result":"ok"}]},{"rsync_base_uri":"rsync://repo.example/b/","manifest_rsync_uri":"rsync://repo.example/b/b.mft","publication_point_rsync_uri":"rsync://repo.example/b/","rrdp_notification_uri":"https://repo.example/notify.xml","repo_sync_source":"rrdp","repo_sync_phase":"rrdp_delta","repo_sync_duration_ms":2000,"repo_terminal_state":"fresh","objects":[{"kind":"roa","result":"ok"}]}],"repo_sync_stats":{"publication_points_total":2,"by_phase":{"rrdp_delta":{"count":2,"duration_ms_total":3000}},"by_terminal_state":{"fresh":{"count":2,"duration_ms_total":3000}}}}"#,
r#"{"tree":{"instances_processed":2,"instances_failed":0,"warnings":[]},"vrps":[{},{},{}],"aspas":[{}],"downloads":[{"kind":"rrdp_notification","uri":"https://repo.example/notify.xml","success":true,"duration_ms":100,"bytes":111},{"kind":"rrdp_delta","uri":"https://repo.example/session/1/delta.xml","success":true,"duration_ms":200,"bytes":222}],"publication_points":[{"rsync_base_uri":"rsync://repo.example/a/","manifest_rsync_uri":"rsync://repo.example/a/a.mft","publication_point_rsync_uri":"rsync://repo.example/a/","rrdp_notification_uri":"https://repo.example/notify.xml","repo_sync_source":"rrdp","repo_sync_phase":"rrdp_delta","repo_sync_duration_ms":1000,"repo_terminal_state":"fresh","objects":[{"kind":"roa","result":"ok"},{"kind":"manifest","result":"ok"}]},{"rsync_base_uri":"rsync://repo.example/b/","manifest_rsync_uri":"rsync://repo.example/b/b.mft","publication_point_rsync_uri":"rsync://repo.example/b/","rrdp_notification_uri":"https://repo.example/notify.xml","repo_sync_source":"rrdp","repo_sync_phase":"rrdp_delta","repo_sync_duration_ms":2000,"repo_terminal_state":"fresh","objects":[{"kind":"roa","result":"ok"}]}],"repo_sync_stats":{"publication_points_total":2,"by_phase":{"rrdp_delta":{"count":2,"duration_ms_total":3000}},"by_terminal_state":{"fresh":{"count":2,"duration_ms_total":3000}}}}"#,
)
.expect("report");
fs::write(run.join("input.cir"), sample_cir()).expect("cir");
@@ -2041,8 +2163,11 @@ mod tests {
assert!(metrics.contains("ours_rp_large_publication_points"));
assert!(metrics.contains("ours_rp_cir_objects"));
assert!(metrics.contains("ours_rp_ccr_state_items"));
assert!(metrics.contains(r#"ours_rp_vrps{instance="test",kind="total"} 3"#));
assert!(metrics.contains(r#"ours_rp_vrps{instance="test",kind="unique"} 2"#));
let status = render_status_json(&snapshot).expect("status");
assert!(status.contains("topPublicationPointsByObjectCount"));
assert!(status.contains(r#""vrpsUnique": 2"#));
}
#[test]

1381
src/tools/rpki_inter_rp_metrics.rs Normal file
View File

File diff suppressed because it is too large Load Diff