NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260605 移除audit rule index和DB trace

Browse Source
This commit is contained in:
yuyr
2026-06-05 16:02:25 +08:00
parent 9579f65501
commit e597c7c124
29 changed files with 4032 additions and 2298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
src/cli/output.rs
View File

@@ -1,7 +1,10 @@
use std::io::BufWriter;
use std::path::Path;
use crate::audit::AuditReportV2;
use serde::Serialize;
use serde::ser::SerializeSeq;
use crate::audit::{AspaOutput, AuditRunMeta, AuditWarning, VrpOutput};
use crate::ccr::canonical_vrp_prefix;
use super::{PostValidationShared, RunStageTiming};
@@ -12,9 +15,9 @@ pub(super) enum ReportJsonFormat {
Compact,
}
pub(super) fn write_json(
pub(super) fn write_json<T: Serialize>(
path: &Path,
report: &AuditReportV2,
report: &T,
format: ReportJsonFormat,
) -> Result<(), String> {
let f = std::fs::File::create(path)
@@ -28,6 +31,128 @@ pub(super) fn write_json(
Ok(())
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(super) struct ReportJsonWriteTiming {
pub(super) build_ms: u64,
pub(super) write_ms: u64,
}
#[derive(Serialize)]
struct BorrowedAuditReportV2<'a> {
format_version: u32,
meta: AuditRunMeta,
policy: &'a crate::policy::Policy,
tree: BorrowedTreeSummary<'a>,
publication_points: &'a [crate::audit::PublicationPointAudit],
vrps: VrpReportSequence<'a>,
aspas: AspaReportSequence<'a>,
downloads: &'a [crate::audit::AuditDownloadEvent],
download_stats: &'a crate::audit::AuditDownloadStats,
repo_sync_stats: crate::audit::AuditRepoSyncStats,
}
#[derive(Serialize)]
struct BorrowedTreeSummary<'a> {
instances_processed: usize,
instances_failed: usize,
warnings: WarningReportSequence<'a>,
}
struct WarningReportSequence<'a>(&'a [crate::report::Warning]);
impl Serialize for WarningReportSequence<'_> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let mut seq = serializer.serialize_seq(Some(self.0.len()))?;
for warning in self.0 {
seq.serialize_element(&AuditWarning::from(warning))?;
}
seq.end()
}
}
struct VrpReportSequence<'a>(&'a [crate::validation::objects::Vrp]);
impl Serialize for VrpReportSequence<'_> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let mut seq = serializer.serialize_seq(Some(self.0.len()))?;
for vrp in self.0 {
seq.serialize_element(&VrpOutput {
asn: vrp.asn,
prefix: crate::audit::format_roa_ip_prefix(&vrp.prefix),
max_length: vrp.max_length,
})?;
}
seq.end()
}
}
struct AspaReportSequence<'a>(&'a [crate::validation::objects::AspaAttestation]);
impl Serialize for AspaReportSequence<'_> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let mut seq = serializer.serialize_seq(Some(self.0.len()))?;
for aspa in self.0 {
seq.serialize_element(&AspaOutput {
customer_as_id: aspa.customer_as_id,
provider_as_ids: aspa.provider_as_ids.clone(),
})?;
}
seq.end()
}
}
pub(super) fn write_report_json_from_shared(
path: &Path,
policy: &crate::policy::Policy,
validation_time: time::OffsetDateTime,
shared: &PostValidationShared,
format: ReportJsonFormat,
) -> Result<ReportJsonWriteTiming, String> {
use time::format_description::well_known::Rfc3339;
let build_started = std::time::Instant::now();
let validation_time_rfc3339_utc = validation_time
.to_offset(time::UtcOffset::UTC)
.format(&Rfc3339)
.expect("format validation_time");
let repo_sync_stats = super::build_repo_sync_stats(shared.publication_points.as_ref());
let report = BorrowedAuditReportV2 {
format_version: 2,
meta: AuditRunMeta {
validation_time_rfc3339_utc,
},
policy,
tree: BorrowedTreeSummary {
instances_processed: shared.instances_processed,
instances_failed: shared.instances_failed,
warnings: WarningReportSequence(shared.tree_warnings.as_ref()),
},
publication_points: shared.publication_points.as_ref(),
vrps: VrpReportSequence(shared.vrps.as_ref()),
aspas: AspaReportSequence(shared.aspas.as_ref()),
downloads: shared.downloads.as_ref(),
download_stats: &shared.download_stats,
repo_sync_stats,
};
let build_ms = build_started.elapsed().as_millis() as u64;
let write_started = std::time::Instant::now();
write_json(path, &report, format)?;
Ok(ReportJsonWriteTiming {
build_ms,
write_ms: write_started.elapsed().as_millis() as u64,
})
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(super) struct CompareViewTaskOutput {
pub(super) build_ms: Option<u64>,

224
src/cli/tests.rs
View File

@@ -1,4 +1,12 @@
use super::*;
use crate::memory_telemetry::{
MemoryTelemetryCheckpoint, MemoryTelemetrySummary, ProcessMemorySnapshot,
};
use crate::storage::{
RocksDbMemorySnapshot, RocksDbMemoryTotals, VcirCcrProjectionSizeBreakdown,
VcirChildResourceSizeBreakdown, VcirCoreFieldSizeBreakdown, VcirFieldSizeBreakdown,
VcirStorageEntrySummary, VcirStorageSummary,
};
#[test]
fn parse_help_returns_usage() {
@@ -9,6 +17,7 @@ fn parse_help_returns_usage() {
assert!(err.contains("--rsync-mirror-root"), "{err}");
assert!(err.contains("--rsync-scope"), "{err}");
assert!(err.contains("--parallel-phase2-object-workers"), "{err}");
assert!(err.contains("--memory-trim-after-validation"), "{err}");
assert!(!err.contains("--parallel-phase1"), "{err}");
assert!(!err.contains("--parallel-phase2 "), "{err}");
}
@@ -82,6 +91,37 @@ fn parse_accepts_ccr_out_path() {
);
}
#[test]
fn parse_accepts_memory_trim_after_validation() {
let argv = vec![
"rpki".to_string(),
"--db".to_string(),
"db".to_string(),
"--tal-path".to_string(),
"x.tal".to_string(),
"--ta-path".to_string(),
"x.cer".to_string(),
"--memory-trim-after-validation".to_string(),
];
let args = parse_args(&argv).expect("parse args");
assert!(args.memory_trim_after_validation);
}
#[test]
fn parse_disables_memory_trim_after_validation_by_default() {
let argv = vec![
"rpki".to_string(),
"--db".to_string(),
"db".to_string(),
"--tal-path".to_string(),
"x.tal".to_string(),
"--ta-path".to_string(),
"x.cer".to_string(),
];
let args = parse_args(&argv).expect("parse args");
assert!(!args.memory_trim_after_validation);
}
#[test]
fn parse_accepts_report_json_compact_when_report_json_is_set() {
let argv = vec![
@@ -1414,13 +1454,14 @@ fn run_report_task_and_stage_timing_work() {
)
.expect("run report task");
let report = report_output.report.as_ref().expect("report built");
assert_eq!(report.vrps.len(), 2);
assert_eq!(report.aspas.len(), 1);
assert!(report_output.report_write_ms.is_some());
let report_json = std::fs::read_to_string(&report_path).expect("read report json");
assert!(!report_json.contains('\n'), "{report_json}");
let report: serde_json::Value =
serde_json::from_str(&report_json).expect("parse compact report json");
assert_eq!(report["vrps"].as_array().unwrap().len(), 2);
assert_eq!(report["aspas"].as_array().unwrap().len(), 1);
let stage_timing = RunStageTiming {
validation_ms: 1,
@@ -1442,12 +1483,62 @@ fn run_report_task_and_stage_timing_work() {
rrdp_download_ms_total: 13,
rsync_download_ms_total: 14,
download_bytes_total: 15,
vcir_storage_summary_ms: Some(16),
vcir_storage: Some(VcirStorageSummary {
entry_count: 2,
vcir_value_bytes: 100,
vcir_value_bytes_max: 60,
vcir_value_bytes_max_manifest_rsync_uri: Some(
"rsync://example.test/repo/max.mft".to_string(),
),
core_fields: VcirCoreFieldSizeBreakdown {
manifest_rsync_uri_bytes: 10,
..VcirCoreFieldSizeBreakdown::default()
},
ccr_projection: VcirCcrProjectionSizeBreakdown {
manifest_sha256_bytes: 32,
..VcirCcrProjectionSizeBreakdown::default()
},
child_resources: VcirChildResourceSizeBreakdown {
effective_ip_resource_cbor_bytes: 12,
effective_as_resource_cbor_bytes: 6,
},
field_sizes: VcirFieldSizeBreakdown {
local_output_count: 1,
local_output_payload_json_bytes: 70,
local_output_payload_typed_body_bytes: 20,
..VcirFieldSizeBreakdown::default()
},
local_output_old_projection_bytes: 80,
local_output_typed_projection_bytes: 30,
local_output_projection_saved_bytes: 50,
top_entries_by_vcir_value_bytes: vec![VcirStorageEntrySummary {
manifest_rsync_uri: "rsync://example.test/repo/max.mft".to_string(),
vcir_value_bytes: 60,
local_vrp_count: 1,
local_aspa_count: 0,
local_router_key_count: 0,
accepted_object_count: 1,
rejected_object_count: 0,
child_count: 0,
core_fields: VcirCoreFieldSizeBreakdown::default(),
ccr_projection: VcirCcrProjectionSizeBreakdown::default(),
child_resources: VcirChildResourceSizeBreakdown::default(),
field_sizes: VcirFieldSizeBreakdown::default(),
local_output_old_projection_bytes: 1,
local_output_typed_projection_bytes: 1,
local_output_projection_saved_bytes: 0,
}],
}),
memory_telemetry: None,
};
write_stage_timing(Some(&report_path), &stage_timing).expect("write stage timing");
let stage_timing_json =
std::fs::read_to_string(dir.path().join("stage-timing.json")).expect("read timing");
assert!(stage_timing_json.contains("\"validation_ms\""));
assert!(stage_timing_json.contains("\"ccr_build_ms\""));
assert!(stage_timing_json.contains("\"vcir_storage\""));
assert!(stage_timing_json.contains("\"local_output_projection_saved_bytes\""));
let ccr_path = dir.path().join("result.ccr");
write_stage_timing(Some(&ccr_path), &stage_timing).expect("write stage timing via ccr path");
@@ -1457,11 +1548,136 @@ fn run_report_task_and_stage_timing_work() {
);
let skipped = ReportTaskOutput::skipped();
assert!(skipped.report.is_none());
assert_eq!(skipped.report_build_ms, 0);
assert!(skipped.report_write_ms.is_none());
}
#[test]
fn stage_timing_serializes_memory_telemetry() {
let dir = tempfile::tempdir().expect("tmpdir");
let report_path = dir.path().join("report.json");
let stage_timing = RunStageTiming {
validation_ms: 1,
report_build_ms: 2,
report_write_ms: None,
ccr_build_ms: None,
ccr_build_breakdown: None,
ccr_write_ms: None,
compare_view_build_ms: None,
compare_view_write_ms: None,
cir_build_cir_ms: None,
cir_write_cir_ms: None,
cir_total_ms: None,
total_ms: 3,
publication_points: 4,
repo_sync_ms_total: 5,
publication_point_repo_sync_ms_total: 6,
download_event_count: 7,
rrdp_download_ms_total: 8,
rsync_download_ms_total: 9,
download_bytes_total: 10,
vcir_storage_summary_ms: None,
vcir_storage: None,
memory_telemetry: Some(MemoryTelemetrySummary {
checkpoints: vec![MemoryTelemetryCheckpoint {
label: "after_validation".to_string(),
elapsed_ms: 11,
process: ProcessMemorySnapshot {
label: "after_validation".to_string(),
vm_rss_kb: Some(12),
vm_size_kb: None,
vm_data_kb: None,
vm_swap_kb: None,
rss_anon_kb: Some(13),
rss_file_kb: None,
rss_shmem_kb: None,
threads: Some(14),
fd_count: Some(15),
smaps_rollup: None,
smaps_mapping_summary: None,
errors: Vec::new(),
},
rocksdb: RocksDbMemorySnapshot {
databases: Vec::new(),
totals: RocksDbMemoryTotals {
cur_size_all_mem_tables: 16,
size_all_mem_tables: 17,
estimate_table_readers_mem: 18,
block_cache_capacity: 19,
block_cache_usage: 20,
block_cache_pinned_usage: 21,
},
},
}],
object_graph: None,
malloc_trim_probes: Vec::new(),
}),
};
write_stage_timing(Some(&report_path), &stage_timing).expect("write stage timing");
let value: serde_json::Value = serde_json::from_str(
&std::fs::read_to_string(dir.path().join("stage-timing.json")).unwrap(),
)
.expect("parse stage timing json");
let checkpoint = &value["memory_telemetry"]["checkpoints"][0];
assert_eq!(checkpoint["label"], "after_validation");
assert_eq!(checkpoint["process"]["vm_rss_kb"], 12);
assert_eq!(
checkpoint["rocksdb"]["totals"]["cur_size_all_mem_tables"],
16
);
assert!(
value["memory_telemetry"]
.as_object()
.expect("memory telemetry object")
.get("malloc_trim_probes")
.is_none()
);
}
#[test]
fn shared_object_graph_estimate_counts_audit_and_outputs() {
let mut shared = synthetic_post_validation_shared();
let mut publication_points = shared
.publication_points
.iter()
.cloned()
.collect::<Vec<_>>();
publication_points[0].rsync_base_uri = "rsync://example.test/repo/".to_string();
publication_points[0].manifest_rsync_uri = "rsync://example.test/repo/a.mft".to_string();
publication_points[0].publication_point_rsync_uri = "rsync://example.test/repo/".to_string();
publication_points[0].objects = vec![crate::audit::ObjectAuditEntry {
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
sha256_hex: "11".repeat(32),
kind: crate::audit::AuditObjectKind::Roa,
result: crate::audit::AuditObjectResult::Ok,
detail: None,
}];
shared.publication_points = publication_points.into();
let graph = estimate_shared_object_graph(&shared);
let publication_points_section = graph
.sections
.iter()
.find(|section| section.name == "publication_points")
.expect("publication points section");
let object_count = publication_points_section
.details
.iter()
.find(|metric| metric.name == "object_audit_entry_count")
.expect("object count metric");
assert_eq!(object_count.value, 1);
assert!(publication_points_section.estimated_bytes > 0);
let vrps_section = graph
.sections
.iter()
.find(|section| section.name == "vrps")
.expect("vrps section");
assert_eq!(vrps_section.item_count, 2);
assert!(graph.total_estimated_bytes >= publication_points_section.estimated_bytes);
}
#[test]
fn run_compare_view_task_writes_csv_from_shared_output() {
let shared = synthetic_post_validation_shared();