20260408_2 增加CIR sequence，未验证drop analysis，遇到问题是static pool保存太慢，拖慢整体录制，待解决
This commit is contained in:
250
src/bin/cir_drop_report.rs
Normal file
250
src/bin/cir_drop_report.rs
Normal file
@@ -0,0 +1,250 @@
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use rpki::bundle::decode_ccr_compare_views;
|
||||
use rpki::ccr::decode_content_info;
|
||||
use rpki::cir::{decode_cir, resolve_static_pool_file};
|
||||
use rpki::data_model::roa::RoaObject;
|
||||
|
||||
const USAGE: &str = "Usage: cir_drop_report --cir <path> --ccr <path> --report-json <path> --static-root <path> --json-out <path> --md-out <path>";
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct DroppedObjectRecord {
|
||||
uri: String,
|
||||
sha256: String,
|
||||
kind: String,
|
||||
reason_code: String,
|
||||
reason_text: Option<String>,
|
||||
publication_point: Option<String>,
|
||||
manifest_uri: Option<String>,
|
||||
derived_vrp_count: usize,
|
||||
}
|
||||
|
||||
fn classify_reason(detail: Option<&str>, result: &str) -> String {
|
||||
let text = detail.unwrap_or("").to_ascii_lowercase();
|
||||
if text.contains("fetch") {
|
||||
"fetch_failed".to_string()
|
||||
} else if text.contains("manifest") {
|
||||
"manifest_invalid".to_string()
|
||||
} else if text.contains("crl") {
|
||||
"crl_invalid".to_string()
|
||||
} else if text.contains("policy") {
|
||||
"policy_rejected".to_string()
|
||||
} else if text.contains("parse") {
|
||||
"object_parse_failed".to_string()
|
||||
} else if text.contains("signature") || text.contains("cms") {
|
||||
"cms_signature_invalid".to_string()
|
||||
} else if text.contains("resource") {
|
||||
"resource_invalid".to_string()
|
||||
} else if text.contains("expired") || text.contains("not yet valid") {
|
||||
"expired_or_not_yet_valid".to_string()
|
||||
} else if result == "skipped" {
|
||||
"skipped".to_string()
|
||||
} else if result == "error" {
|
||||
"error".to_string()
|
||||
} else {
|
||||
"other".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_args(argv: &[String]) -> Result<(PathBuf, PathBuf, PathBuf, PathBuf, PathBuf, PathBuf), String> {
|
||||
let mut cir = None;
|
||||
let mut ccr = None;
|
||||
let mut report = None;
|
||||
let mut static_root = None;
|
||||
let mut json_out = None;
|
||||
let mut md_out = None;
|
||||
let mut i = 1usize;
|
||||
while i < argv.len() {
|
||||
match argv[i].as_str() {
|
||||
"--cir" => {
|
||||
i += 1;
|
||||
cir = Some(PathBuf::from(argv.get(i).ok_or("--cir requires a value")?));
|
||||
}
|
||||
"--ccr" => {
|
||||
i += 1;
|
||||
ccr = Some(PathBuf::from(argv.get(i).ok_or("--ccr requires a value")?));
|
||||
}
|
||||
"--report-json" => {
|
||||
i += 1;
|
||||
report = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--report-json requires a value")?,
|
||||
));
|
||||
}
|
||||
"--static-root" => {
|
||||
i += 1;
|
||||
static_root = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--static-root requires a value")?,
|
||||
));
|
||||
}
|
||||
"--json-out" => {
|
||||
i += 1;
|
||||
json_out = Some(PathBuf::from(
|
||||
argv.get(i).ok_or("--json-out requires a value")?,
|
||||
));
|
||||
}
|
||||
"--md-out" => {
|
||||
i += 1;
|
||||
md_out = Some(PathBuf::from(argv.get(i).ok_or("--md-out requires a value")?));
|
||||
}
|
||||
"-h" | "--help" => return Err(USAGE.to_string()),
|
||||
other => return Err(format!("unknown argument: {other}\n\n{USAGE}")),
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
Ok((
|
||||
cir.ok_or_else(|| format!("--cir is required\n\n{USAGE}"))?,
|
||||
ccr.ok_or_else(|| format!("--ccr is required\n\n{USAGE}"))?,
|
||||
report.ok_or_else(|| format!("--report-json is required\n\n{USAGE}"))?,
|
||||
static_root.ok_or_else(|| format!("--static-root is required\n\n{USAGE}"))?,
|
||||
json_out.ok_or_else(|| format!("--json-out is required\n\n{USAGE}"))?,
|
||||
md_out.ok_or_else(|| format!("--md-out is required\n\n{USAGE}"))?,
|
||||
))
|
||||
}
|
||||
|
||||
fn main() -> Result<(), String> {
|
||||
let argv: Vec<String> = std::env::args().collect();
|
||||
let (cir_path, ccr_path, report_path, static_root, json_out, md_out) = parse_args(&argv)?;
|
||||
|
||||
let cir = decode_cir(&std::fs::read(&cir_path).map_err(|e| format!("read cir failed: {e}"))?)
|
||||
.map_err(|e| format!("decode cir failed: {e}"))?;
|
||||
let ccr = decode_content_info(
|
||||
&std::fs::read(&ccr_path).map_err(|e| format!("read ccr failed: {e}"))?,
|
||||
)
|
||||
.map_err(|e| format!("decode ccr failed: {e}"))?;
|
||||
let (vrps, vaps) =
|
||||
decode_ccr_compare_views(&ccr, "unknown").map_err(|e| format!("decode compare views failed: {e}"))?;
|
||||
let report: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(&report_path).map_err(|e| format!("read report failed: {e}"))?)
|
||||
.map_err(|e| format!("parse report failed: {e}"))?;
|
||||
|
||||
let mut object_hash_by_uri = BTreeMap::new();
|
||||
for object in &cir.objects {
|
||||
object_hash_by_uri.insert(object.rsync_uri.clone(), hex::encode(&object.sha256));
|
||||
}
|
||||
|
||||
let publication_points = report["publication_points"]
|
||||
.as_array()
|
||||
.ok_or("report.publication_points must be an array")?;
|
||||
|
||||
let mut dropped_objects = Vec::new();
|
||||
let mut dropped_vrp_rows = BTreeSet::new();
|
||||
let mut dropped_by_kind: BTreeMap<String, usize> = BTreeMap::new();
|
||||
let mut dropped_by_reason: BTreeMap<String, usize> = BTreeMap::new();
|
||||
let mut unknown_roa_objects = 0usize;
|
||||
|
||||
for pp in publication_points {
|
||||
let publication_point = pp["publication_point_rsync_uri"].as_str().map(str::to_string);
|
||||
let manifest_uri = pp["manifest_rsync_uri"].as_str().map(str::to_string);
|
||||
for obj in pp["objects"].as_array().into_iter().flatten() {
|
||||
let result = obj["result"].as_str().unwrap_or("unknown");
|
||||
if result == "ok" {
|
||||
continue;
|
||||
}
|
||||
let uri = obj["rsync_uri"].as_str().unwrap_or("").to_string();
|
||||
let hash = obj["sha256_hex"]
|
||||
.as_str()
|
||||
.map(str::to_string)
|
||||
.or_else(|| object_hash_by_uri.get(&uri).cloned())
|
||||
.unwrap_or_default();
|
||||
let kind = obj["kind"].as_str().unwrap_or("other").to_string();
|
||||
let detail = obj["detail"].as_str().map(str::to_string);
|
||||
let reason_code = classify_reason(detail.as_deref(), result);
|
||||
*dropped_by_kind.entry(kind.clone()).or_insert(0) += 1;
|
||||
*dropped_by_reason.entry(reason_code.clone()).or_insert(0) += 1;
|
||||
|
||||
let mut derived_vrp_count = 0usize;
|
||||
if kind == "roa" && !hash.is_empty() {
|
||||
match resolve_static_pool_file(&static_root, &hash) {
|
||||
Ok(path) => {
|
||||
if let Ok(bytes) = std::fs::read(&path) {
|
||||
if let Ok(roa) = RoaObject::decode_der(&bytes) {
|
||||
for family in roa.roa.ip_addr_blocks {
|
||||
for addr in family.addresses {
|
||||
let prefix = match addr.prefix.afi {
|
||||
rpki::data_model::roa::RoaAfi::Ipv4 => format!(
|
||||
"{}.{}.{}.{}/{}",
|
||||
addr.prefix.addr[0],
|
||||
addr.prefix.addr[1],
|
||||
addr.prefix.addr[2],
|
||||
addr.prefix.addr[3],
|
||||
addr.prefix.prefix_len
|
||||
),
|
||||
rpki::data_model::roa::RoaAfi::Ipv6 => {
|
||||
let bytes: [u8; 16] = addr.prefix.addr;
|
||||
format!("{}/{}", std::net::Ipv6Addr::from(bytes), addr.prefix.prefix_len)
|
||||
}
|
||||
};
|
||||
let max_len = addr.max_length.unwrap_or(addr.prefix.prefix_len);
|
||||
dropped_vrp_rows.insert((roa.roa.as_id, prefix, max_len));
|
||||
derived_vrp_count += 1;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
unknown_roa_objects += 1;
|
||||
}
|
||||
} else {
|
||||
unknown_roa_objects += 1;
|
||||
}
|
||||
}
|
||||
Err(_) => unknown_roa_objects += 1,
|
||||
}
|
||||
}
|
||||
|
||||
dropped_objects.push(DroppedObjectRecord {
|
||||
uri,
|
||||
sha256: hash,
|
||||
kind,
|
||||
reason_code,
|
||||
reason_text: detail,
|
||||
publication_point: publication_point.clone(),
|
||||
manifest_uri: manifest_uri.clone(),
|
||||
derived_vrp_count,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let output = serde_json::json!({
|
||||
"summary": {
|
||||
"finalVrpCount": vrps.len(),
|
||||
"finalVapCount": vaps.len(),
|
||||
"droppedVrpCount": dropped_vrp_rows.len(),
|
||||
"droppedObjectCount": dropped_objects.len(),
|
||||
"droppedByKind": dropped_by_kind,
|
||||
"droppedByReason": dropped_by_reason,
|
||||
"unknownDroppedRoaObjects": unknown_roa_objects,
|
||||
},
|
||||
"objects": dropped_objects,
|
||||
});
|
||||
|
||||
if let Some(parent) = json_out.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| format!("create json parent failed: {e}"))?;
|
||||
}
|
||||
std::fs::write(&json_out, serde_json::to_vec_pretty(&output).unwrap())
|
||||
.map_err(|e| format!("write json failed: {e}"))?;
|
||||
|
||||
let mut md = String::new();
|
||||
md.push_str("# CIR Drop Report\n\n");
|
||||
md.push_str(&format!("- `final_vrp_count`: `{}`\n", vrps.len()));
|
||||
md.push_str(&format!("- `final_vap_count`: `{}`\n", vaps.len()));
|
||||
md.push_str(&format!("- `dropped_vrp_count`: `{}`\n", output["summary"]["droppedVrpCount"]));
|
||||
md.push_str(&format!("- `dropped_object_count`: `{}`\n", output["summary"]["droppedObjectCount"]));
|
||||
md.push_str(&format!(
|
||||
"- `unknown_dropped_roa_objects`: `{}`\n\n",
|
||||
output["summary"]["unknownDroppedRoaObjects"]
|
||||
));
|
||||
md.push_str("## Dropped By Kind\n\n");
|
||||
for (kind, count) in output["summary"]["droppedByKind"].as_object().into_iter().flatten() {
|
||||
md.push_str(&format!("- `{kind}`: `{}`\n", count.as_u64().unwrap_or(0)));
|
||||
}
|
||||
md.push_str("\n## Dropped By Reason\n\n");
|
||||
for (reason, count) in output["summary"]["droppedByReason"].as_object().into_iter().flatten() {
|
||||
md.push_str(&format!("- `{reason}`: `{}`\n", count.as_u64().unwrap_or(0)));
|
||||
}
|
||||
if let Some(parent) = md_out.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| format!("create markdown parent failed: {e}"))?;
|
||||
}
|
||||
std::fs::write(&md_out, md).map_err(|e| format!("write markdown failed: {e}"))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
102
src/bin/cir_ta_only_fixture.rs
Normal file
102
src/bin/cir_ta_only_fixture.rs
Normal file
@@ -0,0 +1,102 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use rpki::cir::{encode_cir, write_bytes_to_static_pool, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal, CIR_VERSION_V1};
|
||||
use sha2::Digest;
|
||||
|
||||
const USAGE: &str = "Usage: cir_ta_only_fixture --tal-path <path> --ta-path <path> --tal-uri <url> --validation-time <rfc3339> --cir-out <path> --static-root <path>";
|
||||
|
||||
fn parse_args(argv: &[String]) -> Result<(PathBuf, PathBuf, String, time::OffsetDateTime, PathBuf, PathBuf), String> {
|
||||
let mut tal_path = None;
|
||||
let mut ta_path = None;
|
||||
let mut tal_uri = None;
|
||||
let mut validation_time = None;
|
||||
let mut cir_out = None;
|
||||
let mut static_root = None;
|
||||
let mut i = 1usize;
|
||||
while i < argv.len() {
|
||||
match argv[i].as_str() {
|
||||
"--tal-path" => {
|
||||
i += 1;
|
||||
tal_path = Some(PathBuf::from(argv.get(i).ok_or("--tal-path requires a value")?));
|
||||
}
|
||||
"--ta-path" => {
|
||||
i += 1;
|
||||
ta_path = Some(PathBuf::from(argv.get(i).ok_or("--ta-path requires a value")?));
|
||||
}
|
||||
"--tal-uri" => {
|
||||
i += 1;
|
||||
tal_uri = Some(argv.get(i).ok_or("--tal-uri requires a value")?.clone());
|
||||
}
|
||||
"--validation-time" => {
|
||||
i += 1;
|
||||
let raw = argv.get(i).ok_or("--validation-time requires a value")?;
|
||||
validation_time = Some(
|
||||
time::OffsetDateTime::parse(
|
||||
raw,
|
||||
&time::format_description::well_known::Rfc3339,
|
||||
)
|
||||
.map_err(|e| format!("invalid validation time: {e}"))?,
|
||||
);
|
||||
}
|
||||
"--cir-out" => {
|
||||
i += 1;
|
||||
cir_out = Some(PathBuf::from(argv.get(i).ok_or("--cir-out requires a value")?));
|
||||
}
|
||||
"--static-root" => {
|
||||
i += 1;
|
||||
static_root =
|
||||
Some(PathBuf::from(argv.get(i).ok_or("--static-root requires a value")?));
|
||||
}
|
||||
"-h" | "--help" => return Err(USAGE.to_string()),
|
||||
other => return Err(format!("unknown argument: {other}\n\n{USAGE}")),
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
Ok((
|
||||
tal_path.ok_or_else(|| format!("--tal-path is required\n\n{USAGE}"))?,
|
||||
ta_path.ok_or_else(|| format!("--ta-path is required\n\n{USAGE}"))?,
|
||||
tal_uri.ok_or_else(|| format!("--tal-uri is required\n\n{USAGE}"))?,
|
||||
validation_time.ok_or_else(|| format!("--validation-time is required\n\n{USAGE}"))?,
|
||||
cir_out.ok_or_else(|| format!("--cir-out is required\n\n{USAGE}"))?,
|
||||
static_root.ok_or_else(|| format!("--static-root is required\n\n{USAGE}"))?,
|
||||
))
|
||||
}
|
||||
|
||||
fn main() -> Result<(), String> {
|
||||
let argv: Vec<String> = std::env::args().collect();
|
||||
let (tal_path, ta_path, tal_uri, validation_time, cir_out, static_root) = parse_args(&argv)?;
|
||||
|
||||
let tal_bytes = std::fs::read(&tal_path).map_err(|e| format!("read tal failed: {e}"))?;
|
||||
let ta_bytes = std::fs::read(&ta_path).map_err(|e| format!("read ta failed: {e}"))?;
|
||||
let tal = rpki::data_model::tal::Tal::decode_bytes(&tal_bytes)
|
||||
.map_err(|e| format!("decode tal failed: {e}"))?;
|
||||
let ta_rsync_uri = tal
|
||||
.ta_uris
|
||||
.iter()
|
||||
.find(|uri| uri.scheme() == "rsync")
|
||||
.ok_or("tal must contain an rsync URI")?
|
||||
.as_str()
|
||||
.to_string();
|
||||
|
||||
let sha = sha2::Sha256::digest(&ta_bytes);
|
||||
let hash_hex = hex::encode(sha);
|
||||
write_bytes_to_static_pool(&static_root, validation_time.date(), &hash_hex, &ta_bytes)
|
||||
.map_err(|e| format!("write static pool failed: {e}"))?;
|
||||
|
||||
let cir = CanonicalInputRepresentation {
|
||||
version: CIR_VERSION_V1,
|
||||
hash_alg: CirHashAlgorithm::Sha256,
|
||||
validation_time,
|
||||
objects: vec![CirObject {
|
||||
rsync_uri: ta_rsync_uri,
|
||||
sha256: sha.to_vec(),
|
||||
}],
|
||||
tals: vec![CirTal { tal_uri, tal_bytes }],
|
||||
};
|
||||
let der = encode_cir(&cir).map_err(|e| format!("encode cir failed: {e}"))?;
|
||||
if let Some(parent) = cir_out.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| format!("create cir parent failed: {e}"))?;
|
||||
}
|
||||
std::fs::write(&cir_out, der).map_err(|e| format!("write cir failed: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user