diff --git a/deploy/docker-installer/compose/grafana/dashboards/ours-rp-soak-overview.json b/deploy/docker-installer/compose/grafana/dashboards/ours-rp-soak-overview.json index 8f47bb3..2bb4238 100644 --- a/deploy/docker-installer/compose/grafana/dashboards/ours-rp-soak-overview.json +++ b/deploy/docker-installer/compose/grafana/dashboards/ours-rp-soak-overview.json @@ -1108,6 +1108,85 @@ ], "title": "CCR Format Check Results", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "decimals": 0, + "min": 0, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "ber_compatible_cms_encoding" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unclassified" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "gray", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 68, + "w": 24, + "h": 8 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "sum by (warning_category) (ours_rp_warnings_by_category)", + "legendFormat": "{{warning_category}}", + "refId": "A" + } + ], + "title": "Warnings by Category", + "type": "timeseries" } ], "refresh": "5s", @@ -1128,6 +1207,6 @@ "timezone": "browser", "title": "Ours RP Soak Overview", "uid": "ours-rp-soak-overview", - "version": 4, + "version": 5, "weekStart": "" } diff --git a/docker/ours-rp-runtime.Dockerfile b/docker/ours-rp-runtime.Dockerfile index 2122bf3..2f2c334 100644 --- a/docker/ours-rp-runtime.Dockerfile +++ b/docker/ours-rp-runtime.Dockerfile @@ -81,15 +81,24 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ ;; \ esac; \ fi; \ + # The public crate CDN occasionally rejects a burst of parallel fetches + # from a BuildKit worker. Keep the release build resilient without + # changing the produced binaries or their dependency graph. + export \ + CARGO_NET_RETRY=10 \ + CARGO_HTTP_TIMEOUT=120 \ + CARGO_HTTP_MULTIPLEXING=false; \ cargo build --release --target "$target_triple" \ --bin rpki \ --bin rpki_daemon \ - --bin db_stats; \ + --bin db_stats \ + --bin rpki_der_ber_audit; \ mkdir -p /build-out/bin; \ cp \ "target/$target_triple/release/rpki" \ "target/$target_triple/release/rpki_daemon" \ "target/$target_triple/release/db_stats" \ + "target/$target_triple/release/rpki_der_ber_audit" \ /build-out/bin/ FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS runtime diff --git a/scripts/docker/build_docker_runtime_image.sh b/scripts/docker/build_docker_runtime_image.sh index a28fed7..170204d 100755 --- a/scripts/docker/build_docker_runtime_image.sh +++ b/scripts/docker/build_docker_runtime_image.sh @@ -14,6 +14,10 @@ BUILDER_NAME="${BUILDER_NAME:-default}" INSTALL_BINFMT="${INSTALL_BINFMT:-1}" SAVE_IMAGE="${SAVE_IMAGE:-1}" LOAD_IMAGE="${LOAD_IMAGE:-1}" +# BuildKit's default bridge network can differ from the Docker daemon proxy +# configuration. Set BUILD_NETWORK=host when the build container needs the +# host's known-good registry connectivity. +BUILD_NETWORK="${BUILD_NETWORK:-default}" usage() { cat <<'USAGE' @@ -216,6 +220,7 @@ echo "builder_source_image: $builder_source_image" echo "runtime_source_image: $runtime_source_image" echo "builder_local_image: $BUILDER_IMAGE" echo "runtime_local_image: $RUNTIME_IMAGE" +echo "build_network: $BUILD_NETWORK" echo "source_commit: $SOURCE_COMMIT_FULL" echo "source_dirty: $SOURCE_DIRTY" echo "build_timestamp_utc: $BUILD_TIMESTAMP_UTC" @@ -225,6 +230,7 @@ if [[ "$LOAD_IMAGE" == "1" ]]; then docker buildx build \ --platform "$target_platform" \ --builder "$BUILDER_NAME" \ + --network "$BUILD_NETWORK" \ --pull=false \ --load \ --build-arg BUILDKIT_INLINE_CACHE=1 \ @@ -242,6 +248,7 @@ else docker buildx build \ --platform "$target_platform" \ --builder "$BUILDER_NAME" \ + --network "$BUILD_NETWORK" \ --pull=false \ --output "type=docker,dest=$raw_tar_path" \ --build-arg BUILDKIT_INLINE_CACHE=1 \ diff --git a/src/audit.rs b/src/audit.rs index bdf503e..9825025 100644 --- a/src/audit.rs +++ b/src/audit.rs @@ -36,6 +36,7 @@ pub struct ObjectAuditEntry { #[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub struct AuditWarning { pub message: String, + pub category: String, pub rfc_refs: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub context: Option, @@ -102,6 +103,7 @@ impl From<&crate::report::Warning> for AuditWarning { fn from(w: &crate::report::Warning) -> Self { Self { message: w.message.clone(), + category: w.category.as_str().to_string(), rfc_refs: w.rfc_refs.iter().map(|r| r.0.to_string()).collect(), context: w.context.clone(), } @@ -238,6 +240,15 @@ mod tests { assert!(value.get("cir_fresh_objects").is_none()); assert!(value.get("cir_cached_objects").is_none()); } + + #[test] + fn warning_audit_keeps_the_warning_category() { + let warning = crate::report::Warning::new("BER-compatible CMS accepted") + .with_category(crate::report::WarningCategory::BerCompatibleCmsEncoding); + + let audit_warning = AuditWarning::from(&warning); + assert_eq!(audit_warning.category, "ber_compatible_cms_encoding"); + } } #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] diff --git a/src/bin/rpki_der_ber_audit.rs b/src/bin/rpki_der_ber_audit.rs new file mode 100644 index 0000000..b4d25a9 --- /dev/null +++ b/src/bin/rpki_der_ber_audit.rs @@ -0,0 +1,437 @@ +use rocksdb::{DB, IteratorMode, Options}; +use rpki::data_model::crl::RpkixCrl; +use rpki::data_model::rc::ResourceCertificate; +use rpki::data_model::signed_object::RpkiSignedObject; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Default)] +struct Args { + input: Option, + repo_bytes_db: Option, + output: Option, + pretty: bool, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +enum EncodingStatus { + Der, + BerCompatibleOnly, + ParseFailed, + NotRpki, +} + +#[derive(Clone, Debug, Serialize)] +struct ObjectRecord { + sha256: String, + object_type: String, + encoding_status: EncodingStatus, + bytes: u64, + path: String, + #[serde(skip_serializing_if = "Option::is_none")] + detail: Option, +} + +#[derive(Debug, Serialize)] +struct AuditReport { + generated_by: &'static str, + input: String, + files_seen: u64, + recognized_rpki_objects: u64, + duplicate_content_files: u64, + non_rpki_entries: u64, + scan_errors: Vec, + status_counts: BTreeMap, + object_type_status_counts: BTreeMap>, + objects: Vec, +} + +fn usage() -> &'static str { + "Usage: rpki_der_ber_audit (--input |--repo-bytes-db ) [--output ] [--compact]" +} + +fn main() { + if let Err(err) = real_main() { + eprintln!("{err}"); + std::process::exit(1); + } +} + +fn real_main() -> Result<(), String> { + let args = parse_args(&std::env::args().collect::>())?; + let input_label = args + .input + .as_ref() + .or(args.repo_bytes_db.as_ref()) + .expect("validated input"); + let mut report = AuditReport { + generated_by: "rpki_der_ber_audit", + input: input_label.display().to_string(), + files_seen: 0, + recognized_rpki_objects: 0, + duplicate_content_files: 0, + non_rpki_entries: 0, + scan_errors: Vec::new(), + status_counts: BTreeMap::new(), + object_type_status_counts: BTreeMap::new(), + objects: Vec::new(), + }; + let mut seen_hashes = BTreeSet::new(); + if let Some(input) = args.input.as_ref() { + scan_directory(input, &mut report, &mut seen_hashes)?; + } else if let Some(repo_bytes_db) = args.repo_bytes_db.as_ref() { + scan_repo_bytes_db(repo_bytes_db, &mut report, &mut seen_hashes)?; + } + report + .objects + .sort_by(|left, right| left.sha256.cmp(&right.sha256)); + let rendered = if args.pretty { + serde_json::to_string_pretty(&report) + } else { + serde_json::to_string(&report) + } + .map_err(|err| format!("serialize audit report: {err}"))?; + + if let Some(output) = args.output { + if let Some(parent) = output.parent() { + std::fs::create_dir_all(parent) + .map_err(|err| format!("create output directory {}: {err}", parent.display()))?; + } + std::fs::write(&output, rendered) + .map_err(|err| format!("write audit report {}: {err}", output.display()))?; + } else { + println!("{rendered}"); + } + Ok(()) +} + +fn parse_args(argv: &[String]) -> Result { + let mut args = Args { + pretty: true, + ..Args::default() + }; + let mut index = 1usize; + while index < argv.len() { + match argv[index].as_str() { + "--input" | "--in" => { + index += 1; + args.input = Some(PathBuf::from( + argv.get(index).ok_or("--input requires a value")?, + )); + } + "--repo-bytes-db" => { + index += 1; + args.repo_bytes_db = Some(PathBuf::from( + argv.get(index).ok_or("--repo-bytes-db requires a value")?, + )); + } + "--output" | "--out" => { + index += 1; + args.output = Some(PathBuf::from( + argv.get(index).ok_or("--output requires a value")?, + )); + } + "--compact" => args.pretty = false, + "--pretty" => args.pretty = true, + "-h" | "--help" => return Err(usage().to_string()), + other => return Err(format!("unknown argument: {other}\n{}", usage())), + } + index += 1; + } + if args.input.is_some() == args.repo_bytes_db.is_some() { + return Err(format!("select exactly one input source\n{}", usage())); + } + if let Some(input) = args.input.as_ref() + && !input.is_dir() + { + return Err(format!("--input must be a directory: {}", input.display())); + } + if let Some(repo_bytes_db) = args.repo_bytes_db.as_ref() + && !repo_bytes_db.is_dir() + { + return Err(format!( + "--repo-bytes-db must be a RocksDB directory: {}", + repo_bytes_db.display() + )); + } + Ok(args) +} + +fn scan_directory( + directory: &Path, + report: &mut AuditReport, + seen_hashes: &mut BTreeSet, +) -> Result<(), String> { + let entries = std::fs::read_dir(directory) + .map_err(|err| format!("read directory {}: {err}", directory.display()))?; + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(err) => { + report.scan_errors.push(format!( + "read directory entry under {}: {err}", + directory.display() + )); + continue; + } + }; + let path = entry.path(); + let file_type = match entry.file_type() { + Ok(file_type) => file_type, + Err(err) => { + report + .scan_errors + .push(format!("read file type {}: {err}", path.display())); + continue; + } + }; + if file_type.is_dir() { + scan_directory(&path, report, seen_hashes)?; + } else if file_type.is_file() { + scan_file(&path, report, seen_hashes); + } + } + Ok(()) +} + +fn scan_file(path: &Path, report: &mut AuditReport, seen_hashes: &mut BTreeSet) { + report.files_seen += 1; + let Some(object_type) = object_type_from_path(path) else { + return; + }; + let bytes = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(err) => { + report + .scan_errors + .push(format!("read {}: {err}", path.display())); + return; + } + }; + let sha256 = hex::encode(Sha256::digest(&bytes)); + let (encoding_status, detail) = classify_object(&object_type, &bytes); + record_object( + report, + seen_hashes, + sha256, + object_type, + encoding_status, + bytes.len() as u64, + path.display().to_string(), + detail, + ); +} + +fn scan_repo_bytes_db( + repo_bytes_db: &Path, + report: &mut AuditReport, + seen_hashes: &mut BTreeSet, +) -> Result<(), String> { + let mut options = Options::default(); + options.create_if_missing(false); + let db = DB::open_for_read_only(&options, repo_bytes_db, false) + .map_err(|err| format!("open repo-bytes DB {}: {err}", repo_bytes_db.display()))?; + for entry in db.iterator(IteratorMode::Start) { + let (key, bytes) = entry.map_err(|err| format!("iterate repo-bytes DB: {err}"))?; + let Ok(key) = std::str::from_utf8(&key) else { + continue; + }; + let Some(sha256) = key.strip_prefix("sha256:") else { + continue; + }; + report.files_seen += 1; + let (object_type, encoding_status, detail) = classify_repo_blob(&bytes); + record_object( + report, + seen_hashes, + sha256.to_string(), + object_type, + encoding_status, + bytes.len() as u64, + format!("rocksdb://{}/{}", repo_bytes_db.display(), key), + detail, + ); + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn record_object( + report: &mut AuditReport, + seen_hashes: &mut BTreeSet, + sha256: String, + object_type: String, + encoding_status: EncodingStatus, + bytes: u64, + path: String, + detail: Option, +) { + if matches!(encoding_status, EncodingStatus::NotRpki) { + report.non_rpki_entries += 1; + } else { + report.recognized_rpki_objects += 1; + } + if !seen_hashes.insert(sha256.clone()) { + report.duplicate_content_files += 1; + return; + } + let status_name = encoding_status_name(&encoding_status).to_string(); + *report.status_counts.entry(status_name.clone()).or_default() += 1; + *report + .object_type_status_counts + .entry(object_type.clone()) + .or_default() + .entry(status_name) + .or_default() += 1; + report.objects.push(ObjectRecord { + sha256, + object_type, + encoding_status, + bytes, + path, + detail, + }); +} + +fn object_type_from_path(path: &Path) -> Option { + let extension = path.extension()?.to_str()?.to_ascii_lowercase(); + match extension.as_str() { + "mft" | "roa" | "asa" | "aspa" | "gbr" | "cer" | "crl" => Some(extension), + _ => None, + } +} + +fn classify_object(object_type: &str, bytes: &[u8]) -> (EncodingStatus, Option) { + match object_type { + "mft" | "roa" | "asa" | "aspa" | "gbr" => match RpkiSignedObject::parse_der(bytes) { + Ok(_) => match RpkiSignedObject::strict_cms_der_error(bytes) { + Some(err) => (EncodingStatus::BerCompatibleOnly, Some(err.to_string())), + None => (EncodingStatus::Der, None), + }, + Err(err) => (EncodingStatus::ParseFailed, Some(err.to_string())), + }, + "cer" => match ResourceCertificate::parse_der(bytes) { + Ok(_) => (EncodingStatus::Der, None), + Err(err) => (EncodingStatus::ParseFailed, Some(err.to_string())), + }, + "crl" => match RpkixCrl::parse_der(bytes) { + Ok(_) => (EncodingStatus::Der, None), + Err(err) => (EncodingStatus::ParseFailed, Some(err.to_string())), + }, + _ => unreachable!("object type was filtered before classification"), + } +} + +fn classify_repo_blob(bytes: &[u8]) -> (String, EncodingStatus, Option) { + if looks_like_tal_metadata(bytes) { + return ( + "tal".to_string(), + EncodingStatus::NotRpki, + Some("TAL text metadata, not an RPKI object".to_string()), + ); + } + if RpkiSignedObject::parse_der(bytes).is_ok() { + return match RpkiSignedObject::strict_cms_der_error(bytes) { + Some(err) => ( + "cms_signed_object".to_string(), + EncodingStatus::BerCompatibleOnly, + Some(err.to_string()), + ), + None => ("cms_signed_object".to_string(), EncodingStatus::Der, None), + }; + } + if ResourceCertificate::parse_der(bytes).is_ok() { + return ("cer".to_string(), EncodingStatus::Der, None); + } + if RpkixCrl::parse_der(bytes).is_ok() { + return ("crl".to_string(), EncodingStatus::Der, None); + } + ( + "unknown".to_string(), + EncodingStatus::ParseFailed, + Some("not parsed as CMS, RPKI resource certificate, or CRL".to_string()), + ) +} + +fn encoding_status_name(status: &EncodingStatus) -> &'static str { + match status { + EncodingStatus::Der => "der", + EncodingStatus::BerCompatibleOnly => "ber_compatible_only", + EncodingStatus::ParseFailed => "parse_failed", + EncodingStatus::NotRpki => "not_rpki", + } +} + +/// Return true for the TAL text format persisted by the live-TA refresh. +/// +/// A TAL is intentionally not DER: it contains an rsync URI, an HTTPS URI, +/// a blank separator, and the base64-encoded trust-anchor certificate. It +/// can share the repo-bytes DB with downloaded RPKI objects, so the audit must +/// keep it out of the object encoding verdict rather than report a false +/// parse failure. +fn looks_like_tal_metadata(bytes: &[u8]) -> bool { + let Ok(text) = std::str::from_utf8(bytes) else { + return false; + }; + let mut lines = text.lines(); + let Some(rsync_uri) = lines.next() else { + return false; + }; + let Some(https_uri) = lines.next() else { + return false; + }; + if !rsync_uri.starts_with("rsync://") || !https_uri.starts_with("https://") { + return false; + } + // `str::lines` removes the blank line, so inspect the raw prefix to make + // sure the URI pair is followed by the TAL separator. + let Some(separator) = text.find("\n\n") else { + return false; + }; + let base64 = &text[separator + 2..]; + !base64.is_empty() + && base64.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'/' | b'=' | b'\n' | b'\r') + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classifies_known_der_manifest_fixture() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.mft", + ); + let bytes = std::fs::read(path).expect("read manifest fixture"); + assert!(matches!( + classify_object("mft", &bytes).0, + EncodingStatus::Der + )); + } + + #[test] + fn recognizes_supported_object_extensions() { + assert_eq!( + object_type_from_path(Path::new("a.roa")), + Some("roa".to_string()) + ); + assert_eq!(object_type_from_path(Path::new("a.txt")), None); + } + + #[test] + fn classifies_tal_metadata_as_not_rpki() { + let tal = b"rsync://rpki.example/ta.cer\nhttps://rpki.example/ta.cer\n\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A\n"; + let (object_type, status, detail) = classify_repo_blob(tal); + assert_eq!(object_type, "tal"); + assert!(matches!(status, EncodingStatus::NotRpki)); + assert_eq!( + detail.as_deref(), + Some("TAL text metadata, not an RPKI object") + ); + } +} diff --git a/src/data_model/signed_object.rs b/src/data_model/signed_object.rs index 26efbb2..524f0c0 100644 --- a/src/data_model/signed_object.rs +++ b/src/data_model/signed_object.rs @@ -330,6 +330,16 @@ impl RpkiSignedObject { parse_signed_object_content_info(der, der, CmsParseMode::DerStrict) } + /// Return the strict-DER CMS parse error for an object that was otherwise + /// accepted through the normal BER-compatible CMS parser. + /// + /// Callers must only surface this as a compatibility warning after normal + /// decoding and validation have succeeded; a strict parse failure alone + /// does not prove that an arbitrary byte string is an RPKI signed object. + pub fn strict_cms_der_error(der: &[u8]) -> Option { + Self::parse_der_strict_cms(der).err() + } + /// Decode a DER-encoded RPKI Signed Object (CMS ContentInfo wrapping SignedData) and enforce /// the profile constraints from RFC 6488 §2-§3 and RFC 9589 §4. pub fn decode_der(der: &[u8]) -> Result { @@ -1541,6 +1551,10 @@ mod tests { let err = RpkiSignedObject::parse_der_strict_cms(&mutated) .expect_err("DER strict rejects constructed OCTET STRING"); assert!(err.to_string().contains("DER"), "{err}"); + let compatibility_error = RpkiSignedObject::strict_cms_der_error(&mutated) + .expect("BER-compatible fixture must report strict-DER incompatibility"); + assert_eq!(compatibility_error.to_string(), err.to_string()); + assert!(RpkiSignedObject::strict_cms_der_error(&der).is_none()); } fn replace_first_subslice(input: &[u8], from: &[u8], to: &[u8]) -> Option> { diff --git a/src/report.rs b/src/report.rs index 99c61e7..942511d 100644 --- a/src/report.rs +++ b/src/report.rs @@ -1,9 +1,31 @@ #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct RfcRef(pub &'static str); +/// Stable categories for warnings written to audit reports and exported metrics. +/// +/// New warning sites should use a specific category when one is available. The +/// default preserves the behaviour and schema of existing callers while making +/// their metric label explicit. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum WarningCategory { + #[default] + Unclassified, + BerCompatibleCmsEncoding, +} + +impl WarningCategory { + pub const fn as_str(self) -> &'static str { + match self { + Self::Unclassified => "unclassified", + Self::BerCompatibleCmsEncoding => "ber_compatible_cms_encoding", + } + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct Warning { pub message: String, + pub category: WarningCategory, pub rfc_refs: Vec, pub context: Option, } @@ -12,11 +34,17 @@ impl Warning { pub fn new(message: impl Into) -> Self { Self { message: message.into(), + category: WarningCategory::Unclassified, rfc_refs: Vec::new(), context: None, } } + pub fn with_category(mut self, category: WarningCategory) -> Self { + self.category = category; + self + } + pub fn with_rfc_refs(mut self, refs: &[RfcRef]) -> Self { self.rfc_refs.extend_from_slice(refs); self diff --git a/src/tools/rpki_artifact_metrics.rs b/src/tools/rpki_artifact_metrics.rs index 2bf6709..0c29782 100644 --- a/src/tools/rpki_artifact_metrics.rs +++ b/src/tools/rpki_artifact_metrics.rs @@ -424,6 +424,7 @@ struct LatestRunMetrics { vaps: u64, publication_points: u64, warnings: u64, + warning_categories: BTreeMap, tree_instances_processed: Option, tree_instances_failed: Option, stage_seconds: BTreeMap, @@ -1477,13 +1478,9 @@ fn parse_report(path: &Path, snapshot: &mut MetricsSnapshot, latest: &mut Latest .map(|a| a.len() as u64) .unwrap_or(0); } - latest.warnings = latest.warnings.max( - report - .pointer("/tree/warnings") - .and_then(|v| v.as_array()) - .map(|a| a.len() as u64) - .unwrap_or(0), - ); + let (reported_warning_count, warning_categories) = collect_warning_categories(&report); + latest.warnings = latest.warnings.max(reported_warning_count); + latest.warning_categories = warning_categories; if let Some(processed) = json_u64(&report, &["tree", "instances_processed"]) { latest.tree_instances_processed = Some(processed); } @@ -1504,6 +1501,47 @@ fn parse_report(path: &Path, snapshot: &mut MetricsSnapshot, latest: &mut Latest } } +fn collect_warning_categories(report: &Value) -> (u64, BTreeMap) { + let mut total = 0u64; + let mut categories = BTreeMap::new(); + total += count_warning_categories( + report.pointer("/tree/warnings").and_then(Value::as_array), + &mut categories, + ); + if let Some(publication_points) = report.get("publication_points").and_then(Value::as_array) { + for publication_point in publication_points { + total += count_warning_categories( + publication_point.get("warnings").and_then(Value::as_array), + &mut categories, + ); + } + } + // Export the baseline and BER-compatibility series even when their count is + // zero, so dashboards can distinguish a clean run from a missing metric. + for category in ["unclassified", "ber_compatible_cms_encoding"] { + categories.entry(category.to_string()).or_insert(0); + } + (total, categories) +} + +fn count_warning_categories( + warnings: Option<&Vec>, + categories: &mut BTreeMap, +) -> u64 { + let Some(warnings) = warnings else { + return 0; + }; + for warning in warnings { + let category = warning + .get("category") + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .unwrap_or("unclassified"); + *categories.entry(category.to_string()).or_default() += 1; + } + warnings.len() as u64 +} + fn extract_publication_point_metrics( pps: &[Value], downloads: Option<&Value>, @@ -2409,6 +2447,17 @@ fn render_latest_metrics(writer: &mut PromWriter<'_>, instance: &str, latest: &L &[label("instance", instance)], latest.warnings as f64, ); + for (category, count) in &latest.warning_categories { + writer.gauge( + "ours_rp_warnings_by_category", + "Latest run warning count by category", + &[ + label("instance", instance), + label("warning_category", category), + ], + *count as f64, + ); + } if let Some(value) = latest.tree_instances_processed { writer.gauge( "ours_rp_tree_instances", @@ -3364,6 +3413,48 @@ mod tests { assert_eq!(args.out_metrics.as_deref(), Some(Path::new("metrics.prom"))); } + #[test] + fn warning_categories_cover_tree_and_publication_point_warnings() { + let report = json!({ + "tree": { + "warnings": [ + {"category": "ber_compatible_cms_encoding"}, + {"category": "ber_compatible_cms_encoding"} + ] + }, + "publication_points": [ + {"warnings": [{"category": "repository_sync"}]}, + {"warnings": [{"message": "legacy warning without category"}]} + ] + }); + + let (total, categories) = collect_warning_categories(&report); + assert_eq!(total, 4); + assert_eq!(categories["ber_compatible_cms_encoding"], 2); + assert_eq!(categories["repository_sync"], 1); + assert_eq!(categories["unclassified"], 1); + } + + #[test] + fn renders_warning_category_metrics() { + let mut warning_categories = BTreeMap::new(); + warning_categories.insert("ber_compatible_cms_encoding".to_string(), 2); + let snapshot = MetricsSnapshot { + instance: "test".to_string(), + latest_run: Some(LatestRunMetrics { + warnings: 2, + warning_categories, + ..LatestRunMetrics::default() + }), + ..MetricsSnapshot::default() + }; + + let metrics = render_metrics(&snapshot); + assert!(metrics.contains( + r#"ours_rp_warnings_by_category{instance="test",warning_category="ber_compatible_cms_encoding"} 2"# + )); + } + #[test] fn scan_fixture_exports_repo_pp_cir_and_ccr_metrics() { let td = TempDir::new().expect("tempdir"); diff --git a/src/validation/objects.rs b/src/validation/objects.rs index 5575436..7a60cd7 100644 --- a/src/validation/objects.rs +++ b/src/validation/objects.rs @@ -8,13 +8,13 @@ use crate::data_model::rc::{ ResourceCertificate, }; use crate::data_model::roa::{IpPrefix, RoaAfi, RoaDecodeError, RoaObject, RoaValidateError}; -use crate::data_model::signed_object::SignedObjectVerifyError; +use crate::data_model::signed_object::{RpkiSignedObject, SignedObjectVerifyError}; use crate::parallel::config::ParallelPhase2Config; use crate::parallel::object_worker::{ ObjectTaskExecutor, ObjectWorkerPool, ObjectWorkerSubmitError, }; use crate::policy::{Policy, ResourceValidationMode, SignedObjectFailurePolicy}; -use crate::report::{RfcRef, Warning}; +use crate::report::{RfcRef, Warning, WarningCategory}; use crate::storage::{ PackFile, PackTime, RoaCacheObjectMeta, RoaCacheProjection, VcirLocalOutput, VcirLocalOutputPayload, VcirOutputType, VcirSourceObjectType, @@ -34,6 +34,22 @@ const RFC_CRLDP: &[RfcRef] = &[RfcRef("RFC 6487 §4.8.6")]; const RFC_CRLDP_AND_LOCKED_PACK: &[RfcRef] = &[RfcRef("RFC 6487 §4.8.6"), RfcRef("RFC 9286 §4.2.1")]; +fn ber_compatible_cms_warning(der: &[u8], rsync_uri: &str, object_kind: &str) -> Option { + let strict_error = RpkiSignedObject::strict_cms_der_error(der)?; + Some( + Warning::new(format!( + "accepted BER-compatible CMS encoding for {object_kind}: {rsync_uri}: {strict_error}" + )) + .with_category(WarningCategory::BerCompatibleCmsEncoding) + .with_rfc_refs(&[RfcRef("X.690 §10"), RfcRef("RFC 6488 §2")]) + .with_context(rsync_uri), + ) +} + +fn ber_compatible_cms_warning_for_file(file: &PackFile, object_kind: &str) -> Option { + ber_compatible_cms_warning(file.bytes().ok()?, &file.rsync_uri, object_kind) +} + fn sha256_hex_to_32(hex_value: &str) -> [u8; 32] { let bytes = hex::decode(hex_value).expect("internal sha256 hex should decode"); let mut out = [0u8; 32]; @@ -1015,6 +1031,12 @@ pub fn process_publication_point_for_issuer_with_cache_options v, @@ -1317,6 +1339,9 @@ pub fn process_publication_point_for_issuer_with_cache_options match policy.signed_object_failure_policy { SignedObjectFailurePolicy::DropObject => { @@ -1426,6 +1451,9 @@ pub fn process_publication_point_for_issuer_with_cache_options match policy.signed_object_failure_policy { SignedObjectFailurePolicy::DropObject => { @@ -2050,6 +2078,12 @@ pub(crate) fn prepare_publication_point_for_parallel_roa_with_cache v, Err(e) => { @@ -2367,6 +2401,9 @@ pub(crate) fn reduce_parallel_roa_stage( result: AuditObjectResult::Ok, detail: None, }); + if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ROA") { + warnings.push(warning); + } } Err(e) => { audit.push(ObjectAuditEntry { @@ -2418,6 +2455,9 @@ pub(crate) fn reduce_parallel_roa_stage( result: AuditObjectResult::Ok, detail: None, }); + if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ASPA") { + warnings.push(warning); + } } Err(e) => { audit.push(ObjectAuditEntry { diff --git a/src/validation/tree_parallel.rs b/src/validation/tree_parallel.rs index faadca3..53982d0 100644 --- a/src/validation/tree_parallel.rs +++ b/src/validation/tree_parallel.rs @@ -3013,6 +3013,7 @@ mod tests { }); sample.audit.warnings.push(crate::audit::AuditWarning { message: "warning".to_string(), + category: "unclassified".to_string(), rfc_refs: Vec::new(), context: None, });