20260826 增加BER兼容告警与分类指标

This commit is contained in:
yuyr 2026-08-26 13:23:29 +08:00
parent 46fb5e5b99
commit ea3f0d5dde
10 changed files with 728 additions and 11 deletions

View File

@ -1108,6 +1108,85 @@
], ],
"title": "CCR Format Check Results", "title": "CCR Format Check Results",
"type": "timeseries" "type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"decimals": 0,
"min": 0,
"unit": "none"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "ber_compatible_cms_encoding"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "orange",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "unclassified"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "gray",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"x": 0,
"y": 68,
"w": 24,
"h": 8
},
"id": 24,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "sum by (warning_category) (ours_rp_warnings_by_category)",
"legendFormat": "{{warning_category}}",
"refId": "A"
}
],
"title": "Warnings by Category",
"type": "timeseries"
} }
], ],
"refresh": "5s", "refresh": "5s",
@ -1128,6 +1207,6 @@
"timezone": "browser", "timezone": "browser",
"title": "Ours RP Soak Overview", "title": "Ours RP Soak Overview",
"uid": "ours-rp-soak-overview", "uid": "ours-rp-soak-overview",
"version": 4, "version": 5,
"weekStart": "" "weekStart": ""
} }

View File

@ -81,15 +81,24 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
;; \ ;; \
esac; \ esac; \
fi; \ fi; \
# The public crate CDN occasionally rejects a burst of parallel fetches
# from a BuildKit worker. Keep the release build resilient without
# changing the produced binaries or their dependency graph.
export \
CARGO_NET_RETRY=10 \
CARGO_HTTP_TIMEOUT=120 \
CARGO_HTTP_MULTIPLEXING=false; \
cargo build --release --target "$target_triple" \ cargo build --release --target "$target_triple" \
--bin rpki \ --bin rpki \
--bin rpki_daemon \ --bin rpki_daemon \
--bin db_stats; \ --bin db_stats \
--bin rpki_der_ber_audit; \
mkdir -p /build-out/bin; \ mkdir -p /build-out/bin; \
cp \ cp \
"target/$target_triple/release/rpki" \ "target/$target_triple/release/rpki" \
"target/$target_triple/release/rpki_daemon" \ "target/$target_triple/release/rpki_daemon" \
"target/$target_triple/release/db_stats" \ "target/$target_triple/release/db_stats" \
"target/$target_triple/release/rpki_der_ber_audit" \
/build-out/bin/ /build-out/bin/
FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS runtime FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS runtime

View File

@ -14,6 +14,10 @@ BUILDER_NAME="${BUILDER_NAME:-default}"
INSTALL_BINFMT="${INSTALL_BINFMT:-1}" INSTALL_BINFMT="${INSTALL_BINFMT:-1}"
SAVE_IMAGE="${SAVE_IMAGE:-1}" SAVE_IMAGE="${SAVE_IMAGE:-1}"
LOAD_IMAGE="${LOAD_IMAGE:-1}" LOAD_IMAGE="${LOAD_IMAGE:-1}"
# BuildKit's default bridge network can differ from the Docker daemon proxy
# configuration. Set BUILD_NETWORK=host when the build container needs the
# host's known-good registry connectivity.
BUILD_NETWORK="${BUILD_NETWORK:-default}"
usage() { usage() {
cat <<'USAGE' cat <<'USAGE'
@ -216,6 +220,7 @@ echo "builder_source_image: $builder_source_image"
echo "runtime_source_image: $runtime_source_image" echo "runtime_source_image: $runtime_source_image"
echo "builder_local_image: $BUILDER_IMAGE" echo "builder_local_image: $BUILDER_IMAGE"
echo "runtime_local_image: $RUNTIME_IMAGE" echo "runtime_local_image: $RUNTIME_IMAGE"
echo "build_network: $BUILD_NETWORK"
echo "source_commit: $SOURCE_COMMIT_FULL" echo "source_commit: $SOURCE_COMMIT_FULL"
echo "source_dirty: $SOURCE_DIRTY" echo "source_dirty: $SOURCE_DIRTY"
echo "build_timestamp_utc: $BUILD_TIMESTAMP_UTC" echo "build_timestamp_utc: $BUILD_TIMESTAMP_UTC"
@ -225,6 +230,7 @@ if [[ "$LOAD_IMAGE" == "1" ]]; then
docker buildx build \ docker buildx build \
--platform "$target_platform" \ --platform "$target_platform" \
--builder "$BUILDER_NAME" \ --builder "$BUILDER_NAME" \
--network "$BUILD_NETWORK" \
--pull=false \ --pull=false \
--load \ --load \
--build-arg BUILDKIT_INLINE_CACHE=1 \ --build-arg BUILDKIT_INLINE_CACHE=1 \
@ -242,6 +248,7 @@ else
docker buildx build \ docker buildx build \
--platform "$target_platform" \ --platform "$target_platform" \
--builder "$BUILDER_NAME" \ --builder "$BUILDER_NAME" \
--network "$BUILD_NETWORK" \
--pull=false \ --pull=false \
--output "type=docker,dest=$raw_tar_path" \ --output "type=docker,dest=$raw_tar_path" \
--build-arg BUILDKIT_INLINE_CACHE=1 \ --build-arg BUILDKIT_INLINE_CACHE=1 \

View File

@ -36,6 +36,7 @@ pub struct ObjectAuditEntry {
#[derive(Clone, Debug, PartialEq, Eq, Serialize)] #[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct AuditWarning { pub struct AuditWarning {
pub message: String, pub message: String,
pub category: String,
pub rfc_refs: Vec<String>, pub rfc_refs: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub context: Option<String>, pub context: Option<String>,
@ -102,6 +103,7 @@ impl From<&crate::report::Warning> for AuditWarning {
fn from(w: &crate::report::Warning) -> Self { fn from(w: &crate::report::Warning) -> Self {
Self { Self {
message: w.message.clone(), message: w.message.clone(),
category: w.category.as_str().to_string(),
rfc_refs: w.rfc_refs.iter().map(|r| r.0.to_string()).collect(), rfc_refs: w.rfc_refs.iter().map(|r| r.0.to_string()).collect(),
context: w.context.clone(), context: w.context.clone(),
} }
@ -238,6 +240,15 @@ mod tests {
assert!(value.get("cir_fresh_objects").is_none()); assert!(value.get("cir_fresh_objects").is_none());
assert!(value.get("cir_cached_objects").is_none()); assert!(value.get("cir_cached_objects").is_none());
} }
#[test]
fn warning_audit_keeps_the_warning_category() {
let warning = crate::report::Warning::new("BER-compatible CMS accepted")
.with_category(crate::report::WarningCategory::BerCompatibleCmsEncoding);
let audit_warning = AuditWarning::from(&warning);
assert_eq!(audit_warning.category, "ber_compatible_cms_encoding");
}
} }
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]

View File

@ -0,0 +1,437 @@
use rocksdb::{DB, IteratorMode, Options};
use rpki::data_model::crl::RpkixCrl;
use rpki::data_model::rc::ResourceCertificate;
use rpki::data_model::signed_object::RpkiSignedObject;
use serde::Serialize;
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
#[derive(Debug, Default)]
struct Args {
input: Option<PathBuf>,
repo_bytes_db: Option<PathBuf>,
output: Option<PathBuf>,
pretty: bool,
}
#[derive(Clone, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
enum EncodingStatus {
Der,
BerCompatibleOnly,
ParseFailed,
NotRpki,
}
#[derive(Clone, Debug, Serialize)]
struct ObjectRecord {
sha256: String,
object_type: String,
encoding_status: EncodingStatus,
bytes: u64,
path: String,
#[serde(skip_serializing_if = "Option::is_none")]
detail: Option<String>,
}
#[derive(Debug, Serialize)]
struct AuditReport {
generated_by: &'static str,
input: String,
files_seen: u64,
recognized_rpki_objects: u64,
duplicate_content_files: u64,
non_rpki_entries: u64,
scan_errors: Vec<String>,
status_counts: BTreeMap<String, u64>,
object_type_status_counts: BTreeMap<String, BTreeMap<String, u64>>,
objects: Vec<ObjectRecord>,
}
fn usage() -> &'static str {
"Usage: rpki_der_ber_audit (--input <raw-rpki-object-directory>|--repo-bytes-db <repo-bytes.db>) [--output <audit.json>] [--compact]"
}
fn main() {
if let Err(err) = real_main() {
eprintln!("{err}");
std::process::exit(1);
}
}
fn real_main() -> Result<(), String> {
let args = parse_args(&std::env::args().collect::<Vec<_>>())?;
let input_label = args
.input
.as_ref()
.or(args.repo_bytes_db.as_ref())
.expect("validated input");
let mut report = AuditReport {
generated_by: "rpki_der_ber_audit",
input: input_label.display().to_string(),
files_seen: 0,
recognized_rpki_objects: 0,
duplicate_content_files: 0,
non_rpki_entries: 0,
scan_errors: Vec::new(),
status_counts: BTreeMap::new(),
object_type_status_counts: BTreeMap::new(),
objects: Vec::new(),
};
let mut seen_hashes = BTreeSet::new();
if let Some(input) = args.input.as_ref() {
scan_directory(input, &mut report, &mut seen_hashes)?;
} else if let Some(repo_bytes_db) = args.repo_bytes_db.as_ref() {
scan_repo_bytes_db(repo_bytes_db, &mut report, &mut seen_hashes)?;
}
report
.objects
.sort_by(|left, right| left.sha256.cmp(&right.sha256));
let rendered = if args.pretty {
serde_json::to_string_pretty(&report)
} else {
serde_json::to_string(&report)
}
.map_err(|err| format!("serialize audit report: {err}"))?;
if let Some(output) = args.output {
if let Some(parent) = output.parent() {
std::fs::create_dir_all(parent)
.map_err(|err| format!("create output directory {}: {err}", parent.display()))?;
}
std::fs::write(&output, rendered)
.map_err(|err| format!("write audit report {}: {err}", output.display()))?;
} else {
println!("{rendered}");
}
Ok(())
}
fn parse_args(argv: &[String]) -> Result<Args, String> {
let mut args = Args {
pretty: true,
..Args::default()
};
let mut index = 1usize;
while index < argv.len() {
match argv[index].as_str() {
"--input" | "--in" => {
index += 1;
args.input = Some(PathBuf::from(
argv.get(index).ok_or("--input requires a value")?,
));
}
"--repo-bytes-db" => {
index += 1;
args.repo_bytes_db = Some(PathBuf::from(
argv.get(index).ok_or("--repo-bytes-db requires a value")?,
));
}
"--output" | "--out" => {
index += 1;
args.output = Some(PathBuf::from(
argv.get(index).ok_or("--output requires a value")?,
));
}
"--compact" => args.pretty = false,
"--pretty" => args.pretty = true,
"-h" | "--help" => return Err(usage().to_string()),
other => return Err(format!("unknown argument: {other}\n{}", usage())),
}
index += 1;
}
if args.input.is_some() == args.repo_bytes_db.is_some() {
return Err(format!("select exactly one input source\n{}", usage()));
}
if let Some(input) = args.input.as_ref()
&& !input.is_dir()
{
return Err(format!("--input must be a directory: {}", input.display()));
}
if let Some(repo_bytes_db) = args.repo_bytes_db.as_ref()
&& !repo_bytes_db.is_dir()
{
return Err(format!(
"--repo-bytes-db must be a RocksDB directory: {}",
repo_bytes_db.display()
));
}
Ok(args)
}
fn scan_directory(
directory: &Path,
report: &mut AuditReport,
seen_hashes: &mut BTreeSet<String>,
) -> Result<(), String> {
let entries = std::fs::read_dir(directory)
.map_err(|err| format!("read directory {}: {err}", directory.display()))?;
for entry in entries {
let entry = match entry {
Ok(entry) => entry,
Err(err) => {
report.scan_errors.push(format!(
"read directory entry under {}: {err}",
directory.display()
));
continue;
}
};
let path = entry.path();
let file_type = match entry.file_type() {
Ok(file_type) => file_type,
Err(err) => {
report
.scan_errors
.push(format!("read file type {}: {err}", path.display()));
continue;
}
};
if file_type.is_dir() {
scan_directory(&path, report, seen_hashes)?;
} else if file_type.is_file() {
scan_file(&path, report, seen_hashes);
}
}
Ok(())
}
fn scan_file(path: &Path, report: &mut AuditReport, seen_hashes: &mut BTreeSet<String>) {
report.files_seen += 1;
let Some(object_type) = object_type_from_path(path) else {
return;
};
let bytes = match std::fs::read(path) {
Ok(bytes) => bytes,
Err(err) => {
report
.scan_errors
.push(format!("read {}: {err}", path.display()));
return;
}
};
let sha256 = hex::encode(Sha256::digest(&bytes));
let (encoding_status, detail) = classify_object(&object_type, &bytes);
record_object(
report,
seen_hashes,
sha256,
object_type,
encoding_status,
bytes.len() as u64,
path.display().to_string(),
detail,
);
}
fn scan_repo_bytes_db(
repo_bytes_db: &Path,
report: &mut AuditReport,
seen_hashes: &mut BTreeSet<String>,
) -> Result<(), String> {
let mut options = Options::default();
options.create_if_missing(false);
let db = DB::open_for_read_only(&options, repo_bytes_db, false)
.map_err(|err| format!("open repo-bytes DB {}: {err}", repo_bytes_db.display()))?;
for entry in db.iterator(IteratorMode::Start) {
let (key, bytes) = entry.map_err(|err| format!("iterate repo-bytes DB: {err}"))?;
let Ok(key) = std::str::from_utf8(&key) else {
continue;
};
let Some(sha256) = key.strip_prefix("sha256:") else {
continue;
};
report.files_seen += 1;
let (object_type, encoding_status, detail) = classify_repo_blob(&bytes);
record_object(
report,
seen_hashes,
sha256.to_string(),
object_type,
encoding_status,
bytes.len() as u64,
format!("rocksdb://{}/{}", repo_bytes_db.display(), key),
detail,
);
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn record_object(
report: &mut AuditReport,
seen_hashes: &mut BTreeSet<String>,
sha256: String,
object_type: String,
encoding_status: EncodingStatus,
bytes: u64,
path: String,
detail: Option<String>,
) {
if matches!(encoding_status, EncodingStatus::NotRpki) {
report.non_rpki_entries += 1;
} else {
report.recognized_rpki_objects += 1;
}
if !seen_hashes.insert(sha256.clone()) {
report.duplicate_content_files += 1;
return;
}
let status_name = encoding_status_name(&encoding_status).to_string();
*report.status_counts.entry(status_name.clone()).or_default() += 1;
*report
.object_type_status_counts
.entry(object_type.clone())
.or_default()
.entry(status_name)
.or_default() += 1;
report.objects.push(ObjectRecord {
sha256,
object_type,
encoding_status,
bytes,
path,
detail,
});
}
fn object_type_from_path(path: &Path) -> Option<String> {
let extension = path.extension()?.to_str()?.to_ascii_lowercase();
match extension.as_str() {
"mft" | "roa" | "asa" | "aspa" | "gbr" | "cer" | "crl" => Some(extension),
_ => None,
}
}
fn classify_object(object_type: &str, bytes: &[u8]) -> (EncodingStatus, Option<String>) {
match object_type {
"mft" | "roa" | "asa" | "aspa" | "gbr" => match RpkiSignedObject::parse_der(bytes) {
Ok(_) => match RpkiSignedObject::strict_cms_der_error(bytes) {
Some(err) => (EncodingStatus::BerCompatibleOnly, Some(err.to_string())),
None => (EncodingStatus::Der, None),
},
Err(err) => (EncodingStatus::ParseFailed, Some(err.to_string())),
},
"cer" => match ResourceCertificate::parse_der(bytes) {
Ok(_) => (EncodingStatus::Der, None),
Err(err) => (EncodingStatus::ParseFailed, Some(err.to_string())),
},
"crl" => match RpkixCrl::parse_der(bytes) {
Ok(_) => (EncodingStatus::Der, None),
Err(err) => (EncodingStatus::ParseFailed, Some(err.to_string())),
},
_ => unreachable!("object type was filtered before classification"),
}
}
fn classify_repo_blob(bytes: &[u8]) -> (String, EncodingStatus, Option<String>) {
if looks_like_tal_metadata(bytes) {
return (
"tal".to_string(),
EncodingStatus::NotRpki,
Some("TAL text metadata, not an RPKI object".to_string()),
);
}
if RpkiSignedObject::parse_der(bytes).is_ok() {
return match RpkiSignedObject::strict_cms_der_error(bytes) {
Some(err) => (
"cms_signed_object".to_string(),
EncodingStatus::BerCompatibleOnly,
Some(err.to_string()),
),
None => ("cms_signed_object".to_string(), EncodingStatus::Der, None),
};
}
if ResourceCertificate::parse_der(bytes).is_ok() {
return ("cer".to_string(), EncodingStatus::Der, None);
}
if RpkixCrl::parse_der(bytes).is_ok() {
return ("crl".to_string(), EncodingStatus::Der, None);
}
(
"unknown".to_string(),
EncodingStatus::ParseFailed,
Some("not parsed as CMS, RPKI resource certificate, or CRL".to_string()),
)
}
fn encoding_status_name(status: &EncodingStatus) -> &'static str {
match status {
EncodingStatus::Der => "der",
EncodingStatus::BerCompatibleOnly => "ber_compatible_only",
EncodingStatus::ParseFailed => "parse_failed",
EncodingStatus::NotRpki => "not_rpki",
}
}
/// Return true for the TAL text format persisted by the live-TA refresh.
///
/// A TAL is intentionally not DER: it contains an rsync URI, an HTTPS URI,
/// a blank separator, and the base64-encoded trust-anchor certificate. It
/// can share the repo-bytes DB with downloaded RPKI objects, so the audit must
/// keep it out of the object encoding verdict rather than report a false
/// parse failure.
fn looks_like_tal_metadata(bytes: &[u8]) -> bool {
let Ok(text) = std::str::from_utf8(bytes) else {
return false;
};
let mut lines = text.lines();
let Some(rsync_uri) = lines.next() else {
return false;
};
let Some(https_uri) = lines.next() else {
return false;
};
if !rsync_uri.starts_with("rsync://") || !https_uri.starts_with("https://") {
return false;
}
// `str::lines` removes the blank line, so inspect the raw prefix to make
// sure the URI pair is followed by the TAL separator.
let Some(separator) = text.find("\n\n") else {
return false;
};
let base64 = &text[separator + 2..];
!base64.is_empty()
&& base64.bytes().all(|byte| {
byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'/' | b'=' | b'\n' | b'\r')
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn classifies_known_der_manifest_fixture() {
let path = Path::new(env!("CARGO_MANIFEST_DIR")).join(
"tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.mft",
);
let bytes = std::fs::read(path).expect("read manifest fixture");
assert!(matches!(
classify_object("mft", &bytes).0,
EncodingStatus::Der
));
}
#[test]
fn recognizes_supported_object_extensions() {
assert_eq!(
object_type_from_path(Path::new("a.roa")),
Some("roa".to_string())
);
assert_eq!(object_type_from_path(Path::new("a.txt")), None);
}
#[test]
fn classifies_tal_metadata_as_not_rpki() {
let tal = b"rsync://rpki.example/ta.cer\nhttps://rpki.example/ta.cer\n\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A\n";
let (object_type, status, detail) = classify_repo_blob(tal);
assert_eq!(object_type, "tal");
assert!(matches!(status, EncodingStatus::NotRpki));
assert_eq!(
detail.as_deref(),
Some("TAL text metadata, not an RPKI object")
);
}
}

View File

@ -330,6 +330,16 @@ impl RpkiSignedObject {
parse_signed_object_content_info(der, der, CmsParseMode::DerStrict) parse_signed_object_content_info(der, der, CmsParseMode::DerStrict)
} }
/// Return the strict-DER CMS parse error for an object that was otherwise
/// accepted through the normal BER-compatible CMS parser.
///
/// Callers must only surface this as a compatibility warning after normal
/// decoding and validation have succeeded; a strict parse failure alone
/// does not prove that an arbitrary byte string is an RPKI signed object.
pub fn strict_cms_der_error(der: &[u8]) -> Option<SignedObjectParseError> {
Self::parse_der_strict_cms(der).err()
}
/// Decode a DER-encoded RPKI Signed Object (CMS ContentInfo wrapping SignedData) and enforce /// Decode a DER-encoded RPKI Signed Object (CMS ContentInfo wrapping SignedData) and enforce
/// the profile constraints from RFC 6488 §2-§3 and RFC 9589 §4. /// the profile constraints from RFC 6488 §2-§3 and RFC 9589 §4.
pub fn decode_der(der: &[u8]) -> Result<Self, SignedObjectDecodeError> { pub fn decode_der(der: &[u8]) -> Result<Self, SignedObjectDecodeError> {
@ -1541,6 +1551,10 @@ mod tests {
let err = RpkiSignedObject::parse_der_strict_cms(&mutated) let err = RpkiSignedObject::parse_der_strict_cms(&mutated)
.expect_err("DER strict rejects constructed OCTET STRING"); .expect_err("DER strict rejects constructed OCTET STRING");
assert!(err.to_string().contains("DER"), "{err}"); assert!(err.to_string().contains("DER"), "{err}");
let compatibility_error = RpkiSignedObject::strict_cms_der_error(&mutated)
.expect("BER-compatible fixture must report strict-DER incompatibility");
assert_eq!(compatibility_error.to_string(), err.to_string());
assert!(RpkiSignedObject::strict_cms_der_error(&der).is_none());
} }
fn replace_first_subslice(input: &[u8], from: &[u8], to: &[u8]) -> Option<Vec<u8>> { fn replace_first_subslice(input: &[u8], from: &[u8], to: &[u8]) -> Option<Vec<u8>> {

View File

@ -1,9 +1,31 @@
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct RfcRef(pub &'static str); pub struct RfcRef(pub &'static str);
/// Stable categories for warnings written to audit reports and exported metrics.
///
/// New warning sites should use a specific category when one is available. The
/// default preserves the behaviour and schema of existing callers while making
/// their metric label explicit.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum WarningCategory {
#[default]
Unclassified,
BerCompatibleCmsEncoding,
}
impl WarningCategory {
pub const fn as_str(self) -> &'static str {
match self {
Self::Unclassified => "unclassified",
Self::BerCompatibleCmsEncoding => "ber_compatible_cms_encoding",
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
pub struct Warning { pub struct Warning {
pub message: String, pub message: String,
pub category: WarningCategory,
pub rfc_refs: Vec<RfcRef>, pub rfc_refs: Vec<RfcRef>,
pub context: Option<String>, pub context: Option<String>,
} }
@ -12,11 +34,17 @@ impl Warning {
pub fn new(message: impl Into<String>) -> Self { pub fn new(message: impl Into<String>) -> Self {
Self { Self {
message: message.into(), message: message.into(),
category: WarningCategory::Unclassified,
rfc_refs: Vec::new(), rfc_refs: Vec::new(),
context: None, context: None,
} }
} }
pub fn with_category(mut self, category: WarningCategory) -> Self {
self.category = category;
self
}
pub fn with_rfc_refs(mut self, refs: &[RfcRef]) -> Self { pub fn with_rfc_refs(mut self, refs: &[RfcRef]) -> Self {
self.rfc_refs.extend_from_slice(refs); self.rfc_refs.extend_from_slice(refs);
self self

View File

@ -424,6 +424,7 @@ struct LatestRunMetrics {
vaps: u64, vaps: u64,
publication_points: u64, publication_points: u64,
warnings: u64, warnings: u64,
warning_categories: BTreeMap<String, u64>,
tree_instances_processed: Option<u64>, tree_instances_processed: Option<u64>,
tree_instances_failed: Option<u64>, tree_instances_failed: Option<u64>,
stage_seconds: BTreeMap<String, f64>, stage_seconds: BTreeMap<String, f64>,
@ -1477,13 +1478,9 @@ fn parse_report(path: &Path, snapshot: &mut MetricsSnapshot, latest: &mut Latest
.map(|a| a.len() as u64) .map(|a| a.len() as u64)
.unwrap_or(0); .unwrap_or(0);
} }
latest.warnings = latest.warnings.max( let (reported_warning_count, warning_categories) = collect_warning_categories(&report);
report latest.warnings = latest.warnings.max(reported_warning_count);
.pointer("/tree/warnings") latest.warning_categories = warning_categories;
.and_then(|v| v.as_array())
.map(|a| a.len() as u64)
.unwrap_or(0),
);
if let Some(processed) = json_u64(&report, &["tree", "instances_processed"]) { if let Some(processed) = json_u64(&report, &["tree", "instances_processed"]) {
latest.tree_instances_processed = Some(processed); latest.tree_instances_processed = Some(processed);
} }
@ -1504,6 +1501,47 @@ fn parse_report(path: &Path, snapshot: &mut MetricsSnapshot, latest: &mut Latest
} }
} }
fn collect_warning_categories(report: &Value) -> (u64, BTreeMap<String, u64>) {
let mut total = 0u64;
let mut categories = BTreeMap::new();
total += count_warning_categories(
report.pointer("/tree/warnings").and_then(Value::as_array),
&mut categories,
);
if let Some(publication_points) = report.get("publication_points").and_then(Value::as_array) {
for publication_point in publication_points {
total += count_warning_categories(
publication_point.get("warnings").and_then(Value::as_array),
&mut categories,
);
}
}
// Export the baseline and BER-compatibility series even when their count is
// zero, so dashboards can distinguish a clean run from a missing metric.
for category in ["unclassified", "ber_compatible_cms_encoding"] {
categories.entry(category.to_string()).or_insert(0);
}
(total, categories)
}
fn count_warning_categories(
warnings: Option<&Vec<Value>>,
categories: &mut BTreeMap<String, u64>,
) -> u64 {
let Some(warnings) = warnings else {
return 0;
};
for warning in warnings {
let category = warning
.get("category")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.unwrap_or("unclassified");
*categories.entry(category.to_string()).or_default() += 1;
}
warnings.len() as u64
}
fn extract_publication_point_metrics( fn extract_publication_point_metrics(
pps: &[Value], pps: &[Value],
downloads: Option<&Value>, downloads: Option<&Value>,
@ -2409,6 +2447,17 @@ fn render_latest_metrics(writer: &mut PromWriter<'_>, instance: &str, latest: &L
&[label("instance", instance)], &[label("instance", instance)],
latest.warnings as f64, latest.warnings as f64,
); );
for (category, count) in &latest.warning_categories {
writer.gauge(
"ours_rp_warnings_by_category",
"Latest run warning count by category",
&[
label("instance", instance),
label("warning_category", category),
],
*count as f64,
);
}
if let Some(value) = latest.tree_instances_processed { if let Some(value) = latest.tree_instances_processed {
writer.gauge( writer.gauge(
"ours_rp_tree_instances", "ours_rp_tree_instances",
@ -3364,6 +3413,48 @@ mod tests {
assert_eq!(args.out_metrics.as_deref(), Some(Path::new("metrics.prom"))); assert_eq!(args.out_metrics.as_deref(), Some(Path::new("metrics.prom")));
} }
#[test]
fn warning_categories_cover_tree_and_publication_point_warnings() {
let report = json!({
"tree": {
"warnings": [
{"category": "ber_compatible_cms_encoding"},
{"category": "ber_compatible_cms_encoding"}
]
},
"publication_points": [
{"warnings": [{"category": "repository_sync"}]},
{"warnings": [{"message": "legacy warning without category"}]}
]
});
let (total, categories) = collect_warning_categories(&report);
assert_eq!(total, 4);
assert_eq!(categories["ber_compatible_cms_encoding"], 2);
assert_eq!(categories["repository_sync"], 1);
assert_eq!(categories["unclassified"], 1);
}
#[test]
fn renders_warning_category_metrics() {
let mut warning_categories = BTreeMap::new();
warning_categories.insert("ber_compatible_cms_encoding".to_string(), 2);
let snapshot = MetricsSnapshot {
instance: "test".to_string(),
latest_run: Some(LatestRunMetrics {
warnings: 2,
warning_categories,
..LatestRunMetrics::default()
}),
..MetricsSnapshot::default()
};
let metrics = render_metrics(&snapshot);
assert!(metrics.contains(
r#"ours_rp_warnings_by_category{instance="test",warning_category="ber_compatible_cms_encoding"} 2"#
));
}
#[test] #[test]
fn scan_fixture_exports_repo_pp_cir_and_ccr_metrics() { fn scan_fixture_exports_repo_pp_cir_and_ccr_metrics() {
let td = TempDir::new().expect("tempdir"); let td = TempDir::new().expect("tempdir");

View File

@ -8,13 +8,13 @@ use crate::data_model::rc::{
ResourceCertificate, ResourceCertificate,
}; };
use crate::data_model::roa::{IpPrefix, RoaAfi, RoaDecodeError, RoaObject, RoaValidateError}; use crate::data_model::roa::{IpPrefix, RoaAfi, RoaDecodeError, RoaObject, RoaValidateError};
use crate::data_model::signed_object::SignedObjectVerifyError; use crate::data_model::signed_object::{RpkiSignedObject, SignedObjectVerifyError};
use crate::parallel::config::ParallelPhase2Config; use crate::parallel::config::ParallelPhase2Config;
use crate::parallel::object_worker::{ use crate::parallel::object_worker::{
ObjectTaskExecutor, ObjectWorkerPool, ObjectWorkerSubmitError, ObjectTaskExecutor, ObjectWorkerPool, ObjectWorkerSubmitError,
}; };
use crate::policy::{Policy, ResourceValidationMode, SignedObjectFailurePolicy}; use crate::policy::{Policy, ResourceValidationMode, SignedObjectFailurePolicy};
use crate::report::{RfcRef, Warning}; use crate::report::{RfcRef, Warning, WarningCategory};
use crate::storage::{ use crate::storage::{
PackFile, PackTime, RoaCacheObjectMeta, RoaCacheProjection, VcirLocalOutput, PackFile, PackTime, RoaCacheObjectMeta, RoaCacheProjection, VcirLocalOutput,
VcirLocalOutputPayload, VcirOutputType, VcirSourceObjectType, VcirLocalOutputPayload, VcirOutputType, VcirSourceObjectType,
@ -34,6 +34,22 @@ const RFC_CRLDP: &[RfcRef] = &[RfcRef("RFC 6487 §4.8.6")];
const RFC_CRLDP_AND_LOCKED_PACK: &[RfcRef] = const RFC_CRLDP_AND_LOCKED_PACK: &[RfcRef] =
&[RfcRef("RFC 6487 §4.8.6"), RfcRef("RFC 9286 §4.2.1")]; &[RfcRef("RFC 6487 §4.8.6"), RfcRef("RFC 9286 §4.2.1")];
fn ber_compatible_cms_warning(der: &[u8], rsync_uri: &str, object_kind: &str) -> Option<Warning> {
let strict_error = RpkiSignedObject::strict_cms_der_error(der)?;
Some(
Warning::new(format!(
"accepted BER-compatible CMS encoding for {object_kind}: {rsync_uri}: {strict_error}"
))
.with_category(WarningCategory::BerCompatibleCmsEncoding)
.with_rfc_refs(&[RfcRef("X.690 §10"), RfcRef("RFC 6488 §2")])
.with_context(rsync_uri),
)
}
fn ber_compatible_cms_warning_for_file(file: &PackFile, object_kind: &str) -> Option<Warning> {
ber_compatible_cms_warning(file.bytes().ok()?, &file.rsync_uri, object_kind)
}
fn sha256_hex_to_32(hex_value: &str) -> [u8; 32] { fn sha256_hex_to_32(hex_value: &str) -> [u8; 32] {
let bytes = hex::decode(hex_value).expect("internal sha256 hex should decode"); let bytes = hex::decode(hex_value).expect("internal sha256 hex should decode");
let mut out = [0u8; 32]; let mut out = [0u8; 32];
@ -1015,6 +1031,12 @@ pub fn process_publication_point_for_issuer_with_cache_options<P: PublicationPoi
} }
}; };
if let Some(warning) =
ber_compatible_cms_warning(manifest_bytes, manifest_rsync_uri, "manifest")
{
warnings.push(warning);
}
// Decode issuer CA once; if it fails we cannot validate ROA/ASPA EE certificates. // Decode issuer CA once; if it fails we cannot validate ROA/ASPA EE certificates.
let issuer_ca = match decode_resource_certificate_with_policy(issuer_ca_der, policy) { let issuer_ca = match decode_resource_certificate_with_policy(issuer_ca_der, policy) {
Ok(v) => v, Ok(v) => v,
@ -1317,6 +1339,9 @@ pub fn process_publication_point_for_issuer_with_cache_options<P: PublicationPoi
result: AuditObjectResult::Ok, result: AuditObjectResult::Ok,
detail: None, detail: None,
}); });
if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ROA") {
warnings.push(warning);
}
} }
Err(e) => match policy.signed_object_failure_policy { Err(e) => match policy.signed_object_failure_policy {
SignedObjectFailurePolicy::DropObject => { SignedObjectFailurePolicy::DropObject => {
@ -1426,6 +1451,9 @@ pub fn process_publication_point_for_issuer_with_cache_options<P: PublicationPoi
result: AuditObjectResult::Ok, result: AuditObjectResult::Ok,
detail: None, detail: None,
}); });
if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ASPA") {
warnings.push(warning);
}
} }
Err(e) => match policy.signed_object_failure_policy { Err(e) => match policy.signed_object_failure_policy {
SignedObjectFailurePolicy::DropObject => { SignedObjectFailurePolicy::DropObject => {
@ -2050,6 +2078,12 @@ pub(crate) fn prepare_publication_point_for_parallel_roa_with_cache<P: Publicati
} }
}; };
if let Some(warning) =
ber_compatible_cms_warning(manifest_bytes, manifest_rsync_uri, "manifest")
{
warnings.push(warning);
}
let issuer_ca = match decode_resource_certificate_with_policy(issuer_ca_der, policy) { let issuer_ca = match decode_resource_certificate_with_policy(issuer_ca_der, policy) {
Ok(v) => v, Ok(v) => v,
Err(e) => { Err(e) => {
@ -2367,6 +2401,9 @@ pub(crate) fn reduce_parallel_roa_stage(
result: AuditObjectResult::Ok, result: AuditObjectResult::Ok,
detail: None, detail: None,
}); });
if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ROA") {
warnings.push(warning);
}
} }
Err(e) => { Err(e) => {
audit.push(ObjectAuditEntry { audit.push(ObjectAuditEntry {
@ -2418,6 +2455,9 @@ pub(crate) fn reduce_parallel_roa_stage(
result: AuditObjectResult::Ok, result: AuditObjectResult::Ok,
detail: None, detail: None,
}); });
if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ASPA") {
warnings.push(warning);
}
} }
Err(e) => { Err(e) => {
audit.push(ObjectAuditEntry { audit.push(ObjectAuditEntry {

View File

@ -3013,6 +3013,7 @@ mod tests {
}); });
sample.audit.warnings.push(crate::audit::AuditWarning { sample.audit.warnings.push(crate::audit::AuditWarning {
message: "warning".to_string(), message: "warning".to_string(),
category: "unclassified".to_string(),
rfc_refs: Vec::new(), rfc_refs: Vec::new(),
context: None, context: None,
}); });