20260826 增加BER兼容告警与分类指标
This commit is contained in:
parent
46fb5e5b99
commit
ea3f0d5dde
@ -1108,6 +1108,85 @@
|
||||
],
|
||||
"title": "CCR Format Check Results",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "Prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "ber_compatible_cms_encoding"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "orange",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "unclassified"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "gray",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 68,
|
||||
"w": 24,
|
||||
"h": 8
|
||||
},
|
||||
"id": 24,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (warning_category) (ours_rp_warnings_by_category)",
|
||||
"legendFormat": "{{warning_category}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Warnings by Category",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "5s",
|
||||
@ -1128,6 +1207,6 @@
|
||||
"timezone": "browser",
|
||||
"title": "Ours RP Soak Overview",
|
||||
"uid": "ours-rp-soak-overview",
|
||||
"version": 4,
|
||||
"version": 5,
|
||||
"weekStart": ""
|
||||
}
|
||||
|
||||
@ -81,15 +81,24 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
;; \
|
||||
esac; \
|
||||
fi; \
|
||||
# The public crate CDN occasionally rejects a burst of parallel fetches
|
||||
# from a BuildKit worker. Keep the release build resilient without
|
||||
# changing the produced binaries or their dependency graph.
|
||||
export \
|
||||
CARGO_NET_RETRY=10 \
|
||||
CARGO_HTTP_TIMEOUT=120 \
|
||||
CARGO_HTTP_MULTIPLEXING=false; \
|
||||
cargo build --release --target "$target_triple" \
|
||||
--bin rpki \
|
||||
--bin rpki_daemon \
|
||||
--bin db_stats; \
|
||||
--bin db_stats \
|
||||
--bin rpki_der_ber_audit; \
|
||||
mkdir -p /build-out/bin; \
|
||||
cp \
|
||||
"target/$target_triple/release/rpki" \
|
||||
"target/$target_triple/release/rpki_daemon" \
|
||||
"target/$target_triple/release/db_stats" \
|
||||
"target/$target_triple/release/rpki_der_ber_audit" \
|
||||
/build-out/bin/
|
||||
|
||||
FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS runtime
|
||||
|
||||
@ -14,6 +14,10 @@ BUILDER_NAME="${BUILDER_NAME:-default}"
|
||||
INSTALL_BINFMT="${INSTALL_BINFMT:-1}"
|
||||
SAVE_IMAGE="${SAVE_IMAGE:-1}"
|
||||
LOAD_IMAGE="${LOAD_IMAGE:-1}"
|
||||
# BuildKit's default bridge network can differ from the Docker daemon proxy
|
||||
# configuration. Set BUILD_NETWORK=host when the build container needs the
|
||||
# host's known-good registry connectivity.
|
||||
BUILD_NETWORK="${BUILD_NETWORK:-default}"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
@ -216,6 +220,7 @@ echo "builder_source_image: $builder_source_image"
|
||||
echo "runtime_source_image: $runtime_source_image"
|
||||
echo "builder_local_image: $BUILDER_IMAGE"
|
||||
echo "runtime_local_image: $RUNTIME_IMAGE"
|
||||
echo "build_network: $BUILD_NETWORK"
|
||||
echo "source_commit: $SOURCE_COMMIT_FULL"
|
||||
echo "source_dirty: $SOURCE_DIRTY"
|
||||
echo "build_timestamp_utc: $BUILD_TIMESTAMP_UTC"
|
||||
@ -225,6 +230,7 @@ if [[ "$LOAD_IMAGE" == "1" ]]; then
|
||||
docker buildx build \
|
||||
--platform "$target_platform" \
|
||||
--builder "$BUILDER_NAME" \
|
||||
--network "$BUILD_NETWORK" \
|
||||
--pull=false \
|
||||
--load \
|
||||
--build-arg BUILDKIT_INLINE_CACHE=1 \
|
||||
@ -242,6 +248,7 @@ else
|
||||
docker buildx build \
|
||||
--platform "$target_platform" \
|
||||
--builder "$BUILDER_NAME" \
|
||||
--network "$BUILD_NETWORK" \
|
||||
--pull=false \
|
||||
--output "type=docker,dest=$raw_tar_path" \
|
||||
--build-arg BUILDKIT_INLINE_CACHE=1 \
|
||||
|
||||
11
src/audit.rs
11
src/audit.rs
@ -36,6 +36,7 @@ pub struct ObjectAuditEntry {
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
pub struct AuditWarning {
|
||||
pub message: String,
|
||||
pub category: String,
|
||||
pub rfc_refs: Vec<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub context: Option<String>,
|
||||
@ -102,6 +103,7 @@ impl From<&crate::report::Warning> for AuditWarning {
|
||||
fn from(w: &crate::report::Warning) -> Self {
|
||||
Self {
|
||||
message: w.message.clone(),
|
||||
category: w.category.as_str().to_string(),
|
||||
rfc_refs: w.rfc_refs.iter().map(|r| r.0.to_string()).collect(),
|
||||
context: w.context.clone(),
|
||||
}
|
||||
@ -238,6 +240,15 @@ mod tests {
|
||||
assert!(value.get("cir_fresh_objects").is_none());
|
||||
assert!(value.get("cir_cached_objects").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warning_audit_keeps_the_warning_category() {
|
||||
let warning = crate::report::Warning::new("BER-compatible CMS accepted")
|
||||
.with_category(crate::report::WarningCategory::BerCompatibleCmsEncoding);
|
||||
|
||||
let audit_warning = AuditWarning::from(&warning);
|
||||
assert_eq!(audit_warning.category, "ber_compatible_cms_encoding");
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
|
||||
|
||||
437
src/bin/rpki_der_ber_audit.rs
Normal file
437
src/bin/rpki_der_ber_audit.rs
Normal file
@ -0,0 +1,437 @@
|
||||
use rocksdb::{DB, IteratorMode, Options};
|
||||
use rpki::data_model::crl::RpkixCrl;
|
||||
use rpki::data_model::rc::ResourceCertificate;
|
||||
use rpki::data_model::signed_object::RpkiSignedObject;
|
||||
use serde::Serialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Args {
|
||||
input: Option<PathBuf>,
|
||||
repo_bytes_db: Option<PathBuf>,
|
||||
output: Option<PathBuf>,
|
||||
pretty: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum EncodingStatus {
|
||||
Der,
|
||||
BerCompatibleOnly,
|
||||
ParseFailed,
|
||||
NotRpki,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
struct ObjectRecord {
|
||||
sha256: String,
|
||||
object_type: String,
|
||||
encoding_status: EncodingStatus,
|
||||
bytes: u64,
|
||||
path: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
detail: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct AuditReport {
|
||||
generated_by: &'static str,
|
||||
input: String,
|
||||
files_seen: u64,
|
||||
recognized_rpki_objects: u64,
|
||||
duplicate_content_files: u64,
|
||||
non_rpki_entries: u64,
|
||||
scan_errors: Vec<String>,
|
||||
status_counts: BTreeMap<String, u64>,
|
||||
object_type_status_counts: BTreeMap<String, BTreeMap<String, u64>>,
|
||||
objects: Vec<ObjectRecord>,
|
||||
}
|
||||
|
||||
fn usage() -> &'static str {
|
||||
"Usage: rpki_der_ber_audit (--input <raw-rpki-object-directory>|--repo-bytes-db <repo-bytes.db>) [--output <audit.json>] [--compact]"
|
||||
}
|
||||
|
||||
fn main() {
|
||||
if let Err(err) = real_main() {
|
||||
eprintln!("{err}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn real_main() -> Result<(), String> {
|
||||
let args = parse_args(&std::env::args().collect::<Vec<_>>())?;
|
||||
let input_label = args
|
||||
.input
|
||||
.as_ref()
|
||||
.or(args.repo_bytes_db.as_ref())
|
||||
.expect("validated input");
|
||||
let mut report = AuditReport {
|
||||
generated_by: "rpki_der_ber_audit",
|
||||
input: input_label.display().to_string(),
|
||||
files_seen: 0,
|
||||
recognized_rpki_objects: 0,
|
||||
duplicate_content_files: 0,
|
||||
non_rpki_entries: 0,
|
||||
scan_errors: Vec::new(),
|
||||
status_counts: BTreeMap::new(),
|
||||
object_type_status_counts: BTreeMap::new(),
|
||||
objects: Vec::new(),
|
||||
};
|
||||
let mut seen_hashes = BTreeSet::new();
|
||||
if let Some(input) = args.input.as_ref() {
|
||||
scan_directory(input, &mut report, &mut seen_hashes)?;
|
||||
} else if let Some(repo_bytes_db) = args.repo_bytes_db.as_ref() {
|
||||
scan_repo_bytes_db(repo_bytes_db, &mut report, &mut seen_hashes)?;
|
||||
}
|
||||
report
|
||||
.objects
|
||||
.sort_by(|left, right| left.sha256.cmp(&right.sha256));
|
||||
let rendered = if args.pretty {
|
||||
serde_json::to_string_pretty(&report)
|
||||
} else {
|
||||
serde_json::to_string(&report)
|
||||
}
|
||||
.map_err(|err| format!("serialize audit report: {err}"))?;
|
||||
|
||||
if let Some(output) = args.output {
|
||||
if let Some(parent) = output.parent() {
|
||||
std::fs::create_dir_all(parent)
|
||||
.map_err(|err| format!("create output directory {}: {err}", parent.display()))?;
|
||||
}
|
||||
std::fs::write(&output, rendered)
|
||||
.map_err(|err| format!("write audit report {}: {err}", output.display()))?;
|
||||
} else {
|
||||
println!("{rendered}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_args(argv: &[String]) -> Result<Args, String> {
|
||||
let mut args = Args {
|
||||
pretty: true,
|
||||
..Args::default()
|
||||
};
|
||||
let mut index = 1usize;
|
||||
while index < argv.len() {
|
||||
match argv[index].as_str() {
|
||||
"--input" | "--in" => {
|
||||
index += 1;
|
||||
args.input = Some(PathBuf::from(
|
||||
argv.get(index).ok_or("--input requires a value")?,
|
||||
));
|
||||
}
|
||||
"--repo-bytes-db" => {
|
||||
index += 1;
|
||||
args.repo_bytes_db = Some(PathBuf::from(
|
||||
argv.get(index).ok_or("--repo-bytes-db requires a value")?,
|
||||
));
|
||||
}
|
||||
"--output" | "--out" => {
|
||||
index += 1;
|
||||
args.output = Some(PathBuf::from(
|
||||
argv.get(index).ok_or("--output requires a value")?,
|
||||
));
|
||||
}
|
||||
"--compact" => args.pretty = false,
|
||||
"--pretty" => args.pretty = true,
|
||||
"-h" | "--help" => return Err(usage().to_string()),
|
||||
other => return Err(format!("unknown argument: {other}\n{}", usage())),
|
||||
}
|
||||
index += 1;
|
||||
}
|
||||
if args.input.is_some() == args.repo_bytes_db.is_some() {
|
||||
return Err(format!("select exactly one input source\n{}", usage()));
|
||||
}
|
||||
if let Some(input) = args.input.as_ref()
|
||||
&& !input.is_dir()
|
||||
{
|
||||
return Err(format!("--input must be a directory: {}", input.display()));
|
||||
}
|
||||
if let Some(repo_bytes_db) = args.repo_bytes_db.as_ref()
|
||||
&& !repo_bytes_db.is_dir()
|
||||
{
|
||||
return Err(format!(
|
||||
"--repo-bytes-db must be a RocksDB directory: {}",
|
||||
repo_bytes_db.display()
|
||||
));
|
||||
}
|
||||
Ok(args)
|
||||
}
|
||||
|
||||
fn scan_directory(
|
||||
directory: &Path,
|
||||
report: &mut AuditReport,
|
||||
seen_hashes: &mut BTreeSet<String>,
|
||||
) -> Result<(), String> {
|
||||
let entries = std::fs::read_dir(directory)
|
||||
.map_err(|err| format!("read directory {}: {err}", directory.display()))?;
|
||||
for entry in entries {
|
||||
let entry = match entry {
|
||||
Ok(entry) => entry,
|
||||
Err(err) => {
|
||||
report.scan_errors.push(format!(
|
||||
"read directory entry under {}: {err}",
|
||||
directory.display()
|
||||
));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let path = entry.path();
|
||||
let file_type = match entry.file_type() {
|
||||
Ok(file_type) => file_type,
|
||||
Err(err) => {
|
||||
report
|
||||
.scan_errors
|
||||
.push(format!("read file type {}: {err}", path.display()));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if file_type.is_dir() {
|
||||
scan_directory(&path, report, seen_hashes)?;
|
||||
} else if file_type.is_file() {
|
||||
scan_file(&path, report, seen_hashes);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn scan_file(path: &Path, report: &mut AuditReport, seen_hashes: &mut BTreeSet<String>) {
|
||||
report.files_seen += 1;
|
||||
let Some(object_type) = object_type_from_path(path) else {
|
||||
return;
|
||||
};
|
||||
let bytes = match std::fs::read(path) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(err) => {
|
||||
report
|
||||
.scan_errors
|
||||
.push(format!("read {}: {err}", path.display()));
|
||||
return;
|
||||
}
|
||||
};
|
||||
let sha256 = hex::encode(Sha256::digest(&bytes));
|
||||
let (encoding_status, detail) = classify_object(&object_type, &bytes);
|
||||
record_object(
|
||||
report,
|
||||
seen_hashes,
|
||||
sha256,
|
||||
object_type,
|
||||
encoding_status,
|
||||
bytes.len() as u64,
|
||||
path.display().to_string(),
|
||||
detail,
|
||||
);
|
||||
}
|
||||
|
||||
fn scan_repo_bytes_db(
|
||||
repo_bytes_db: &Path,
|
||||
report: &mut AuditReport,
|
||||
seen_hashes: &mut BTreeSet<String>,
|
||||
) -> Result<(), String> {
|
||||
let mut options = Options::default();
|
||||
options.create_if_missing(false);
|
||||
let db = DB::open_for_read_only(&options, repo_bytes_db, false)
|
||||
.map_err(|err| format!("open repo-bytes DB {}: {err}", repo_bytes_db.display()))?;
|
||||
for entry in db.iterator(IteratorMode::Start) {
|
||||
let (key, bytes) = entry.map_err(|err| format!("iterate repo-bytes DB: {err}"))?;
|
||||
let Ok(key) = std::str::from_utf8(&key) else {
|
||||
continue;
|
||||
};
|
||||
let Some(sha256) = key.strip_prefix("sha256:") else {
|
||||
continue;
|
||||
};
|
||||
report.files_seen += 1;
|
||||
let (object_type, encoding_status, detail) = classify_repo_blob(&bytes);
|
||||
record_object(
|
||||
report,
|
||||
seen_hashes,
|
||||
sha256.to_string(),
|
||||
object_type,
|
||||
encoding_status,
|
||||
bytes.len() as u64,
|
||||
format!("rocksdb://{}/{}", repo_bytes_db.display(), key),
|
||||
detail,
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn record_object(
|
||||
report: &mut AuditReport,
|
||||
seen_hashes: &mut BTreeSet<String>,
|
||||
sha256: String,
|
||||
object_type: String,
|
||||
encoding_status: EncodingStatus,
|
||||
bytes: u64,
|
||||
path: String,
|
||||
detail: Option<String>,
|
||||
) {
|
||||
if matches!(encoding_status, EncodingStatus::NotRpki) {
|
||||
report.non_rpki_entries += 1;
|
||||
} else {
|
||||
report.recognized_rpki_objects += 1;
|
||||
}
|
||||
if !seen_hashes.insert(sha256.clone()) {
|
||||
report.duplicate_content_files += 1;
|
||||
return;
|
||||
}
|
||||
let status_name = encoding_status_name(&encoding_status).to_string();
|
||||
*report.status_counts.entry(status_name.clone()).or_default() += 1;
|
||||
*report
|
||||
.object_type_status_counts
|
||||
.entry(object_type.clone())
|
||||
.or_default()
|
||||
.entry(status_name)
|
||||
.or_default() += 1;
|
||||
report.objects.push(ObjectRecord {
|
||||
sha256,
|
||||
object_type,
|
||||
encoding_status,
|
||||
bytes,
|
||||
path,
|
||||
detail,
|
||||
});
|
||||
}
|
||||
|
||||
fn object_type_from_path(path: &Path) -> Option<String> {
|
||||
let extension = path.extension()?.to_str()?.to_ascii_lowercase();
|
||||
match extension.as_str() {
|
||||
"mft" | "roa" | "asa" | "aspa" | "gbr" | "cer" | "crl" => Some(extension),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_object(object_type: &str, bytes: &[u8]) -> (EncodingStatus, Option<String>) {
|
||||
match object_type {
|
||||
"mft" | "roa" | "asa" | "aspa" | "gbr" => match RpkiSignedObject::parse_der(bytes) {
|
||||
Ok(_) => match RpkiSignedObject::strict_cms_der_error(bytes) {
|
||||
Some(err) => (EncodingStatus::BerCompatibleOnly, Some(err.to_string())),
|
||||
None => (EncodingStatus::Der, None),
|
||||
},
|
||||
Err(err) => (EncodingStatus::ParseFailed, Some(err.to_string())),
|
||||
},
|
||||
"cer" => match ResourceCertificate::parse_der(bytes) {
|
||||
Ok(_) => (EncodingStatus::Der, None),
|
||||
Err(err) => (EncodingStatus::ParseFailed, Some(err.to_string())),
|
||||
},
|
||||
"crl" => match RpkixCrl::parse_der(bytes) {
|
||||
Ok(_) => (EncodingStatus::Der, None),
|
||||
Err(err) => (EncodingStatus::ParseFailed, Some(err.to_string())),
|
||||
},
|
||||
_ => unreachable!("object type was filtered before classification"),
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_repo_blob(bytes: &[u8]) -> (String, EncodingStatus, Option<String>) {
|
||||
if looks_like_tal_metadata(bytes) {
|
||||
return (
|
||||
"tal".to_string(),
|
||||
EncodingStatus::NotRpki,
|
||||
Some("TAL text metadata, not an RPKI object".to_string()),
|
||||
);
|
||||
}
|
||||
if RpkiSignedObject::parse_der(bytes).is_ok() {
|
||||
return match RpkiSignedObject::strict_cms_der_error(bytes) {
|
||||
Some(err) => (
|
||||
"cms_signed_object".to_string(),
|
||||
EncodingStatus::BerCompatibleOnly,
|
||||
Some(err.to_string()),
|
||||
),
|
||||
None => ("cms_signed_object".to_string(), EncodingStatus::Der, None),
|
||||
};
|
||||
}
|
||||
if ResourceCertificate::parse_der(bytes).is_ok() {
|
||||
return ("cer".to_string(), EncodingStatus::Der, None);
|
||||
}
|
||||
if RpkixCrl::parse_der(bytes).is_ok() {
|
||||
return ("crl".to_string(), EncodingStatus::Der, None);
|
||||
}
|
||||
(
|
||||
"unknown".to_string(),
|
||||
EncodingStatus::ParseFailed,
|
||||
Some("not parsed as CMS, RPKI resource certificate, or CRL".to_string()),
|
||||
)
|
||||
}
|
||||
|
||||
fn encoding_status_name(status: &EncodingStatus) -> &'static str {
|
||||
match status {
|
||||
EncodingStatus::Der => "der",
|
||||
EncodingStatus::BerCompatibleOnly => "ber_compatible_only",
|
||||
EncodingStatus::ParseFailed => "parse_failed",
|
||||
EncodingStatus::NotRpki => "not_rpki",
|
||||
}
|
||||
}
|
||||
|
||||
/// Return true for the TAL text format persisted by the live-TA refresh.
|
||||
///
|
||||
/// A TAL is intentionally not DER: it contains an rsync URI, an HTTPS URI,
|
||||
/// a blank separator, and the base64-encoded trust-anchor certificate. It
|
||||
/// can share the repo-bytes DB with downloaded RPKI objects, so the audit must
|
||||
/// keep it out of the object encoding verdict rather than report a false
|
||||
/// parse failure.
|
||||
fn looks_like_tal_metadata(bytes: &[u8]) -> bool {
|
||||
let Ok(text) = std::str::from_utf8(bytes) else {
|
||||
return false;
|
||||
};
|
||||
let mut lines = text.lines();
|
||||
let Some(rsync_uri) = lines.next() else {
|
||||
return false;
|
||||
};
|
||||
let Some(https_uri) = lines.next() else {
|
||||
return false;
|
||||
};
|
||||
if !rsync_uri.starts_with("rsync://") || !https_uri.starts_with("https://") {
|
||||
return false;
|
||||
}
|
||||
// `str::lines` removes the blank line, so inspect the raw prefix to make
|
||||
// sure the URI pair is followed by the TAL separator.
|
||||
let Some(separator) = text.find("\n\n") else {
|
||||
return false;
|
||||
};
|
||||
let base64 = &text[separator + 2..];
|
||||
!base64.is_empty()
|
||||
&& base64.bytes().all(|byte| {
|
||||
byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'/' | b'=' | b'\n' | b'\r')
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn classifies_known_der_manifest_fixture() {
|
||||
let path = Path::new(env!("CARGO_MANIFEST_DIR")).join(
|
||||
"tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.mft",
|
||||
);
|
||||
let bytes = std::fs::read(path).expect("read manifest fixture");
|
||||
assert!(matches!(
|
||||
classify_object("mft", &bytes).0,
|
||||
EncodingStatus::Der
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recognizes_supported_object_extensions() {
|
||||
assert_eq!(
|
||||
object_type_from_path(Path::new("a.roa")),
|
||||
Some("roa".to_string())
|
||||
);
|
||||
assert_eq!(object_type_from_path(Path::new("a.txt")), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_tal_metadata_as_not_rpki() {
|
||||
let tal = b"rsync://rpki.example/ta.cer\nhttps://rpki.example/ta.cer\n\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A\n";
|
||||
let (object_type, status, detail) = classify_repo_blob(tal);
|
||||
assert_eq!(object_type, "tal");
|
||||
assert!(matches!(status, EncodingStatus::NotRpki));
|
||||
assert_eq!(
|
||||
detail.as_deref(),
|
||||
Some("TAL text metadata, not an RPKI object")
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -330,6 +330,16 @@ impl RpkiSignedObject {
|
||||
parse_signed_object_content_info(der, der, CmsParseMode::DerStrict)
|
||||
}
|
||||
|
||||
/// Return the strict-DER CMS parse error for an object that was otherwise
|
||||
/// accepted through the normal BER-compatible CMS parser.
|
||||
///
|
||||
/// Callers must only surface this as a compatibility warning after normal
|
||||
/// decoding and validation have succeeded; a strict parse failure alone
|
||||
/// does not prove that an arbitrary byte string is an RPKI signed object.
|
||||
pub fn strict_cms_der_error(der: &[u8]) -> Option<SignedObjectParseError> {
|
||||
Self::parse_der_strict_cms(der).err()
|
||||
}
|
||||
|
||||
/// Decode a DER-encoded RPKI Signed Object (CMS ContentInfo wrapping SignedData) and enforce
|
||||
/// the profile constraints from RFC 6488 §2-§3 and RFC 9589 §4.
|
||||
pub fn decode_der(der: &[u8]) -> Result<Self, SignedObjectDecodeError> {
|
||||
@ -1541,6 +1551,10 @@ mod tests {
|
||||
let err = RpkiSignedObject::parse_der_strict_cms(&mutated)
|
||||
.expect_err("DER strict rejects constructed OCTET STRING");
|
||||
assert!(err.to_string().contains("DER"), "{err}");
|
||||
let compatibility_error = RpkiSignedObject::strict_cms_der_error(&mutated)
|
||||
.expect("BER-compatible fixture must report strict-DER incompatibility");
|
||||
assert_eq!(compatibility_error.to_string(), err.to_string());
|
||||
assert!(RpkiSignedObject::strict_cms_der_error(&der).is_none());
|
||||
}
|
||||
|
||||
fn replace_first_subslice(input: &[u8], from: &[u8], to: &[u8]) -> Option<Vec<u8>> {
|
||||
|
||||
@ -1,9 +1,31 @@
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct RfcRef(pub &'static str);
|
||||
|
||||
/// Stable categories for warnings written to audit reports and exported metrics.
|
||||
///
|
||||
/// New warning sites should use a specific category when one is available. The
|
||||
/// default preserves the behaviour and schema of existing callers while making
|
||||
/// their metric label explicit.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum WarningCategory {
|
||||
#[default]
|
||||
Unclassified,
|
||||
BerCompatibleCmsEncoding,
|
||||
}
|
||||
|
||||
impl WarningCategory {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Unclassified => "unclassified",
|
||||
Self::BerCompatibleCmsEncoding => "ber_compatible_cms_encoding",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Warning {
|
||||
pub message: String,
|
||||
pub category: WarningCategory,
|
||||
pub rfc_refs: Vec<RfcRef>,
|
||||
pub context: Option<String>,
|
||||
}
|
||||
@ -12,11 +34,17 @@ impl Warning {
|
||||
pub fn new(message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
message: message.into(),
|
||||
category: WarningCategory::Unclassified,
|
||||
rfc_refs: Vec::new(),
|
||||
context: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_category(mut self, category: WarningCategory) -> Self {
|
||||
self.category = category;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_rfc_refs(mut self, refs: &[RfcRef]) -> Self {
|
||||
self.rfc_refs.extend_from_slice(refs);
|
||||
self
|
||||
|
||||
@ -424,6 +424,7 @@ struct LatestRunMetrics {
|
||||
vaps: u64,
|
||||
publication_points: u64,
|
||||
warnings: u64,
|
||||
warning_categories: BTreeMap<String, u64>,
|
||||
tree_instances_processed: Option<u64>,
|
||||
tree_instances_failed: Option<u64>,
|
||||
stage_seconds: BTreeMap<String, f64>,
|
||||
@ -1477,13 +1478,9 @@ fn parse_report(path: &Path, snapshot: &mut MetricsSnapshot, latest: &mut Latest
|
||||
.map(|a| a.len() as u64)
|
||||
.unwrap_or(0);
|
||||
}
|
||||
latest.warnings = latest.warnings.max(
|
||||
report
|
||||
.pointer("/tree/warnings")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|a| a.len() as u64)
|
||||
.unwrap_or(0),
|
||||
);
|
||||
let (reported_warning_count, warning_categories) = collect_warning_categories(&report);
|
||||
latest.warnings = latest.warnings.max(reported_warning_count);
|
||||
latest.warning_categories = warning_categories;
|
||||
if let Some(processed) = json_u64(&report, &["tree", "instances_processed"]) {
|
||||
latest.tree_instances_processed = Some(processed);
|
||||
}
|
||||
@ -1504,6 +1501,47 @@ fn parse_report(path: &Path, snapshot: &mut MetricsSnapshot, latest: &mut Latest
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_warning_categories(report: &Value) -> (u64, BTreeMap<String, u64>) {
|
||||
let mut total = 0u64;
|
||||
let mut categories = BTreeMap::new();
|
||||
total += count_warning_categories(
|
||||
report.pointer("/tree/warnings").and_then(Value::as_array),
|
||||
&mut categories,
|
||||
);
|
||||
if let Some(publication_points) = report.get("publication_points").and_then(Value::as_array) {
|
||||
for publication_point in publication_points {
|
||||
total += count_warning_categories(
|
||||
publication_point.get("warnings").and_then(Value::as_array),
|
||||
&mut categories,
|
||||
);
|
||||
}
|
||||
}
|
||||
// Export the baseline and BER-compatibility series even when their count is
|
||||
// zero, so dashboards can distinguish a clean run from a missing metric.
|
||||
for category in ["unclassified", "ber_compatible_cms_encoding"] {
|
||||
categories.entry(category.to_string()).or_insert(0);
|
||||
}
|
||||
(total, categories)
|
||||
}
|
||||
|
||||
fn count_warning_categories(
|
||||
warnings: Option<&Vec<Value>>,
|
||||
categories: &mut BTreeMap<String, u64>,
|
||||
) -> u64 {
|
||||
let Some(warnings) = warnings else {
|
||||
return 0;
|
||||
};
|
||||
for warning in warnings {
|
||||
let category = warning
|
||||
.get("category")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or("unclassified");
|
||||
*categories.entry(category.to_string()).or_default() += 1;
|
||||
}
|
||||
warnings.len() as u64
|
||||
}
|
||||
|
||||
fn extract_publication_point_metrics(
|
||||
pps: &[Value],
|
||||
downloads: Option<&Value>,
|
||||
@ -2409,6 +2447,17 @@ fn render_latest_metrics(writer: &mut PromWriter<'_>, instance: &str, latest: &L
|
||||
&[label("instance", instance)],
|
||||
latest.warnings as f64,
|
||||
);
|
||||
for (category, count) in &latest.warning_categories {
|
||||
writer.gauge(
|
||||
"ours_rp_warnings_by_category",
|
||||
"Latest run warning count by category",
|
||||
&[
|
||||
label("instance", instance),
|
||||
label("warning_category", category),
|
||||
],
|
||||
*count as f64,
|
||||
);
|
||||
}
|
||||
if let Some(value) = latest.tree_instances_processed {
|
||||
writer.gauge(
|
||||
"ours_rp_tree_instances",
|
||||
@ -3364,6 +3413,48 @@ mod tests {
|
||||
assert_eq!(args.out_metrics.as_deref(), Some(Path::new("metrics.prom")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warning_categories_cover_tree_and_publication_point_warnings() {
|
||||
let report = json!({
|
||||
"tree": {
|
||||
"warnings": [
|
||||
{"category": "ber_compatible_cms_encoding"},
|
||||
{"category": "ber_compatible_cms_encoding"}
|
||||
]
|
||||
},
|
||||
"publication_points": [
|
||||
{"warnings": [{"category": "repository_sync"}]},
|
||||
{"warnings": [{"message": "legacy warning without category"}]}
|
||||
]
|
||||
});
|
||||
|
||||
let (total, categories) = collect_warning_categories(&report);
|
||||
assert_eq!(total, 4);
|
||||
assert_eq!(categories["ber_compatible_cms_encoding"], 2);
|
||||
assert_eq!(categories["repository_sync"], 1);
|
||||
assert_eq!(categories["unclassified"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_warning_category_metrics() {
|
||||
let mut warning_categories = BTreeMap::new();
|
||||
warning_categories.insert("ber_compatible_cms_encoding".to_string(), 2);
|
||||
let snapshot = MetricsSnapshot {
|
||||
instance: "test".to_string(),
|
||||
latest_run: Some(LatestRunMetrics {
|
||||
warnings: 2,
|
||||
warning_categories,
|
||||
..LatestRunMetrics::default()
|
||||
}),
|
||||
..MetricsSnapshot::default()
|
||||
};
|
||||
|
||||
let metrics = render_metrics(&snapshot);
|
||||
assert!(metrics.contains(
|
||||
r#"ours_rp_warnings_by_category{instance="test",warning_category="ber_compatible_cms_encoding"} 2"#
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scan_fixture_exports_repo_pp_cir_and_ccr_metrics() {
|
||||
let td = TempDir::new().expect("tempdir");
|
||||
|
||||
@ -8,13 +8,13 @@ use crate::data_model::rc::{
|
||||
ResourceCertificate,
|
||||
};
|
||||
use crate::data_model::roa::{IpPrefix, RoaAfi, RoaDecodeError, RoaObject, RoaValidateError};
|
||||
use crate::data_model::signed_object::SignedObjectVerifyError;
|
||||
use crate::data_model::signed_object::{RpkiSignedObject, SignedObjectVerifyError};
|
||||
use crate::parallel::config::ParallelPhase2Config;
|
||||
use crate::parallel::object_worker::{
|
||||
ObjectTaskExecutor, ObjectWorkerPool, ObjectWorkerSubmitError,
|
||||
};
|
||||
use crate::policy::{Policy, ResourceValidationMode, SignedObjectFailurePolicy};
|
||||
use crate::report::{RfcRef, Warning};
|
||||
use crate::report::{RfcRef, Warning, WarningCategory};
|
||||
use crate::storage::{
|
||||
PackFile, PackTime, RoaCacheObjectMeta, RoaCacheProjection, VcirLocalOutput,
|
||||
VcirLocalOutputPayload, VcirOutputType, VcirSourceObjectType,
|
||||
@ -34,6 +34,22 @@ const RFC_CRLDP: &[RfcRef] = &[RfcRef("RFC 6487 §4.8.6")];
|
||||
const RFC_CRLDP_AND_LOCKED_PACK: &[RfcRef] =
|
||||
&[RfcRef("RFC 6487 §4.8.6"), RfcRef("RFC 9286 §4.2.1")];
|
||||
|
||||
fn ber_compatible_cms_warning(der: &[u8], rsync_uri: &str, object_kind: &str) -> Option<Warning> {
|
||||
let strict_error = RpkiSignedObject::strict_cms_der_error(der)?;
|
||||
Some(
|
||||
Warning::new(format!(
|
||||
"accepted BER-compatible CMS encoding for {object_kind}: {rsync_uri}: {strict_error}"
|
||||
))
|
||||
.with_category(WarningCategory::BerCompatibleCmsEncoding)
|
||||
.with_rfc_refs(&[RfcRef("X.690 §10"), RfcRef("RFC 6488 §2")])
|
||||
.with_context(rsync_uri),
|
||||
)
|
||||
}
|
||||
|
||||
fn ber_compatible_cms_warning_for_file(file: &PackFile, object_kind: &str) -> Option<Warning> {
|
||||
ber_compatible_cms_warning(file.bytes().ok()?, &file.rsync_uri, object_kind)
|
||||
}
|
||||
|
||||
fn sha256_hex_to_32(hex_value: &str) -> [u8; 32] {
|
||||
let bytes = hex::decode(hex_value).expect("internal sha256 hex should decode");
|
||||
let mut out = [0u8; 32];
|
||||
@ -1015,6 +1031,12 @@ pub fn process_publication_point_for_issuer_with_cache_options<P: PublicationPoi
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(warning) =
|
||||
ber_compatible_cms_warning(manifest_bytes, manifest_rsync_uri, "manifest")
|
||||
{
|
||||
warnings.push(warning);
|
||||
}
|
||||
|
||||
// Decode issuer CA once; if it fails we cannot validate ROA/ASPA EE certificates.
|
||||
let issuer_ca = match decode_resource_certificate_with_policy(issuer_ca_der, policy) {
|
||||
Ok(v) => v,
|
||||
@ -1317,6 +1339,9 @@ pub fn process_publication_point_for_issuer_with_cache_options<P: PublicationPoi
|
||||
result: AuditObjectResult::Ok,
|
||||
detail: None,
|
||||
});
|
||||
if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ROA") {
|
||||
warnings.push(warning);
|
||||
}
|
||||
}
|
||||
Err(e) => match policy.signed_object_failure_policy {
|
||||
SignedObjectFailurePolicy::DropObject => {
|
||||
@ -1426,6 +1451,9 @@ pub fn process_publication_point_for_issuer_with_cache_options<P: PublicationPoi
|
||||
result: AuditObjectResult::Ok,
|
||||
detail: None,
|
||||
});
|
||||
if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ASPA") {
|
||||
warnings.push(warning);
|
||||
}
|
||||
}
|
||||
Err(e) => match policy.signed_object_failure_policy {
|
||||
SignedObjectFailurePolicy::DropObject => {
|
||||
@ -2050,6 +2078,12 @@ pub(crate) fn prepare_publication_point_for_parallel_roa_with_cache<P: Publicati
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(warning) =
|
||||
ber_compatible_cms_warning(manifest_bytes, manifest_rsync_uri, "manifest")
|
||||
{
|
||||
warnings.push(warning);
|
||||
}
|
||||
|
||||
let issuer_ca = match decode_resource_certificate_with_policy(issuer_ca_der, policy) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
@ -2367,6 +2401,9 @@ pub(crate) fn reduce_parallel_roa_stage(
|
||||
result: AuditObjectResult::Ok,
|
||||
detail: None,
|
||||
});
|
||||
if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ROA") {
|
||||
warnings.push(warning);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
audit.push(ObjectAuditEntry {
|
||||
@ -2418,6 +2455,9 @@ pub(crate) fn reduce_parallel_roa_stage(
|
||||
result: AuditObjectResult::Ok,
|
||||
detail: None,
|
||||
});
|
||||
if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ASPA") {
|
||||
warnings.push(warning);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
audit.push(ObjectAuditEntry {
|
||||
|
||||
@ -3013,6 +3013,7 @@ mod tests {
|
||||
});
|
||||
sample.audit.warnings.push(crate::audit::AuditWarning {
|
||||
message: "warning".to_string(),
|
||||
category: "unclassified".to_string(),
|
||||
rfc_refs: Vec::new(),
|
||||
context: None,
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user