NASP/rpki
64
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260616 增加产品UI查询服务

Browse Source
This commit is contained in:
yuyr 2026-06-17 10:16:04 +08:00
parent 4e6bd687db
commit 6ef2c98890
20 changed files with 7526 additions and 426 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.toml
View File

@ -21,7 +21,7 @@ ring = "0.17.14"
x509-parser = { version = "0.18.0", features = ["verify"] }
url = "2.5.8"
serde = { version = "1.0.218", features = ["derive"] }
serde_json = "1.0.140"
serde_json = { version = "1.0.140", features = ["raw_value"] }
toml = "0.8.20"
rocksdb = { version = "0.22.0", optional = true, default-features = false, features = ["lz4"] }
serde_cbor = "0.11.2"

2
scripts/coverage.sh
View File

@ -27,7 +27,7 @@ cleanup() {
}
trap cleanup EXIT
IGNORE_REGEX='repository_view_stats\.rs|db_stats\.rs|rrdp_state_dump\.rs|ccr_dump\.rs|ccr_verify\.rs|ccr_to_routinator_csv\.rs|ccr_to_compare_views\.rs|cir_materialize\.rs|cir_extract_inputs\.rs|cir_drop_report\.rs|cir_ta_only_fixture\.rs|cir_dump_reject_list\.rs|rpki_object_parse\.rs|triage_ccr_cir_pair\.rs|rpki_artifact_metrics|rpki_inter_rp_metrics|rpki_daemon\.rs|sequence_triage_ccr_cir|ccr_state_compare\.rs|cir_state_compare\.rs|cir_probe_rpki_client_cache\.rs|ccr/compare_view\.rs|progress_log\.rs|cli\.rs|validation/run_tree_from_tal\.rs|validation/tree_parallel\.rs|validation/tree_runner|validation/from_tal\.rs|sync/store_projection\.rs|sync/repo\.rs|sync/rrdp|(^|/)storage(/|\.rs$)|cir/materialize\.rs'
IGNORE_REGEX='repository_view_stats\.rs|db_stats\.rs|rrdp_state_dump\.rs|ccr_dump\.rs|ccr_verify\.rs|ccr_to_routinator_csv\.rs|ccr_to_compare_views\.rs|cir_materialize\.rs|cir_extract_inputs\.rs|cir_drop_report\.rs|cir_ta_only_fixture\.rs|cir_dump_reject_list\.rs|rpki_object_parse\.rs|rpki_query_indexer\.rs|rpki_query_service\.rs|triage_ccr_cir_pair\.rs|rpki_artifact_metrics|rpki_inter_rp_metrics|rpki_daemon\.rs|sequence_triage_ccr_cir|ccr_state_compare\.rs|cir_state_compare\.rs|cir_probe_rpki_client_cache\.rs|ccr/compare_view\.rs|progress_log\.rs|cli\.rs|validation/run_tree_from_tal\.rs|validation/tree_parallel\.rs|validation/tree_runner|validation/from_tal\.rs|sync/store_projection\.rs|sync/repo\.rs|sync/rrdp|(^|/)storage(/|\.rs$)|cir/materialize\.rs'
# Preserve colored output even though we post-process output by running under a pseudo-TTY.
# We run tests only once, then generate both CLI text + HTML reports without rerunning tests.

4
scripts/soak/build_portable_soak_package.sh
View File

@ -14,7 +14,7 @@ Usage:
scripts/soak/build_portable_soak_package.sh [--out-dir <path>] [--profile <profile>]
Requires release binaries to already exist. Build them first, for example:
cargo build --release --bin rpki --bin rpki_daemon --bin db_stats
cargo build --release --bin rpki --bin rpki_daemon --bin db_stats --bin rpki_artifact_metrics --bin rpki_query_service --bin rpki_query_indexer
USAGE
}
@ -53,7 +53,7 @@ else
TARGET_BIN_DIR="$REPO_ROOT/target/$PROFILE"
fi
REQUIRED_BINS=(rpki rpki_daemon db_stats rpki_artifact_metrics)
REQUIRED_BINS=(rpki rpki_daemon db_stats rpki_artifact_metrics rpki_query_service rpki_query_indexer)
OPTIONAL_BINS=(
ccr_dump
ccr_state_compare

19
scripts/soak/portable-soak.env.example
View File

@ -90,6 +90,25 @@ METRICS_LISTEN=0.0.0.0:9556
METRICS_POLL_SECS=5
METRICS_INSTANCE=remote231-24h
# 是否启动 query service。启动后默认只索引启动后新完成的 run，避免补历史 run 影响 RP 性能。
START_QUERY_SERVICE=0
# query service 退出策略。长期运行时通常保持 0，让服务持续可查。
STOP_QUERY_SERVICE_ON_EXIT=0
# query service 监听地址和索引保留策略。
QUERY_LISTEN=0.0.0.0:9560
QUERY_WATCH_INTERVAL_SECS=5
QUERY_RETAIN_INDEXED_RUNS=10
QUERY_PROJECTION_ENTRY_LIMIT=20
# 默认不设置最小 run seq，由 query service 自动从“启动时最新 run 的下一轮”开始跟踪。
# 如需实验性指定起点，可设置为具体数字，例如 QUERY_WATCH_MIN_RUN_SEQ=7208。
QUERY_WATCH_MIN_RUN_SEQ=
# 是否允许 query service 补索引历史 run。默认 0，避免大量历史 report.json 扫描和 query-db 写入拖慢 RP。
QUERY_WATCH_BACKFILL=0
# 是否启动 package 内置 Prometheus/Grafana monitor stack。
START_MONITOR_STACK=1

48
scripts/soak/run_24h_soak_with_metrics.sh
View File

@ -20,6 +20,7 @@ SOAK_SCRIPT="${SOAK_SCRIPT:-$PACKAGE_ROOT/run_soak.sh}"
HOURLY_REPORT_SCRIPT="${HOURLY_REPORT_SCRIPT:-$PACKAGE_ROOT/scripts/soak/hourly_soak_report.py}"
SOAK_DURATION_SECS="${SOAK_DURATION_SECS:-0}"
SOAK_INTERVAL_SECS="${SOAK_INTERVAL_SECS:-${INTERVAL_SECS:-0}}"
HOURLY_REPORT_INTERVAL_SECS="${HOURLY_REPORT_INTERVAL_SECS:-3600}"
SOAK_RETAIN_RUNS="${SOAK_RETAIN_RUNS:-100}"
CLEAN_TMP_AFTER_RUN="${CLEAN_TMP_AFTER_RUN:-1}"
@ -30,6 +31,17 @@ STOP_METRICS_SERVICE_ON_EXIT="${STOP_METRICS_SERVICE_ON_EXIT:-0}"
METRICS_LISTEN="${METRICS_LISTEN:-0.0.0.0:9556}"
METRICS_POLL_SECS="${METRICS_POLL_SECS:-5}"
METRICS_INSTANCE="${METRICS_INSTANCE:-remote231-24h}"
START_QUERY_SERVICE="${START_QUERY_SERVICE:-0}"
STOP_QUERY_SERVICE_ON_EXIT="${STOP_QUERY_SERVICE_ON_EXIT:-0}"
QUERY_LISTEN="${QUERY_LISTEN:-0.0.0.0:9560}"
QUERY_DB="${QUERY_DB:-$RUN_ROOT/state/query-db}"
QUERY_EXPORT_ROOT="${QUERY_EXPORT_ROOT:-$RUN_ROOT/state/query-exports}"
QUERY_REPO_BYTES_DB="${QUERY_REPO_BYTES_DB:-$RUN_ROOT/state/db/repo-bytes.db}"
QUERY_WATCH_INTERVAL_SECS="${QUERY_WATCH_INTERVAL_SECS:-5}"
QUERY_WATCH_MIN_RUN_SEQ="${QUERY_WATCH_MIN_RUN_SEQ:-}"
QUERY_WATCH_BACKFILL="${QUERY_WATCH_BACKFILL:-0}"
QUERY_RETAIN_INDEXED_RUNS="${QUERY_RETAIN_INDEXED_RUNS:-10}"
QUERY_PROJECTION_ENTRY_LIMIT="${QUERY_PROJECTION_ENTRY_LIMIT:-20}"
PROMETHEUS_RETENTION="${PROMETHEUS_RETENTION:-7d}"
SEND_FEISHU="${SEND_FEISHU:-1}"
FEISHU_DRY_RUN="${FEISHU_DRY_RUN:-0}"
@ -45,6 +57,7 @@ WARNING_MAX="${WARNING_MAX:--1}"
SOAK_PID=""
METRICS_PID=""
QUERY_PID=""
REPORTER_STOP=0
die() {
@ -75,6 +88,10 @@ cleanup() {
kill "$METRICS_PID" >/dev/null 2>&1 || true
wait "$METRICS_PID" >/dev/null 2>&1 || true
fi
if is_true "$STOP_QUERY_SERVICE_ON_EXIT" && [[ -n "$QUERY_PID" ]] && kill -0 "$QUERY_PID" >/dev/null 2>&1; then
kill "$QUERY_PID" >/dev/null 2>&1 || true
wait "$QUERY_PID" >/dev/null 2>&1 || true
fi
if is_true "$START_MONITOR_STACK" && is_true "$STOP_MONITOR_STACK_ON_EXIT" && [[ -f "$MONITOR_DIR/docker-compose.yml" ]]; then
(cd "$MONITOR_DIR" && PROMETHEUS_RETENTION="$PROMETHEUS_RETENTION" docker compose down) >/dev/null 2>&1 || true
fi
@ -116,6 +133,7 @@ format_epoch_rfc3339() {
main() {
validate_non_negative_int "SOAK_DURATION_SECS" "$SOAK_DURATION_SECS"
validate_non_negative_int "SOAK_INTERVAL_SECS" "$SOAK_INTERVAL_SECS"
validate_non_negative_int "HOURLY_REPORT_INTERVAL_SECS" "$HOURLY_REPORT_INTERVAL_SECS"
[[ "$HOURLY_REPORT_INTERVAL_SECS" != "0" ]] || die "HOURLY_REPORT_INTERVAL_SECS must be > 0"
[[ -x "$SOAK_SCRIPT" ]] || die "missing executable: $SOAK_SCRIPT"
@ -137,6 +155,34 @@ main() {
echo "$METRICS_PID" > "$LOG_ROOT/metrics.pid"
fi
if is_true "$START_QUERY_SERVICE"; then
[[ -x "$BIN_DIR/rpki_query_service" ]] || die "missing executable: $BIN_DIR/rpki_query_service"
[[ -x "$BIN_DIR/rpki_query_indexer" ]] || die "missing executable: $BIN_DIR/rpki_query_indexer"
mkdir -p "$QUERY_EXPORT_ROOT"
query_args=(
"$BIN_DIR/rpki_query_service"
--query-db "$QUERY_DB"
--repo-bytes-db "$QUERY_REPO_BYTES_DB"
--export-root "$QUERY_EXPORT_ROOT"
--listen "$QUERY_LISTEN"
--watch-run-root "$RUN_ROOT"
--watch-interval-secs "$QUERY_WATCH_INTERVAL_SECS"
--indexer-bin "$BIN_DIR/rpki_query_indexer"
--retain-indexed-runs "$QUERY_RETAIN_INDEXED_RUNS"
--projection-entry-limit "$QUERY_PROJECTION_ENTRY_LIMIT"
)
if [[ -n "$QUERY_WATCH_MIN_RUN_SEQ" ]]; then
query_args+=(--watch-min-run-seq "$QUERY_WATCH_MIN_RUN_SEQ")
fi
if is_true "$QUERY_WATCH_BACKFILL"; then
query_args+=(--watch-backfill)
fi
"${query_args[@]}" \
> "$LOG_ROOT/query-service.stdout" 2> "$LOG_ROOT/query-service.stderr" &
QUERY_PID="$!"
echo "$QUERY_PID" > "$LOG_ROOT/query-service.pid"
fi
if is_true "$START_MONITOR_STACK"; then
if [[ ! -f "$MONITOR_DIR/docker-compose.yml" ]]; then
die "missing monitor compose: $MONITOR_DIR/docker-compose.yml"
@ -169,7 +215,7 @@ main() {
fi
printf '\n# Generated by run_24h_soak_with_metrics.sh\n'
printf 'MAX_RUNS=-1\n'
printf 'INTERVAL_SECS=0\n'
printf 'INTERVAL_SECS=%q\n' "$SOAK_INTERVAL_SECS"
if (( SOAK_DURATION_SECS > 0 )); then
printf 'STOP_AFTER_SECS=%q\n' "$SOAK_DURATION_SECS"
else

59
src/audit.rs
View File

@ -41,6 +41,63 @@ pub struct AuditWarning {
pub context: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct QueryAuditManifest {
pub schema_version: u32,
pub status: String,
pub events_path: String,
pub events_count: u64,
pub events_sha256: String,
pub writer_version: u32,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ValidationEventCounts {
#[serde(skip_serializing_if = "Option::is_none")]
pub objects: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub warnings: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub vrps: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub aspas: Option<u64>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ValidationEvent {
pub schema_version: u32,
pub seq: u64,
pub event_type: String,
pub validation_time: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub pp_node_id: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub pp_manifest_uri: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub pp_rsync_base_uri: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub repo_sync_phase: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub repo_terminal_state: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub object_uri: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub sha256: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub object_type: Option<AuditObjectKind>,
#[serde(skip_serializing_if = "Option::is_none")]
pub result: Option<AuditObjectResult>,
#[serde(skip_serializing_if = "Option::is_none")]
pub reason: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub counts: Option<ValidationEventCounts>,
}
impl From<&crate::report::Warning> for AuditWarning {
fn from(w: &crate::report::Warning) -> Self {
Self {
@ -222,6 +279,8 @@ pub struct AuditReportV2 {
pub downloads: Vec<AuditDownloadEvent>,
pub download_stats: AuditDownloadStats,
pub repo_sync_stats: AuditRepoSyncStats,
#[serde(rename = "queryAudit", skip_serializing_if = "Option::is_none")]
pub query_audit: Option<QueryAuditManifest>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]

426
src/bin/rpki_object_parse.rs
View File

@ -1,29 +1,6 @@
use std::net::{Ipv4Addr, Ipv6Addr};
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use rpki::data_model::aspa::AspaObject;
use rpki::data_model::crl::RpkixCrl;
use rpki::data_model::manifest::ManifestObject;
use rpki::data_model::rc::{
AccessDescription, RcExtensions, ResourceCertificate, SubjectInfoAccess,
};
use rpki::data_model::roa::{IpPrefix as RoaIpPrefix, RoaAfi, RoaObject};
use rpki::data_model::signed_object::{
ResourceEeCertificate, RpkiSignedObject, SignedAttrsProfiled, SignerInfoProfiled,
};
use rpki::data_model::ta::TaCertificate;
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum ObjectType {
Auto,
Cer,
Mft,
Crl,
Roa,
Aspa,
}
use rpki::object_projection::{ObjectType, parse_object_json, resolve_object_type};
#[derive(Debug, PartialEq, Eq)]
struct Args {
@ -91,7 +68,8 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
"--type" => {
index += 1;
let value = argv.get(index).ok_or("--type requires a value")?;
args.object_type = parse_object_type(value)?;
args.object_type =
ObjectType::parse(value).map_err(|err| format!("{err}\n{}", usage()))?;
}
"--input" | "--in" => {
index += 1;
@ -130,18 +108,6 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
Ok(args)
}
fn parse_object_type(value: &str) -> Result<ObjectType, String> {
match value.to_ascii_lowercase().as_str() {
"auto" => Ok(ObjectType::Auto),
"cer" | ".cer" | "cert" | "certificate" => Ok(ObjectType::Cer),
"mft" | ".mft" | "manifest" => Ok(ObjectType::Mft),
"crl" | ".crl" => Ok(ObjectType::Crl),
"roa" | ".roa" => Ok(ObjectType::Roa),
"asa" | ".asa" | "aspa" => Ok(ObjectType::Aspa),
_ => Err(format!("unsupported --type: {value}\n{}", usage())),
}
}
fn parse_limit(value: &str) -> Result<usize, String> {
if value.eq_ignore_ascii_case("all") {
return Ok(usize::MAX);
@ -151,390 +117,10 @@ fn parse_limit(value: &str) -> Result<usize, String> {
.map_err(|_| format!("invalid --entry-limit: {value}"))
}
fn resolve_object_type(object_type: ObjectType, path: &Path) -> Result<ObjectType, String> {
if object_type != ObjectType::Auto {
return Ok(object_type);
}
match path
.extension()
.and_then(|v| v.to_str())
.map(|v| v.to_ascii_lowercase())
.as_deref()
{
Some("cer") => Ok(ObjectType::Cer),
Some("mft") => Ok(ObjectType::Mft),
Some("crl") => Ok(ObjectType::Crl),
Some("roa") => Ok(ObjectType::Roa),
Some("asa") | Some("aspa") => Ok(ObjectType::Aspa),
_ => Err(format!(
"cannot infer object type from path: {}",
path.display()
)),
}
}
fn parse_object_json(
object_type: ObjectType,
input_path: &Path,
bytes: &[u8],
entry_limit: usize,
) -> Value {
let object = match object_type {
ObjectType::Auto => unreachable!("auto must be resolved"),
ObjectType::Cer => parse_cer_json(bytes),
ObjectType::Mft => parse_mft_json(bytes, entry_limit),
ObjectType::Crl => parse_crl_json(bytes, entry_limit),
ObjectType::Roa => parse_roa_json(bytes, entry_limit),
ObjectType::Aspa => parse_aspa_json(bytes, entry_limit),
};
json!({
"tool": "rpki_object_parse",
"schemaVersion": 1,
"input": {
"path": input_path.display().to_string(),
"type": object_type_label(object_type),
"bytes": bytes_summary(bytes),
},
"object": object,
})
}
fn parse_cer_json(bytes: &[u8]) -> Value {
match ResourceCertificate::decode_der(bytes) {
Ok(cert) => {
let ta_profile = match TaCertificate::decode_der(bytes) {
Ok(ta) => json!({
"valid": true,
"selfSignature": result_json(ta.verify_self_signature().map_err(|e| e.to_string())),
}),
Err(err) => json!({
"valid": false,
"error": err.to_string(),
}),
};
json!({
"type": "cer",
"decode": {"profileValid": true},
"resourceCertificate": resource_certificate_json(&cert),
"trustAnchorProfile": ta_profile,
})
}
Err(err) => json!({
"type": "cer",
"decode": {"profileValid": false, "error": err.to_string()},
}),
}
}
fn parse_mft_json(bytes: &[u8], entry_limit: usize) -> Value {
match ManifestObject::decode_der(bytes) {
Ok(mft) => {
let files = mft.manifest.parse_files();
let (file_sample, file_list_error) = match files {
Ok(entries) => (
json!({
"count": entries.len(),
"truncated": entries.len() > entry_limit,
"entries": entries.iter().take(entry_limit).map(|item| {
json!({"fileName": item.file_name, "hashHex": hex::encode(item.hash_bytes)})
}).collect::<Vec<_>>(),
}),
Value::Null,
),
Err(err) => (Value::Null, json!(err.to_string())),
};
json!({
"type": "mft",
"decode": {"profileValid": true},
"eContentType": mft.econtent_type,
"signedObject": signed_object_json(&mft.signed_object),
"manifest": {
"version": mft.manifest.version,
"manifestNumberHex": mft.manifest.manifest_number.to_hex_upper(),
"thisUpdate": format_time(mft.manifest.this_update),
"nextUpdate": format_time(mft.manifest.next_update),
"fileHashAlg": mft.manifest.file_hash_alg,
"fileCount": mft.manifest.file_count(),
"fileList": file_sample,
"fileListError": file_list_error,
},
"embeddedEeProfile": result_json(mft.validate_embedded_ee_cert().map_err(|e| e.to_string())),
"cmsSignature": result_json(mft.signed_object.verify_signature().map_err(|e| e.to_string())),
})
}
Err(err) => json!({
"type": "mft",
"decode": {"profileValid": false, "error": err.to_string()},
}),
}
}
fn parse_crl_json(bytes: &[u8], entry_limit: usize) -> Value {
match RpkixCrl::decode_der(bytes) {
Ok(crl) => json!({
"type": "crl",
"decode": {"profileValid": true},
"rawDer": bytes_summary(&crl.raw_der),
"version": crl.version,
"issuer": crl.issuer_dn,
"signatureAlgorithm": crl.signature_algorithm_oid,
"thisUpdate": format_time(crl.this_update.utc),
"nextUpdate": format_time(crl.next_update.utc),
"extensions": {
"authorityKeyIdentifier": hex::encode(&crl.extensions.authority_key_identifier),
"crlNumberHex": crl.extensions.crl_number.to_hex_upper(),
"crlNumber": crl.extensions.crl_number.to_u64(),
},
"revokedCertificates": {
"count": crl.revoked_certs.len(),
"truncated": crl.revoked_certs.len() > entry_limit,
"entries": crl.revoked_certs.iter().take(entry_limit).map(|item| {
json!({
"serialNumberHex": item.serial_number.to_hex_upper(),
"serialNumber": item.serial_number.to_u64(),
"revocationDate": format_time(item.revocation_date.utc),
})
}).collect::<Vec<_>>(),
},
}),
Err(err) => json!({
"type": "crl",
"decode": {"profileValid": false, "error": err.to_string()},
}),
}
}
fn parse_roa_json(bytes: &[u8], entry_limit: usize) -> Value {
match RoaObject::decode_der(bytes) {
Ok(roa) => json!({
"type": "roa",
"decode": {"profileValid": true},
"eContentType": roa.econtent_type,
"signedObject": signed_object_json(&roa.signed_object),
"roa": {
"version": roa.roa.version,
"asId": roa.roa.as_id,
"ipAddressFamilies": roa.roa.ip_addr_blocks.iter().map(|family| {
json!({
"afi": format!("{:?}", family.afi),
"addressCount": family.addresses.len(),
"truncated": family.addresses.len() > entry_limit,
"addresses": family.addresses.iter().take(entry_limit).map(|entry| {
json!({
"prefix": roa_prefix_string(&entry.prefix),
"maxLength": entry.max_length,
})
}).collect::<Vec<_>>(),
})
}).collect::<Vec<_>>(),
},
"embeddedEeProfile": result_json(roa.validate_embedded_ee_cert().map_err(|e| e.to_string())),
"cmsSignature": result_json(roa.signed_object.verify_signature().map_err(|e| e.to_string())),
}),
Err(err) => json!({
"type": "roa",
"decode": {"profileValid": false, "error": err.to_string()},
}),
}
}
fn parse_aspa_json(bytes: &[u8], entry_limit: usize) -> Value {
match AspaObject::decode_der(bytes) {
Ok(aspa) => json!({
"type": "aspa",
"decode": {"profileValid": true},
"eContentType": aspa.econtent_type,
"signedObject": signed_object_json(&aspa.signed_object),
"aspa": {
"version": aspa.aspa.version,
"customerAsId": aspa.aspa.customer_as_id,
"providerCount": aspa.aspa.provider_as_ids.len(),
"providersTruncated": aspa.aspa.provider_as_ids.len() > entry_limit,
"providerAsIds": aspa.aspa.provider_as_ids.iter().take(entry_limit).copied().collect::<Vec<_>>(),
},
"embeddedEeProfile": result_json(aspa.validate_embedded_ee_cert().map_err(|e| e.to_string())),
"cmsSignature": result_json(aspa.signed_object.verify_signature().map_err(|e| e.to_string())),
}),
Err(err) => json!({
"type": "aspa",
"decode": {"profileValid": false, "error": err.to_string()},
}),
}
}
fn resource_certificate_json(cert: &ResourceCertificate) -> Value {
let tbs = &cert.tbs;
json!({
"rawDer": bytes_summary(&cert.raw_der),
"kind": format!("{:?}", cert.kind),
"version": tbs.version,
"serialNumberHex": hex::encode(tbs.serial_number.to_bytes_be()),
"signatureAlgorithm": tbs.signature_algorithm,
"issuer": tbs.issuer_name.to_string(),
"subject": tbs.subject_name.to_string(),
"validity": {
"notBefore": format_time(tbs.validity_not_before),
"notAfter": format_time(tbs.validity_not_after),
},
"subjectPublicKeyInfo": bytes_summary(&tbs.subject_public_key_info),
"extensions": rc_extensions_json(&tbs.extensions),
})
}
fn rc_extensions_json(ext: &RcExtensions) -> Value {
json!({
"basicConstraintsCa": ext.basic_constraints_ca,
"subjectKeyIdentifier": ext.subject_key_identifier.as_ref().map(|v| hex::encode(v)),
"authorityKeyIdentifier": ext.authority_key_identifier.as_ref().map(|v| hex::encode(v)),
"crlDistributionPointsUris": ext.crl_distribution_points_uris,
"caIssuersUris": ext.ca_issuers_uris,
"subjectInfoAccess": subject_info_access_json(ext.subject_info_access.as_ref()),
"certificatePoliciesOid": ext.certificate_policies_oid,
"ipResources": serde_json::to_value(&ext.ip_resources).unwrap_or(Value::Null),
"asResources": serde_json::to_value(&ext.as_resources).unwrap_or(Value::Null),
})
}
fn subject_info_access_json(value: Option<&SubjectInfoAccess>) -> Value {
match value {
None => Value::Null,
Some(SubjectInfoAccess::Ca(ca)) => json!({
"kind": "ca",
"accessDescriptions": ca.access_descriptions.iter().map(access_description_json).collect::<Vec<_>>(),
}),
Some(SubjectInfoAccess::Ee(ee)) => json!({
"kind": "ee",
"signedObjectUris": ee.signed_object_uris,
"accessDescriptions": ee.access_descriptions.iter().map(access_description_json).collect::<Vec<_>>(),
}),
}
}
fn access_description_json(value: &AccessDescription) -> Value {
json!({
"accessMethodOid": value.access_method_oid,
"accessLocation": value.access_location,
})
}
fn signed_object_json(signed_object: &RpkiSignedObject) -> Value {
let signed_data = &signed_object.signed_data;
json!({
"rawDer": bytes_summary(&signed_object.raw_der),
"contentInfoContentType": signed_object.content_info_content_type,
"signedData": {
"version": signed_data.version,
"digestAlgorithms": signed_data.digest_algorithms,
"encapContentInfo": {
"eContentType": signed_data.encap_content_info.econtent_type,
"eContent": bytes_summary(&signed_data.encap_content_info.econtent),
},
"certificates": signed_data.certificates.iter().map(ee_certificate_json).collect::<Vec<_>>(),
"crlsPresent": signed_data.crls_present,
"signerInfos": signed_data.signer_infos.iter().map(signer_info_json).collect::<Vec<_>>(),
},
})
}
fn ee_certificate_json(cert: &ResourceEeCertificate) -> Value {
json!({
"rawDer": bytes_summary(&cert.raw_der),
"subjectKeyIdentifier": hex::encode(&cert.subject_key_identifier),
"spkiDer": bytes_summary(&cert.spki_der),
"rsaPublicKey": {
"modulus": bytes_summary(&cert.rsa_public_modulus),
"exponent": bytes_summary(&cert.rsa_public_exponent),
},
"tbsCertificate": bytes_summary(&cert.tbs_certificate_der),
"certificateSignature": bytes_summary(&cert.signature_bytes),
"keyUsageSummary": format!("{:?}", cert.key_usage_summary),
"siaSignedObjectUris": cert.sia_signed_object_uris,
"resourceCertificate": resource_certificate_json(&cert.resource_cert),
})
}
fn signer_info_json(info: &SignerInfoProfiled) -> Value {
json!({
"version": info.version,
"sidSki": hex::encode(&info.sid_ski),
"digestAlgorithm": info.digest_algorithm,
"signatureAlgorithm": info.signature_algorithm,
"signedAttrs": signed_attrs_json(&info.signed_attrs),
"unsignedAttrsPresent": info.unsigned_attrs_present,
"signature": bytes_summary(&info.signature),
"signedAttrsDerForSignature": bytes_summary(&info.signed_attrs_der_for_signature),
})
}
fn signed_attrs_json(attrs: &SignedAttrsProfiled) -> Value {
json!({
"contentType": attrs.content_type,
"messageDigest": hex::encode(&attrs.message_digest),
"signingTime": {
"utc": format_time(attrs.signing_time.utc),
"encoding": format!("{:?}", attrs.signing_time.encoding),
},
"otherAttrsPresent": attrs.other_attrs_present,
})
}
fn result_json(result: Result<(), String>) -> Value {
match result {
Ok(()) => json!({"valid": true}),
Err(err) => json!({"valid": false, "error": err}),
}
}
fn object_type_label(object_type: ObjectType) -> &'static str {
match object_type {
ObjectType::Auto => "auto",
ObjectType::Cer => "cer",
ObjectType::Mft => "mft",
ObjectType::Crl => "crl",
ObjectType::Roa => "roa",
ObjectType::Aspa => "aspa",
}
}
fn bytes_summary(bytes: &[u8]) -> Value {
let head_len = bytes.len().min(16);
let tail_len = bytes.len().min(16);
json!({
"len": bytes.len(),
"sha256": sha256_hex(bytes),
"headHex": hex::encode(&bytes[..head_len]),
"tailHex": hex::encode(&bytes[bytes.len().saturating_sub(tail_len)..]),
})
}
fn sha256_hex(bytes: &[u8]) -> String {
hex::encode(Sha256::digest(bytes))
}
fn format_time(value: time::OffsetDateTime) -> String {
value
.to_offset(time::UtcOffset::UTC)
.format(&time::format_description::well_known::Rfc3339)
.unwrap_or_else(|_| value.unix_timestamp().to_string())
}
fn roa_prefix_string(prefix: &RoaIpPrefix) -> String {
let bytes = prefix.addr_bytes();
match prefix.afi {
RoaAfi::Ipv4 => {
let octets = [bytes[0], bytes[1], bytes[2], bytes[3]];
format!("{}/{}", Ipv4Addr::from(octets), prefix.prefix_len)
}
RoaAfi::Ipv6 => {
let mut octets = [0u8; 16];
octets.copy_from_slice(bytes);
format!("{}/{}", Ipv6Addr::from(octets), prefix.prefix_len)
}
}
}
#[cfg(test)]
mod tests {
use std::path::Path;
use super::*;
#[test]

278
src/bin/rpki_query_indexer.rs Normal file
View File

@ -0,0 +1,278 @@
use std::path::PathBuf;
use rpki::query_db::{ArtifactIndexerConfig, index_artifacts};
#[derive(Debug, PartialEq, Eq)]
struct Args {
query_db: PathBuf,
run_root: Option<PathBuf>,
run_dir: Option<PathBuf>,
repo_bytes_db: Option<PathBuf>,
projection_entry_limit: usize,
min_run_seq: Option<u64>,
retain_indexed_runs: Option<usize>,
dump_summary: bool,
}
fn usage() -> &'static str {
"Usage: rpki_query_indexer --query-db <path> (--run-root <path>|--run-dir <path>) [--repo-bytes-db <path>] [--projection-entry-limit <n>] [--min-run-seq <n>] [--retain-indexed-runs <n>] [--dump-summary]"
}
fn parse_args(argv: &[String]) -> Result<Args, String> {
let mut query_db = None;
let mut run_root = None;
let mut run_dir = None;
let mut repo_bytes_db = None;
let mut projection_entry_limit = 50usize;
let mut min_run_seq = None;
let mut retain_indexed_runs = Some(10usize);
let mut dump_summary = false;
let mut index = 1usize;
while index < argv.len() {
match argv[index].as_str() {
"--query-db" => {
index += 1;
query_db = Some(PathBuf::from(value_at(argv, index, "--query-db")?));
}
"--run-root" => {
index += 1;
run_root = Some(PathBuf::from(value_at(argv, index, "--run-root")?));
}
"--run-dir" => {
index += 1;
run_dir = Some(PathBuf::from(value_at(argv, index, "--run-dir")?));
}
"--repo-bytes-db" => {
index += 1;
repo_bytes_db = Some(PathBuf::from(value_at(argv, index, "--repo-bytes-db")?));
}
"--projection-entry-limit" => {
index += 1;
let raw = value_at(argv, index, "--projection-entry-limit")?;
projection_entry_limit = raw
.parse::<usize>()
.map_err(|_| format!("invalid --projection-entry-limit: {raw}"))?;
}
"--min-run-seq" => {
index += 1;
let raw = value_at(argv, index, "--min-run-seq")?;
min_run_seq = Some(
raw.parse::<u64>()
.map_err(|_| format!("invalid --min-run-seq: {raw}"))?,
);
}
"--retain-indexed-runs" => {
index += 1;
let raw = value_at(argv, index, "--retain-indexed-runs")?;
retain_indexed_runs = Some(
raw.parse::<usize>()
.map_err(|_| format!("invalid --retain-indexed-runs: {raw}"))?,
);
}
"--dump-summary" => dump_summary = true,
"-h" | "--help" => return Err(usage().to_string()),
other => return Err(format!("unknown argument: {other}\n{}", usage())),
}
index += 1;
}
if run_root.is_some() == run_dir.is_some() {
return Err(format!(
"exactly one of --run-root or --run-dir is required\n{}",
usage()
));
}
Ok(Args {
query_db: query_db.ok_or_else(|| format!("--query-db is required\n{}", usage()))?,
run_root,
run_dir,
repo_bytes_db,
projection_entry_limit,
min_run_seq,
retain_indexed_runs,
dump_summary,
})
}
fn value_at<'a>(argv: &'a [String], index: usize, flag: &str) -> Result<&'a str, String> {
argv.get(index)
.map(String::as_str)
.ok_or_else(|| format!("{flag} requires a value"))
}
fn main() {
let argv = std::env::args().collect::<Vec<_>>();
let args = match parse_args(&argv) {
Ok(args) => args,
Err(err) => {
eprintln!("{err}");
std::process::exit(2);
}
};
let summary = match index_artifacts(&ArtifactIndexerConfig {
query_db_path: args.query_db,
run_root: args.run_root,
run_dir: args.run_dir,
repo_bytes_db_path: args.repo_bytes_db,
projection_entry_limit: args.projection_entry_limit,
min_run_seq: args.min_run_seq,
retain_indexed_runs: args.retain_indexed_runs,
}) {
Ok(summary) => summary,
Err(err) => {
eprintln!("query index failed: {err}");
std::process::exit(1);
}
};
if args.dump_summary {
println!(
"{}",
serde_json::to_string_pretty(&summary).expect("serialize summary")
);
} else {
println!(
"indexed_runs={} runs_deleted={} retained_runs={} repos={} publication_points={} objects={} latest_ready_run={}",
summary.runs_indexed,
summary.runs_deleted,
summary.retained_runs,
summary.repos_indexed,
summary.publication_points_indexed,
summary.object_instances_indexed,
summary.latest_ready_run.as_deref().unwrap_or("none")
);
}
if !summary.errors.is_empty() {
eprintln!(
"index completed with {} per-run errors",
summary.errors.len()
);
std::process::exit(1);
}
}
#[cfg(test)]
mod tests {
use super::*;
fn argv(args: &[&str]) -> Vec<String> {
std::iter::once("rpki_query_indexer")
.chain(args.iter().copied())
.map(str::to_string)
.collect()
}
#[test]
fn parse_args_accepts_run_root_and_optional_flags() {
let args = parse_args(&argv(&[
"--query-db",
"query.db",
"--run-root",
"runs",
"--repo-bytes-db",
"repo-bytes.db",
"--projection-entry-limit",
"7",
"--min-run-seq",
"42",
"--retain-indexed-runs",
"3",
"--dump-summary",
]))
.expect("args");
assert_eq!(args.query_db, PathBuf::from("query.db"));
assert_eq!(args.run_root.as_deref(), Some(std::path::Path::new("runs")));
assert_eq!(args.run_dir, None);
assert_eq!(
args.repo_bytes_db.as_deref(),
Some(std::path::Path::new("repo-bytes.db"))
);
assert_eq!(args.projection_entry_limit, 7);
assert_eq!(args.min_run_seq, Some(42));
assert_eq!(args.retain_indexed_runs, Some(3));
assert!(args.dump_summary);
}
#[test]
fn parse_args_accepts_single_run_dir() {
let args = parse_args(&argv(&[
"--query-db",
"query.db",
"--run-dir",
"runs/run_0001",
]))
.expect("args");
assert_eq!(
args.run_dir.as_deref(),
Some(std::path::Path::new("runs/run_0001"))
);
assert_eq!(args.run_root, None);
assert_eq!(args.projection_entry_limit, 50);
assert_eq!(args.min_run_seq, None);
assert_eq!(args.retain_indexed_runs, Some(10));
}
#[test]
fn parse_args_rejects_invalid_or_ambiguous_input() {
assert!(
parse_args(&argv(&["--query-db", "query.db"]))
.unwrap_err()
.contains("exactly one")
);
assert!(
parse_args(&argv(&[
"--query-db",
"query.db",
"--run-root",
"runs",
"--run-dir",
"runs/run_0001",
]))
.unwrap_err()
.contains("exactly one")
);
assert!(
parse_args(&argv(&["--run-root", "runs"]))
.unwrap_err()
.contains("--query-db")
);
assert!(
parse_args(&argv(&[
"--query-db",
"query.db",
"--run-root",
"runs",
"--projection-entry-limit",
"bad",
]))
.unwrap_err()
.contains("invalid --projection-entry-limit")
);
assert!(
parse_args(&argv(&[
"--query-db",
"query.db",
"--run-root",
"runs",
"--unknown",
]))
.unwrap_err()
.contains("unknown argument")
);
assert!(
parse_args(&argv(&[
"--query-db",
"query.db",
"--run-root",
"runs",
"--retain-indexed-runs",
"bad",
]))
.unwrap_err()
.contains("invalid --retain-indexed-runs")
);
assert!(
parse_args(&argv(&["--query-db"]))
.unwrap_err()
.contains("requires a value")
);
}
}

2897
src/bin/rpki_query_service.rs Normal file
View File

File diff suppressed because it is too large Load Diff

12
src/blob_store.rs
View File

@ -192,6 +192,18 @@ impl ExternalRepoBytesDb {
})
}
pub fn open_read_only(path: impl Into<PathBuf>) -> StorageResult<Self> {
let path = path.into();
let mut opts = Options::default();
opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
let db = DB::open_for_read_only(&opts, &path, false)
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
Ok(Self {
path,
db: Arc::new(db),
})
}
pub fn put_blob_bytes_batch(&self, blobs: &[(String, Vec<u8>)]) -> StorageResult<()> {
if blobs.is_empty() {
return Ok(());

1
src/cli.rs
View File

@ -1608,6 +1608,7 @@ fn build_report(
downloads: shared.downloads.iter().cloned().collect(),
download_stats: shared.download_stats.clone(),
repo_sync_stats,
query_audit: None,
}
}

152
src/cli/output.rs
View File

@ -3,8 +3,12 @@ use std::path::Path;
use serde::Serialize;
use serde::ser::SerializeSeq;
use sha2::Digest;
use crate::audit::{AspaOutput, AuditRunMeta, AuditWarning, VrpOutput};
use crate::audit::{
AspaOutput, AuditRunMeta, AuditWarning, QueryAuditManifest, ValidationEvent,
ValidationEventCounts, VrpOutput,
};
use crate::ccr::canonical_vrp_prefix;
use super::{PostValidationShared, RunStageTiming};
@ -49,6 +53,8 @@ struct BorrowedAuditReportV2<'a> {
downloads: &'a [crate::audit::AuditDownloadEvent],
download_stats: &'a crate::audit::AuditDownloadStats,
repo_sync_stats: crate::audit::AuditRepoSyncStats,
#[serde(rename = "queryAudit", skip_serializing_if = "Option::is_none")]
query_audit: Option<QueryAuditManifest>,
}
#[derive(Serialize)]
@ -125,6 +131,7 @@ pub(super) fn write_report_json_from_shared(
.format(&Rfc3339)
.expect("format validation_time");
let repo_sync_stats = super::build_repo_sync_stats(shared.publication_points.as_ref());
let query_audit = write_validation_events_sidecar(path, &validation_time_rfc3339_utc, shared)?;
let report = BorrowedAuditReportV2 {
format_version: 2,
meta: AuditRunMeta {
@ -142,6 +149,7 @@ pub(super) fn write_report_json_from_shared(
downloads: shared.downloads.as_ref(),
download_stats: &shared.download_stats,
repo_sync_stats,
query_audit: Some(query_audit),
};
let build_ms = build_started.elapsed().as_millis() as u64;
@ -153,6 +161,148 @@ pub(super) fn write_report_json_from_shared(
})
}
fn write_validation_events_sidecar(
report_path: &Path,
validation_time: &str,
shared: &PostValidationShared,
) -> Result<QueryAuditManifest, String> {
let events_path = report_path.with_file_name("validation-events.jsonl");
if let Some(parent) = events_path.parent() {
std::fs::create_dir_all(parent)
.map_err(|e| format!("create validation events parent failed: {e}"))?;
}
let mut writer = BufWriter::new(std::fs::File::create(&events_path).map_err(|e| {
format!(
"create validation events failed: {}: {e}",
events_path.display()
)
})?);
let mut seq = 0u64;
let mut hasher = sha2::Sha256::new();
emit_validation_events(validation_time, shared, &mut seq, &mut |event| {
let mut line = serde_json::to_vec(&event)
.map_err(|e| format!("serialize validation event failed: {e}"))?;
line.push(b'\n');
std::io::Write::write_all(&mut writer, &line)
.map_err(|e| format!("write validation event failed: {e}"))?;
hasher.update(&line);
Ok(())
})?;
std::io::Write::flush(&mut writer)
.map_err(|e| format!("flush validation events failed: {e}"))?;
let events_count = seq;
let events_sha256 = hex::encode(hasher.finalize());
Ok(QueryAuditManifest {
schema_version: 1,
status: "complete".to_string(),
events_path: events_path
.file_name()
.and_then(|name| name.to_str())
.unwrap_or("validation-events.jsonl")
.to_string(),
events_count,
events_sha256,
writer_version: 1,
error: None,
})
}
fn emit_validation_events(
validation_time: &str,
shared: &PostValidationShared,
seq: &mut u64,
emit: &mut impl FnMut(ValidationEvent) -> Result<(), String>,
) -> Result<(), String> {
emit(next_event(seq, "run_summary", validation_time, |event| {
event.counts = Some(ValidationEventCounts {
objects: Some(
shared
.publication_points
.iter()
.map(|pp| pp.objects.len() as u64)
.sum(),
),
warnings: Some(
(shared.tree_warnings.len()
+ shared
.publication_points
.iter()
.map(|pp| pp.warnings.len())
.sum::<usize>()) as u64,
),
vrps: Some(shared.vrps.len() as u64),
aspas: Some(shared.aspas.len() as u64),
});
}))?;
for pp in shared.publication_points.iter() {
emit(next_event(
seq,
"publication_point",
validation_time,
|event| {
event.pp_node_id = pp.node_id;
event.pp_manifest_uri = Some(pp.manifest_rsync_uri.clone());
event.pp_rsync_base_uri = Some(pp.rsync_base_uri.clone());
event.repo_sync_phase = pp.repo_sync_phase.clone();
event.repo_terminal_state = Some(pp.repo_terminal_state.clone());
event.counts = Some(ValidationEventCounts {
objects: Some(pp.objects.len() as u64),
warnings: Some(pp.warnings.len() as u64),
vrps: None,
aspas: None,
});
},
))?;
for object in &pp.objects {
emit(next_event(seq, "object", validation_time, |event| {
event.pp_node_id = pp.node_id;
event.pp_manifest_uri = Some(pp.manifest_rsync_uri.clone());
event.object_uri = Some(object.rsync_uri.clone());
event.sha256 = Some(object.sha256_hex.clone());
event.object_type = Some(object.kind.clone());
event.result = Some(object.result.clone());
event.reason = object.detail.clone();
}))?;
}
for warning in &pp.warnings {
emit(next_event(seq, "warning", validation_time, |event| {
event.pp_node_id = pp.node_id;
event.pp_manifest_uri = Some(pp.manifest_rsync_uri.clone());
event.reason = Some(warning.message.clone());
}))?;
}
}
Ok(())
}
fn next_event(
seq: &mut u64,
event_type: &str,
validation_time: &str,
fill: impl FnOnce(&mut ValidationEvent),
) -> ValidationEvent {
*seq += 1;
let mut event = ValidationEvent {
schema_version: 1,
seq: *seq,
event_type: event_type.to_string(),
validation_time: validation_time.to_string(),
pp_node_id: None,
pp_manifest_uri: None,
pp_rsync_base_uri: None,
repo_sync_phase: None,
repo_terminal_state: None,
object_uri: None,
sha256: None,
object_type: None,
result: None,
reason: None,
counts: None,
};
fill(&mut event);
event
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(super) struct CompareViewTaskOutput {
pub(super) build_ms: Option<u64>,

22
src/cli/tests.rs
View File

@ -1452,6 +1452,14 @@ fn synthetic_post_validation_shared() -> PostValidationShared {
let mut pp1 = crate::audit::PublicationPointAudit::default();
pp1.source = "fresh".to_string();
pp1.rrdp_notification_uri = Some("https://example.test/n1.xml".to_string());
pp1.manifest_rsync_uri = "rsync://example.test/repo/pp1/manifest.mft".to_string();
pp1.objects.push(crate::audit::ObjectAuditEntry {
rsync_uri: "rsync://example.test/repo/pp1/a.roa".to_string(),
sha256_hex: "11".repeat(32),
kind: crate::audit::AuditObjectKind::Roa,
result: crate::audit::AuditObjectResult::Ok,
detail: None,
});
let mut pp2 = crate::audit::PublicationPointAudit::default();
pp2.source = "fresh".to_string();
pp2.rrdp_notification_uri = Some("https://example.test/n1.xml".to_string());
@ -1549,6 +1557,19 @@ fn run_report_task_and_stage_timing_work() {
serde_json::from_str(&report_json).expect("parse compact report json");
assert_eq!(report["vrps"].as_array().unwrap().len(), 2);
assert_eq!(report["aspas"].as_array().unwrap().len(), 1);
assert_eq!(report["queryAudit"]["status"].as_str(), Some("complete"));
assert!(report["queryAudit"]["eventsCount"].as_u64().unwrap() > 0);
let events_path = dir.path().join(
report["queryAudit"]["eventsPath"]
.as_str()
.expect("events path"),
);
let events = std::fs::read_to_string(events_path).expect("read validation events");
assert!(
events
.lines()
.any(|line| line.contains("\"eventType\":\"object\""))
);
let stage_timing = RunStageTiming {
validation_ms: 1,
@ -1858,6 +1879,7 @@ fn write_json_writes_report() {
downloads: Vec::new(),
download_stats: crate::audit::AuditDownloadStats::default(),
repo_sync_stats: crate::audit::AuditRepoSyncStats::default(),
query_audit: None,
};
let dir = tempfile::tempdir().expect("tmpdir");

6
src/lib.rs
View File

@ -19,12 +19,18 @@ pub mod fetch;
#[cfg(feature = "full")]
pub mod memory_telemetry;
#[cfg(feature = "full")]
pub mod object_projection;
#[cfg(feature = "full")]
pub mod parallel;
#[cfg(feature = "full")]
pub mod policy;
#[cfg(feature = "full")]
pub mod progress_log;
#[cfg(feature = "full")]
pub mod query;
#[cfg(feature = "full")]
pub mod query_db;
#[cfg(feature = "full")]
pub mod replay;
#[cfg(feature = "full")]
pub mod report;

668
src/object_projection.rs Normal file
View File

@ -0,0 +1,668 @@
use std::net::{Ipv4Addr, Ipv6Addr};
use std::path::Path;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
use crate::data_model::aspa::AspaObject;
use crate::data_model::crl::RpkixCrl;
use crate::data_model::manifest::ManifestObject;
use crate::data_model::rc::{
AccessDescription, RcExtensions, ResourceCertificate, SubjectInfoAccess,
};
use crate::data_model::roa::{IpPrefix as RoaIpPrefix, RoaAfi, RoaObject};
use crate::data_model::signed_object::{
ResourceEeCertificate, RpkiSignedObject, SignedAttrsProfiled, SignerInfoProfiled,
};
use crate::data_model::ta::TaCertificate;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ObjectType {
Auto,
Cer,
Mft,
Crl,
Roa,
Aspa,
}
impl ObjectType {
pub fn parse(value: &str) -> Result<Self, String> {
match value.to_ascii_lowercase().as_str() {
"auto" => Ok(Self::Auto),
"cer" | ".cer" | "cert" | "certificate" => Ok(Self::Cer),
"mft" | ".mft" | "manifest" => Ok(Self::Mft),
"crl" | ".crl" => Ok(Self::Crl),
"roa" | ".roa" => Ok(Self::Roa),
"asa" | ".asa" | "aspa" => Ok(Self::Aspa),
_ => Err(format!("unsupported object type: {value}")),
}
}
pub fn label(self) -> &'static str {
object_type_label(self)
}
}
pub fn resolve_object_type(object_type: ObjectType, path: &Path) -> Result<ObjectType, String> {
if object_type != ObjectType::Auto {
return Ok(object_type);
}
match path
.extension()
.and_then(|v| v.to_str())
.map(|v| v.to_ascii_lowercase())
.as_deref()
{
Some("cer") => Ok(ObjectType::Cer),
Some("mft") => Ok(ObjectType::Mft),
Some("crl") => Ok(ObjectType::Crl),
Some("roa") => Ok(ObjectType::Roa),
Some("asa") | Some("aspa") => Ok(ObjectType::Aspa),
_ => Err(format!(
"cannot infer object type from path: {}",
path.display()
)),
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ObjectProjectionRecord {
pub schema_version: u32,
pub sha256: String,
pub object_type: String,
pub parse_status: String,
pub error_summary: Option<String>,
pub projection: Value,
}
pub fn build_object_projection(
object_type: ObjectType,
input_path: &Path,
bytes: &[u8],
entry_limit: usize,
) -> ObjectProjectionRecord {
let resolved = match resolve_object_type(object_type, input_path) {
Ok(value) => value,
Err(err) => {
return ObjectProjectionRecord {
schema_version: 1,
sha256: sha256_hex(bytes),
object_type: "unknown".to_string(),
parse_status: "error".to_string(),
error_summary: Some(err),
projection: json!({"decode": {"profileValid": false}}),
};
}
};
let projection = parse_object_json(resolved, input_path, bytes, entry_limit);
let parse_status = if projection
.get("object")
.and_then(|v| v.get("decode"))
.and_then(|v| v.get("profileValid"))
.and_then(Value::as_bool)
.unwrap_or(false)
{
"ok"
} else {
"error"
};
let error_summary = projection
.get("object")
.and_then(|v| v.get("decode"))
.and_then(|v| v.get("error"))
.and_then(Value::as_str)
.map(str::to_string);
ObjectProjectionRecord {
schema_version: 1,
sha256: sha256_hex(bytes),
object_type: resolved.label().to_string(),
parse_status: parse_status.to_string(),
error_summary,
projection,
}
}
pub fn parse_object_json(
object_type: ObjectType,
input_path: &Path,
bytes: &[u8],
entry_limit: usize,
) -> Value {
let object = match object_type {
ObjectType::Auto => unreachable!("auto must be resolved"),
ObjectType::Cer => parse_cer_json(bytes),
ObjectType::Mft => parse_mft_json(bytes, entry_limit),
ObjectType::Crl => parse_crl_json(bytes, entry_limit),
ObjectType::Roa => parse_roa_json(bytes, entry_limit),
ObjectType::Aspa => parse_aspa_json(bytes, entry_limit),
};
json!({
"tool": "rpki_object_parse",
"schemaVersion": 1,
"input": {
"path": input_path.display().to_string(),
"type": object_type_label(object_type),
"bytes": bytes_summary(bytes),
},
"object": object,
})
}
pub fn parse_cer_json(bytes: &[u8]) -> Value {
match ResourceCertificate::decode_der(bytes) {
Ok(cert) => {
let ta_profile = match TaCertificate::decode_der(bytes) {
Ok(ta) => json!({
"valid": true,
"selfSignature": result_json(ta.verify_self_signature().map_err(|e| e.to_string())),
}),
Err(err) => json!({
"valid": false,
"error": err.to_string(),
}),
};
json!({
"type": "cer",
"decode": {"profileValid": true},
"resourceCertificate": resource_certificate_json(&cert),
"trustAnchorProfile": ta_profile,
})
}
Err(err) => json!({
"type": "cer",
"decode": {"profileValid": false, "error": err.to_string()},
}),
}
}
pub fn parse_mft_json(bytes: &[u8], entry_limit: usize) -> Value {
match ManifestObject::decode_der(bytes) {
Ok(mft) => {
let files = mft.manifest.parse_files();
let (file_sample, file_list_error) = match files {
Ok(entries) => (
json!({
"count": entries.len(),
"truncated": entries.len() > entry_limit,
"entries": entries.iter().take(entry_limit).map(|item| {
json!({"fileName": item.file_name, "hashHex": hex::encode(item.hash_bytes)})
}).collect::<Vec<_>>(),
}),
Value::Null,
),
Err(err) => (Value::Null, json!(err.to_string())),
};
json!({
"type": "mft",
"decode": {"profileValid": true},
"eContentType": mft.econtent_type,
"signedObject": signed_object_json(&mft.signed_object),
"manifest": {
"version": mft.manifest.version,
"manifestNumberHex": mft.manifest.manifest_number.to_hex_upper(),
"thisUpdate": format_time(mft.manifest.this_update),
"nextUpdate": format_time(mft.manifest.next_update),
"fileHashAlg": mft.manifest.file_hash_alg,
"fileCount": mft.manifest.file_count(),
"fileList": file_sample,
"fileListError": file_list_error,
},
"embeddedEeProfile": result_json(mft.validate_embedded_ee_cert().map_err(|e| e.to_string())),
"cmsSignature": result_json(mft.signed_object.verify_signature().map_err(|e| e.to_string())),
})
}
Err(err) => json!({
"type": "mft",
"decode": {"profileValid": false, "error": err.to_string()},
}),
}
}
pub fn parse_crl_json(bytes: &[u8], entry_limit: usize) -> Value {
match RpkixCrl::decode_der(bytes) {
Ok(crl) => json!({
"type": "crl",
"decode": {"profileValid": true},
"rawDer": bytes_summary(&crl.raw_der),
"version": crl.version,
"issuer": crl.issuer_dn,
"signatureAlgorithm": crl.signature_algorithm_oid,
"thisUpdate": format_time(crl.this_update.utc),
"nextUpdate": format_time(crl.next_update.utc),
"extensions": {
"authorityKeyIdentifier": hex::encode(&crl.extensions.authority_key_identifier),
"crlNumberHex": crl.extensions.crl_number.to_hex_upper(),
"crlNumber": crl.extensions.crl_number.to_u64(),
},
"revokedCertificates": {
"count": crl.revoked_certs.len(),
"truncated": crl.revoked_certs.len() > entry_limit,
"entries": crl.revoked_certs.iter().take(entry_limit).map(|item| {
json!({
"serialNumberHex": item.serial_number.to_hex_upper(),
"serialNumber": item.serial_number.to_u64(),
"revocationDate": format_time(item.revocation_date.utc),
})
}).collect::<Vec<_>>(),
},
}),
Err(err) => json!({
"type": "crl",
"decode": {"profileValid": false, "error": err.to_string()},
}),
}
}
pub fn manifest_file_entries_page(
bytes: &[u8],
offset: usize,
limit: usize,
) -> Result<(usize, Vec<Value>), String> {
let mft = ManifestObject::decode_der(bytes).map_err(|err| err.to_string())?;
let entries = mft.manifest.parse_files().map_err(|err| err.to_string())?;
let total = entries.len();
let end = (offset + limit).min(total);
let page = entries[offset.min(total)..end]
.iter()
.map(|item| json!({"fileName": item.file_name, "hashHex": hex::encode(&item.hash_bytes)}))
.collect::<Vec<_>>();
Ok((total, page))
}
pub fn crl_revoked_entries_page(
bytes: &[u8],
offset: usize,
limit: usize,
) -> Result<(usize, Vec<Value>), String> {
let crl = RpkixCrl::decode_der(bytes).map_err(|err| err.to_string())?;
let total = crl.revoked_certs.len();
let end = (offset + limit).min(total);
let page = crl.revoked_certs[offset.min(total)..end]
.iter()
.map(|item| {
json!({
"serialNumberHex": item.serial_number.to_hex_upper(),
"serialNumber": item.serial_number.to_u64(),
"revocationDate": format_time(item.revocation_date.utc),
})
})
.collect::<Vec<_>>();
Ok((total, page))
}
pub fn parse_roa_json(bytes: &[u8], entry_limit: usize) -> Value {
match RoaObject::decode_der(bytes) {
Ok(roa) => json!({
"type": "roa",
"decode": {"profileValid": true},
"eContentType": roa.econtent_type,
"signedObject": signed_object_json(&roa.signed_object),
"roa": {
"version": roa.roa.version,
"asId": roa.roa.as_id,
"ipAddressFamilies": roa.roa.ip_addr_blocks.iter().map(|family| {
json!({
"afi": format!("{:?}", family.afi),
"addressCount": family.addresses.len(),
"truncated": family.addresses.len() > entry_limit,
"addresses": family.addresses.iter().take(entry_limit).map(|entry| {
json!({
"prefix": roa_prefix_string(&entry.prefix),
"maxLength": entry.max_length,
})
}).collect::<Vec<_>>(),
})
}).collect::<Vec<_>>(),
},
"embeddedEeProfile": result_json(roa.validate_embedded_ee_cert().map_err(|e| e.to_string())),
"cmsSignature": result_json(roa.signed_object.verify_signature().map_err(|e| e.to_string())),
}),
Err(err) => json!({
"type": "roa",
"decode": {"profileValid": false, "error": err.to_string()},
}),
}
}
pub fn parse_aspa_json(bytes: &[u8], entry_limit: usize) -> Value {
match AspaObject::decode_der(bytes) {
Ok(aspa) => json!({
"type": "aspa",
"decode": {"profileValid": true},
"eContentType": aspa.econtent_type,
"signedObject": signed_object_json(&aspa.signed_object),
"aspa": {
"version": aspa.aspa.version,
"customerAsId": aspa.aspa.customer_as_id,
"providerCount": aspa.aspa.provider_as_ids.len(),
"providersTruncated": aspa.aspa.provider_as_ids.len() > entry_limit,
"providerAsIds": aspa.aspa.provider_as_ids.iter().take(entry_limit).copied().collect::<Vec<_>>(),
},
"embeddedEeProfile": result_json(aspa.validate_embedded_ee_cert().map_err(|e| e.to_string())),
"cmsSignature": result_json(aspa.signed_object.verify_signature().map_err(|e| e.to_string())),
}),
Err(err) => json!({
"type": "aspa",
"decode": {"profileValid": false, "error": err.to_string()},
}),
}
}
fn resource_certificate_json(cert: &ResourceCertificate) -> Value {
let tbs = &cert.tbs;
json!({
"rawDer": bytes_summary(&cert.raw_der),
"kind": format!("{:?}", cert.kind),
"version": tbs.version,
"serialNumberHex": hex::encode(tbs.serial_number.to_bytes_be()),
"signatureAlgorithm": tbs.signature_algorithm,
"issuer": tbs.issuer_name.to_string(),
"subject": tbs.subject_name.to_string(),
"validity": {
"notBefore": format_time(tbs.validity_not_before),
"notAfter": format_time(tbs.validity_not_after),
},
"subjectPublicKeyInfo": bytes_summary(&tbs.subject_public_key_info),
"extensions": rc_extensions_json(&tbs.extensions),
})
}
fn rc_extensions_json(ext: &RcExtensions) -> Value {
json!({
"basicConstraintsCa": ext.basic_constraints_ca,
"subjectKeyIdentifier": ext.subject_key_identifier.as_ref().map(|v| hex::encode(v)),
"authorityKeyIdentifier": ext.authority_key_identifier.as_ref().map(|v| hex::encode(v)),
"crlDistributionPointsUris": ext.crl_distribution_points_uris,
"caIssuersUris": ext.ca_issuers_uris,
"subjectInfoAccess": subject_info_access_json(ext.subject_info_access.as_ref()),
"certificatePoliciesOid": ext.certificate_policies_oid,
"ipResources": serde_json::to_value(&ext.ip_resources).unwrap_or(Value::Null),
"asResources": serde_json::to_value(&ext.as_resources).unwrap_or(Value::Null),
})
}
fn subject_info_access_json(value: Option<&SubjectInfoAccess>) -> Value {
match value {
None => Value::Null,
Some(SubjectInfoAccess::Ca(ca)) => json!({
"kind": "ca",
"accessDescriptions": ca.access_descriptions.iter().map(access_description_json).collect::<Vec<_>>(),
}),
Some(SubjectInfoAccess::Ee(ee)) => json!({
"kind": "ee",
"signedObjectUris": ee.signed_object_uris,
"accessDescriptions": ee.access_descriptions.iter().map(access_description_json).collect::<Vec<_>>(),
}),
}
}
fn access_description_json(value: &AccessDescription) -> Value {
json!({
"accessMethodOid": value.access_method_oid,
"accessLocation": value.access_location,
})
}
fn signed_object_json(signed_object: &RpkiSignedObject) -> Value {
let signed_data = &signed_object.signed_data;
json!({
"rawDer": bytes_summary(&signed_object.raw_der),
"contentInfoContentType": signed_object.content_info_content_type,
"signedData": {
"version": signed_data.version,
"digestAlgorithms": signed_data.digest_algorithms,
"encapContentInfo": {
"eContentType": signed_data.encap_content_info.econtent_type,
"eContent": bytes_summary(&signed_data.encap_content_info.econtent),
},
"certificates": signed_data.certificates.iter().map(ee_certificate_json).collect::<Vec<_>>(),
"crlsPresent": signed_data.crls_present,
"signerInfos": signed_data.signer_infos.iter().map(signer_info_json).collect::<Vec<_>>(),
},
})
}
fn ee_certificate_json(cert: &ResourceEeCertificate) -> Value {
json!({
"rawDer": bytes_summary(&cert.raw_der),
"subjectKeyIdentifier": hex::encode(&cert.subject_key_identifier),
"spkiDer": bytes_summary(&cert.spki_der),
"rsaPublicKey": {
"modulus": bytes_summary(&cert.rsa_public_modulus),
"exponent": bytes_summary(&cert.rsa_public_exponent),
},
"tbsCertificate": bytes_summary(&cert.tbs_certificate_der),
"certificateSignature": bytes_summary(&cert.signature_bytes),
"keyUsageSummary": format!("{:?}", cert.key_usage_summary),
"siaSignedObjectUris": cert.sia_signed_object_uris,
"resourceCertificate": resource_certificate_json(&cert.resource_cert),
})
}
fn signer_info_json(info: &SignerInfoProfiled) -> Value {
json!({
"version": info.version,
"sidSki": hex::encode(&info.sid_ski),
"digestAlgorithm": info.digest_algorithm,
"signatureAlgorithm": info.signature_algorithm,
"signedAttrs": signed_attrs_json(&info.signed_attrs),
"unsignedAttrsPresent": info.unsigned_attrs_present,
"signature": bytes_summary(&info.signature),
"signedAttrsDerForSignature": bytes_summary(&info.signed_attrs_der_for_signature),
})
}
fn signed_attrs_json(attrs: &SignedAttrsProfiled) -> Value {
json!({
"contentType": attrs.content_type,
"messageDigest": hex::encode(&attrs.message_digest),
"signingTime": {
"utc": format_time(attrs.signing_time.utc),
"encoding": format!("{:?}", attrs.signing_time.encoding),
},
"otherAttrsPresent": attrs.other_attrs_present,
})
}
fn result_json(result: Result<(), String>) -> Value {
match result {
Ok(()) => json!({"valid": true}),
Err(err) => json!({"valid": false, "error": err}),
}
}
fn object_type_label(object_type: ObjectType) -> &'static str {
match object_type {
ObjectType::Auto => "auto",
ObjectType::Cer => "cer",
ObjectType::Mft => "mft",
ObjectType::Crl => "crl",
ObjectType::Roa => "roa",
ObjectType::Aspa => "aspa",
}
}
fn bytes_summary(bytes: &[u8]) -> Value {
let head_len = bytes.len().min(16);
let tail_len = bytes.len().min(16);
json!({
"len": bytes.len(),
"sha256": sha256_hex(bytes),
"headHex": hex::encode(&bytes[..head_len]),
"tailHex": hex::encode(&bytes[bytes.len().saturating_sub(tail_len)..]),
})
}
fn sha256_hex(bytes: &[u8]) -> String {
hex::encode(Sha256::digest(bytes))
}
fn format_time(value: time::OffsetDateTime) -> String {
value
.to_offset(time::UtcOffset::UTC)
.format(&time::format_description::well_known::Rfc3339)
.unwrap_or_else(|_| value.unix_timestamp().to_string())
}
fn roa_prefix_string(prefix: &RoaIpPrefix) -> String {
let bytes = prefix.addr_bytes();
match prefix.afi {
RoaAfi::Ipv4 => {
let octets = [bytes[0], bytes[1], bytes[2], bytes[3]];
format!("{}/{}", Ipv4Addr::from(octets), prefix.prefix_len)
}
RoaAfi::Ipv6 => {
let mut octets = [0u8; 16];
octets.copy_from_slice(bytes);
format!("{}/{}", Ipv6Addr::from(octets), prefix.prefix_len)
}
}
}
#[cfg(test)]
mod tests {
use std::path::Path;
use super::*;
#[test]
fn object_type_parser_and_resolver_cover_aliases() {
assert_eq!(ObjectType::parse("auto").unwrap(), ObjectType::Auto);
assert_eq!(ObjectType::parse(".cer").unwrap(), ObjectType::Cer);
assert_eq!(ObjectType::parse("certificate").unwrap(), ObjectType::Cer);
assert_eq!(ObjectType::parse("manifest").unwrap(), ObjectType::Mft);
assert_eq!(ObjectType::parse(".crl").unwrap(), ObjectType::Crl);
assert_eq!(ObjectType::parse("roa").unwrap(), ObjectType::Roa);
assert_eq!(ObjectType::parse("aspa").unwrap(), ObjectType::Aspa);
assert_eq!(ObjectType::parse(".asa").unwrap(), ObjectType::Aspa);
assert!(ObjectType::parse("unknown").is_err());
assert_eq!(ObjectType::Aspa.label(), "aspa");
assert_eq!(
resolve_object_type(ObjectType::Auto, Path::new("repo/a.cer")).unwrap(),
ObjectType::Cer
);
assert_eq!(
resolve_object_type(ObjectType::Auto, Path::new("repo/a.mft")).unwrap(),
ObjectType::Mft
);
assert_eq!(
resolve_object_type(ObjectType::Auto, Path::new("repo/a.crl")).unwrap(),
ObjectType::Crl
);
assert_eq!(
resolve_object_type(ObjectType::Auto, Path::new("repo/a.roa")).unwrap(),
ObjectType::Roa
);
assert_eq!(
resolve_object_type(ObjectType::Auto, Path::new("repo/a.asa")).unwrap(),
ObjectType::Aspa
);
assert_eq!(
resolve_object_type(ObjectType::Roa, Path::new("repo/a.bin")).unwrap(),
ObjectType::Roa
);
assert!(resolve_object_type(ObjectType::Auto, Path::new("repo/a.bin")).is_err());
}
#[test]
fn invalid_der_returns_error_projection_for_all_object_types() {
let bytes = b"not der";
for object_type in [
ObjectType::Cer,
ObjectType::Mft,
ObjectType::Crl,
ObjectType::Roa,
ObjectType::Aspa,
] {
let value = parse_object_json(object_type, Path::new("bad.der"), bytes, 1);
assert_eq!(
value["object"]["decode"]["profileValid"].as_bool(),
Some(false)
);
assert!(value["object"]["decode"]["error"].as_str().is_some());
}
let record = build_object_projection(ObjectType::Auto, Path::new("bad.bin"), bytes, 1);
assert_eq!(record.object_type, "unknown");
assert_eq!(record.parse_status, "error");
assert!(record.error_summary.is_some());
}
#[test]
fn parses_fixture_objects_into_human_readable_projection() {
let cases = [
(
ObjectType::Cer,
"tests/fixtures/ta/apnic-ta.cer",
"cer",
"resourceCertificate",
),
(
ObjectType::Mft,
"tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft",
"mft",
"manifest",
),
(
ObjectType::Crl,
"tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl",
"crl",
"revokedCertificates",
),
(
ObjectType::Roa,
"tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa",
"roa",
"roa",
),
(
ObjectType::Aspa,
"tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/5m80fwYws_3FiFD7JiQjAqZ1RYQ.asa",
"aspa",
"aspa",
),
];
for (object_type, path, expected_type, expected_section) in cases {
let bytes = std::fs::read(path).expect("fixture");
let record = build_object_projection(object_type, Path::new(path), &bytes, 1);
assert_eq!(record.object_type, expected_type);
assert_eq!(record.parse_status, "ok");
assert_eq!(
record.projection["object"]["decode"]["profileValid"].as_bool(),
Some(true)
);
assert!(record.projection["object"][expected_section].is_object());
}
}
#[test]
fn large_projection_lists_are_paged_from_raw_bytes() {
let mft_bytes = std::fs::read(
"tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft",
)
.expect("mft");
let (total, page) = manifest_file_entries_page(&mft_bytes, 1, 3).expect("mft page");
assert!(total >= 3);
assert_eq!(page.len(), 3);
assert!(page[0]["fileName"].as_str().is_some());
let (_, empty_page) =
manifest_file_entries_page(&mft_bytes, total + 10, 3).expect("empty page");
assert!(empty_page.is_empty());
let crl_bytes = std::fs::read(
"tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl",
)
.expect("crl");
let (total, page) = crl_revoked_entries_page(&crl_bytes, 0, 5).expect("crl page");
assert!(page.len() <= total);
let (_, empty_page) =
crl_revoked_entries_page(&crl_bytes, total + 10, 5).expect("empty crl page");
assert!(empty_page.is_empty());
}
}

139
src/query/artifact_manifest.rs Normal file
View File

@ -0,0 +1,139 @@
use std::collections::BTreeMap;
use std::fs;
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::query_db::QueryDbResult;
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ArtifactFileSummary {
pub path: String,
pub size_bytes: Option<u64>,
pub modified_unix_secs: Option<u64>,
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct QueryAuditArtifactSummary {
pub status: Option<String>,
pub events_path: Option<String>,
pub events_count: Option<u64>,
pub events_sha256: Option<String>,
pub writer_version: Option<u64>,
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ArtifactManifestSummary {
pub files: BTreeMap<String, ArtifactFileSummary>,
pub query_audit: Option<QueryAuditArtifactSummary>,
pub cir_counts_available: bool,
pub validation_events_indexed: bool,
}
impl ArtifactManifestSummary {
pub fn artifact_paths(&self) -> BTreeMap<String, String> {
self.files
.iter()
.map(|(name, item)| (name.clone(), item.path.clone()))
.collect()
}
}
pub fn build_artifact_manifest(
run_dir: &Path,
query_audit: Option<&Value>,
) -> QueryDbResult<ArtifactManifestSummary> {
let mut manifest = ArtifactManifestSummary {
query_audit: query_audit.map(query_audit_summary),
cir_counts_available: false,
validation_events_indexed: false,
..ArtifactManifestSummary::default()
};
for name in [
"report.json",
"input.cir",
"result.cir",
"result.ccr",
"stage-timing.json",
"run-summary.json",
] {
let path = run_dir.join(name);
if path.exists() {
manifest
.files
.insert(name.to_string(), file_summary(&path)?);
}
}
if let Some(events_path) = manifest
.query_audit
.as_ref()
.and_then(|query_audit| query_audit.events_path.as_ref())
{
let path = sidecar_path(run_dir, events_path);
if path.exists() {
manifest
.files
.insert("validationEvents".to_string(), file_summary(&path)?);
}
}
Ok(manifest)
}
fn file_summary(path: &Path) -> QueryDbResult<ArtifactFileSummary> {
let metadata = fs::metadata(path)?;
let modified_unix_secs = metadata
.modified()
.ok()
.and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok())
.map(|duration| duration.as_secs());
Ok(ArtifactFileSummary {
path: path.display().to_string(),
size_bytes: Some(metadata.len()),
modified_unix_secs,
})
}
fn query_audit_summary(value: &Value) -> QueryAuditArtifactSummary {
QueryAuditArtifactSummary {
status: json_str(value, &["status"]).map(str::to_string),
events_path: json_str(value, &["eventsPath"])
.or_else(|| json_str(value, &["events_path"]))
.map(str::to_string),
events_count: json_u64(value, &["eventsCount"])
.or_else(|| json_u64(value, &["events_count"])),
events_sha256: json_str(value, &["eventsSha256"])
.or_else(|| json_str(value, &["events_sha256"]))
.map(str::to_string),
writer_version: json_u64(value, &["writerVersion"])
.or_else(|| json_u64(value, &["writer_version"])),
}
}
pub fn sidecar_path(run_dir: &Path, path: &str) -> PathBuf {
let path = PathBuf::from(path);
if path.is_absolute() {
path
} else {
run_dir.join(path)
}
}
fn json_str<'a>(value: &'a Value, path: &[&str]) -> Option<&'a str> {
let mut current = value;
for key in path {
current = current.get(*key)?;
}
current.as_str()
}
fn json_u64(value: &Value, path: &[&str]) -> Option<u64> {
let mut current = value;
for key in path {
current = current.get(*key)?;
}
current.as_u64()
}

3
src/query/mod.rs Normal file
View File

@ -0,0 +1,3 @@
pub mod artifact_manifest;
pub mod object_resolver;
pub mod report_stream;

16
src/query/object_resolver.rs Normal file
View File

@ -0,0 +1,16 @@
use std::path::{Path, PathBuf};
use crate::query_db::{ObjectInstanceRecord, QueryDb, QueryDbResult};
pub fn report_path_for_run(db: &QueryDb, run_id: &str) -> QueryDbResult<Option<PathBuf>> {
Ok(db
.get_run(run_id)?
.map(|run| Path::new(&run.run_dir).join("report.json")))
}
pub fn resolve_object_from_cache_or_report(
_db: &QueryDb,
_run_id: &str,
) -> QueryDbResult<Option<ObjectInstanceRecord>> {
Ok(None)
}

1206
src/query/report_stream.rs Normal file
View File

File diff suppressed because it is too large Load Diff

1992
src/query_db.rs Normal file
View File

File diff suppressed because it is too large Load Diff