1630 lines
54 KiB
Rust
1630 lines
54 KiB
Rust
use std::collections::BTreeMap;
|
|
use std::fs::File;
|
|
use std::io::BufReader;
|
|
use std::path::Path;
|
|
|
|
use serde::de::{self, DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor};
|
|
use serde::{Deserialize, Deserializer, Serialize};
|
|
use serde_json::value::RawValue;
|
|
use serde_json::{Value, json};
|
|
use sha2::{Digest, Sha256};
|
|
|
|
use crate::query::vrp::VrpRecord;
|
|
use crate::query_db::{
|
|
ObjectInstanceRecord, PublicationPointRecord, QUERY_DB_SCHEMA_VERSION, QueryDbError,
|
|
QueryDbResult, QueryPage, RepositoryRecord, StatsRecord,
|
|
};
|
|
|
|
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
|
pub struct ReportSummary {
|
|
pub validation_time: Option<String>,
|
|
pub publication_points: Vec<PublicationPointRecord>,
|
|
pub repos: Vec<RepositoryRecord>,
|
|
pub stats: ReportStatsSummary,
|
|
pub query_audit: Option<Value>,
|
|
pub download_stats: Option<Value>,
|
|
pub vrps_count: u64,
|
|
pub vrps: Vec<VrpRecord>,
|
|
pub aspas_count: u64,
|
|
pub warnings_count: u64,
|
|
pub objects_count: u64,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
|
pub struct ReportStatsSummary {
|
|
pub object_type_counts: BTreeMap<String, u64>,
|
|
pub result_counts: BTreeMap<String, u64>,
|
|
pub source_counts: BTreeMap<String, u64>,
|
|
pub reason_counts: BTreeMap<String, u64>,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub enum ObjectScope {
|
|
All,
|
|
Repo(String),
|
|
PublicationPoint(String),
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub enum ObjectLookup {
|
|
InstanceId(String),
|
|
Uri(String),
|
|
Sha256(String),
|
|
}
|
|
|
|
/// Optional predicates applied while streaming report objects. `reason` and
|
|
/// `query` must be pre-lowercased by the caller; they are matched as
|
|
/// case-insensitive substrings. The other fields are exact matches.
|
|
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
|
pub struct ObjectFilter {
|
|
pub object_type: Option<String>,
|
|
pub result: Option<String>,
|
|
pub rejected: Option<bool>,
|
|
pub reason: Option<String>,
|
|
pub query: Option<String>,
|
|
pub repo_id: Option<String>,
|
|
pub pp_id: Option<String>,
|
|
pub issues_only: bool,
|
|
}
|
|
|
|
impl ObjectFilter {
|
|
pub fn matches(&self, object: &ObjectInstanceRecord) -> bool {
|
|
if self.issues_only && !(object.rejected || object.result == "error") {
|
|
return false;
|
|
}
|
|
if let Some(rejected) = self.rejected
|
|
&& object.rejected != rejected
|
|
{
|
|
return false;
|
|
}
|
|
if let Some(result) = self.result.as_deref()
|
|
&& !object.result.eq_ignore_ascii_case(result)
|
|
{
|
|
return false;
|
|
}
|
|
if let Some(object_type) = self.object_type.as_deref()
|
|
&& !object.object_type.eq_ignore_ascii_case(object_type)
|
|
{
|
|
return false;
|
|
}
|
|
if let Some(repo_id) = self.repo_id.as_deref()
|
|
&& object.repo_id != repo_id
|
|
{
|
|
return false;
|
|
}
|
|
if let Some(pp_id) = self.pp_id.as_deref()
|
|
&& object.pp_id != pp_id
|
|
{
|
|
return false;
|
|
}
|
|
if let Some(reason) = self.reason.as_deref() {
|
|
let Some(reject_reason) = object.reject_reason.as_deref() else {
|
|
return false;
|
|
};
|
|
if !reject_reason.to_lowercase().contains(reason) {
|
|
return false;
|
|
}
|
|
}
|
|
if let Some(query) = self.query.as_deref()
|
|
&& !object.uri.to_lowercase().contains(query)
|
|
{
|
|
return false;
|
|
}
|
|
true
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
pub struct ObjectResolutionMeta {
|
|
pub mode: String,
|
|
pub cache_hit: bool,
|
|
pub scan_scope: String,
|
|
pub scanned_publication_points: u64,
|
|
pub scanned_objects: u64,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct ObjectLookupResult {
|
|
pub object: ObjectInstanceRecord,
|
|
pub resolution: ObjectResolutionMeta,
|
|
}
|
|
|
|
pub fn summarize_report(report_path: &Path, run_id: &str) -> QueryDbResult<ReportSummary> {
|
|
let file = File::open(report_path)?;
|
|
let reader = BufReader::new(file);
|
|
let mut deserializer = serde_json::Deserializer::from_reader(reader);
|
|
ReportSummarySeed { run_id }
|
|
.deserialize(&mut deserializer)
|
|
.map_err(QueryDbError::from)
|
|
}
|
|
|
|
pub fn list_report_objects(
|
|
report_path: &Path,
|
|
run_id: &str,
|
|
scope: ObjectScope,
|
|
limit: usize,
|
|
cursor: Option<&str>,
|
|
) -> QueryDbResult<QueryPage<ObjectInstanceRecord>> {
|
|
list_report_objects_filtered(
|
|
report_path,
|
|
run_id,
|
|
scope,
|
|
&ObjectFilter::default(),
|
|
limit,
|
|
cursor,
|
|
)
|
|
}
|
|
|
|
pub fn list_report_objects_filtered(
|
|
report_path: &Path,
|
|
run_id: &str,
|
|
scope: ObjectScope,
|
|
filter: &ObjectFilter,
|
|
limit: usize,
|
|
cursor: Option<&str>,
|
|
) -> QueryDbResult<QueryPage<ObjectInstanceRecord>> {
|
|
let (start_pp_index, start_object_index) = parse_object_cursor(cursor)?;
|
|
let file = File::open(report_path)?;
|
|
let reader = BufReader::new(file);
|
|
let mut deserializer = serde_json::Deserializer::from_reader(reader);
|
|
let mut state = ObjectScanState::new_list(
|
|
run_id,
|
|
scope,
|
|
filter.clone(),
|
|
limit,
|
|
start_pp_index,
|
|
start_object_index,
|
|
);
|
|
let scan = ObjectScanSeed { state: &mut state }.deserialize(&mut deserializer);
|
|
match scan {
|
|
Ok(()) => {}
|
|
// A full page plus its next cursor is all the caller needs; the
|
|
// visitor aborts the remaining scan with a sentinel error.
|
|
Err(err) if is_object_scan_complete(&err) => {}
|
|
Err(err) => return Err(QueryDbError::from(err)),
|
|
}
|
|
Ok(QueryPage {
|
|
data: state.data,
|
|
next_cursor: state.next_cursor,
|
|
limit,
|
|
})
|
|
}
|
|
|
|
pub fn lookup_report_object(
|
|
report_path: &Path,
|
|
run_id: &str,
|
|
scope: ObjectScope,
|
|
lookup: ObjectLookup,
|
|
) -> QueryDbResult<Option<ObjectLookupResult>> {
|
|
let file = File::open(report_path)?;
|
|
let reader = BufReader::new(file);
|
|
let mut deserializer = serde_json::Deserializer::from_reader(reader);
|
|
let mut state = ObjectScanState::new_lookup(run_id, scope, lookup);
|
|
ObjectScanSeed { state: &mut state }
|
|
.deserialize(&mut deserializer)
|
|
.map_err(QueryDbError::from)?;
|
|
Ok(state.found.map(|object| ObjectLookupResult {
|
|
object,
|
|
resolution: ObjectResolutionMeta {
|
|
mode: "report_scan".to_string(),
|
|
cache_hit: false,
|
|
scan_scope: state.scope.label(),
|
|
scanned_publication_points: state.scanned_publication_points,
|
|
scanned_objects: state.scanned_objects,
|
|
},
|
|
}))
|
|
}
|
|
|
|
pub fn object_cursor(pp_index: u64, object_index: u64) -> String {
|
|
format!("r1:{pp_index}:{object_index}")
|
|
}
|
|
|
|
/// Sentinel message used to abort the streaming object scan early once a list
|
|
/// page (plus its next cursor) has been collected. The deserializer is left
|
|
/// mid-document and dropped, which is fine because the caller only needs the
|
|
/// state accumulated so far.
|
|
const OBJECT_SCAN_COMPLETE_MSG: &str = "rpki report object scan complete: page collected";
|
|
|
|
fn is_object_scan_complete(err: &serde_json::Error) -> bool {
|
|
err.is_data() && err.to_string().starts_with(OBJECT_SCAN_COMPLETE_MSG)
|
|
}
|
|
|
|
fn parse_object_cursor(cursor: Option<&str>) -> QueryDbResult<(u64, u64)> {
|
|
let Some(cursor) = cursor else {
|
|
return Ok((0, 0));
|
|
};
|
|
let mut parts = cursor.split(':');
|
|
match (parts.next(), parts.next(), parts.next(), parts.next()) {
|
|
(Some("r1"), Some(pp), Some(object), None) => {
|
|
let pp = pp.parse::<u64>().map_err(|_| {
|
|
QueryDbError::InvalidArtifact(format!("invalid object cursor: {cursor}"))
|
|
})?;
|
|
let object = object.parse::<u64>().map_err(|_| {
|
|
QueryDbError::InvalidArtifact(format!("invalid object cursor: {cursor}"))
|
|
})?;
|
|
Ok((pp, object))
|
|
}
|
|
_ => Err(QueryDbError::InvalidArtifact(format!(
|
|
"invalid object cursor: {cursor}"
|
|
))),
|
|
}
|
|
}
|
|
|
|
struct ReportSummarySeed<'a> {
|
|
run_id: &'a str,
|
|
}
|
|
|
|
impl<'de> DeserializeSeed<'de> for ReportSummarySeed<'_> {
|
|
type Value = ReportSummary;
|
|
|
|
fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
|
|
where
|
|
D: Deserializer<'de>,
|
|
{
|
|
deserializer.deserialize_map(ReportSummaryVisitor {
|
|
run_id: self.run_id,
|
|
})
|
|
}
|
|
}
|
|
|
|
struct ReportSummaryVisitor<'a> {
|
|
run_id: &'a str,
|
|
}
|
|
|
|
impl<'de> Visitor<'de> for ReportSummaryVisitor<'_> {
|
|
type Value = ReportSummary;
|
|
|
|
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
formatter.write_str("audit report object")
|
|
}
|
|
|
|
fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
|
|
where
|
|
A: MapAccess<'de>,
|
|
{
|
|
let mut out = ReportSummary::default();
|
|
while let Some(key) = map.next_key::<String>()? {
|
|
match key.as_str() {
|
|
"meta" => {
|
|
let meta: Value = map.next_value()?;
|
|
out.validation_time =
|
|
json_str(&meta, &["validation_time_rfc3339_utc"]).map(str::to_string);
|
|
}
|
|
"tree" => {
|
|
let tree: Value = map.next_value()?;
|
|
out.warnings_count = json_array_len(&tree, &["warnings"]).unwrap_or(0);
|
|
}
|
|
"publication_points" => {
|
|
let pps = map.next_value_seed(PublicationPointsSummarySeed {
|
|
run_id: self.run_id,
|
|
})?;
|
|
out.objects_count = pps.objects_count;
|
|
out.publication_points = pps.publication_points;
|
|
out.repos = pps.repos;
|
|
out.stats = pps.stats;
|
|
}
|
|
"vrps" => {
|
|
let vrps = map.next_value_seed(VrpSequenceSeed {
|
|
run_id: self.run_id,
|
|
})?;
|
|
out.vrps_count = vrps.count;
|
|
out.vrps = vrps.records;
|
|
}
|
|
"aspas" => out.aspas_count = map.next_value::<CountSeq>()?.0,
|
|
"download_stats" => out.download_stats = Some(map.next_value()?),
|
|
"queryAudit" => out.query_audit = Some(map.next_value()?),
|
|
_ => {
|
|
let _ = map.next_value::<IgnoredAny>()?;
|
|
}
|
|
}
|
|
}
|
|
Ok(out)
|
|
}
|
|
}
|
|
|
|
struct PublicationPointsSummarySeed<'a> {
|
|
run_id: &'a str,
|
|
}
|
|
|
|
impl<'de> DeserializeSeed<'de> for PublicationPointsSummarySeed<'_> {
|
|
type Value = PublicationPointsSummary;
|
|
|
|
fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
|
|
where
|
|
D: Deserializer<'de>,
|
|
{
|
|
deserializer.deserialize_seq(PublicationPointsSummaryVisitor {
|
|
run_id: self.run_id,
|
|
})
|
|
}
|
|
}
|
|
|
|
#[derive(Default)]
|
|
struct PublicationPointsSummary {
|
|
publication_points: Vec<PublicationPointRecord>,
|
|
repos: Vec<RepositoryRecord>,
|
|
stats: ReportStatsSummary,
|
|
objects_count: u64,
|
|
}
|
|
|
|
#[derive(Deserialize)]
|
|
struct RawVrpRecord {
|
|
asn: u32,
|
|
prefix: String,
|
|
max_length: u8,
|
|
}
|
|
|
|
struct VrpSequenceSeed<'a> {
|
|
run_id: &'a str,
|
|
}
|
|
|
|
struct VrpSequence {
|
|
count: u64,
|
|
records: Vec<VrpRecord>,
|
|
}
|
|
|
|
impl<'de> DeserializeSeed<'de> for VrpSequenceSeed<'_> {
|
|
type Value = VrpSequence;
|
|
|
|
fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
|
|
where
|
|
D: Deserializer<'de>,
|
|
{
|
|
deserializer.deserialize_seq(VrpSequenceVisitor {
|
|
run_id: self.run_id,
|
|
})
|
|
}
|
|
}
|
|
|
|
struct VrpSequenceVisitor<'a> {
|
|
run_id: &'a str,
|
|
}
|
|
|
|
impl<'de> Visitor<'de> for VrpSequenceVisitor<'_> {
|
|
type Value = VrpSequence;
|
|
|
|
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
formatter.write_str("VRP array")
|
|
}
|
|
|
|
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
|
|
where
|
|
A: SeqAccess<'de>,
|
|
{
|
|
let mut count = 0u64;
|
|
let mut records = Vec::new();
|
|
while let Some(raw) = seq.next_element::<RawVrpRecord>()? {
|
|
let record = VrpRecord::from_report(self.run_id, raw.asn, &raw.prefix, raw.max_length)
|
|
.map_err(de::Error::custom)?;
|
|
count += 1;
|
|
records.push(record);
|
|
}
|
|
Ok(VrpSequence { count, records })
|
|
}
|
|
}
|
|
|
|
struct PublicationPointsSummaryVisitor<'a> {
|
|
run_id: &'a str,
|
|
}
|
|
|
|
impl<'de> Visitor<'de> for PublicationPointsSummaryVisitor<'_> {
|
|
type Value = PublicationPointsSummary;
|
|
|
|
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
formatter.write_str("publication point array")
|
|
}
|
|
|
|
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
|
|
where
|
|
A: SeqAccess<'de>,
|
|
{
|
|
let mut out = PublicationPointsSummary::default();
|
|
let mut repos = BTreeMap::<String, RepoBuilder>::new();
|
|
while let Some(pp) = seq.next_element::<PublicationPointSummary>()? {
|
|
let repo_uri = repo_uri_for_pp(&pp);
|
|
let repo_id = stable_id(&repo_uri);
|
|
let pp_id = pp_id_for(&pp);
|
|
let pp_record = pp.to_record(self.run_id, repo_id.clone(), pp_id);
|
|
repos
|
|
.entry(repo_id.clone())
|
|
.or_insert_with(|| RepoBuilder::new(repo_id, repo_uri))
|
|
.add_pp(&pp_record);
|
|
out.objects_count += pp_record.objects;
|
|
merge_counts(
|
|
&mut out.stats.object_type_counts,
|
|
pp.object_stats.object_type_counts,
|
|
);
|
|
merge_counts(&mut out.stats.result_counts, pp.object_stats.result_counts);
|
|
merge_counts(&mut out.stats.reason_counts, pp.object_stats.reason_counts);
|
|
*out.stats
|
|
.source_counts
|
|
.entry("report".to_string())
|
|
.or_default() += pp_record.objects;
|
|
out.publication_points.push(pp_record);
|
|
}
|
|
out.repos = repos
|
|
.into_values()
|
|
.map(|builder| builder.finish(self.run_id))
|
|
.collect();
|
|
Ok(out)
|
|
}
|
|
}
|
|
|
|
#[derive(Default, Deserialize)]
|
|
struct CountSeq(#[serde(deserialize_with = "count_seq")] u64);
|
|
|
|
fn count_seq<'de, D>(deserializer: D) -> Result<u64, D::Error>
|
|
where
|
|
D: Deserializer<'de>,
|
|
{
|
|
struct CountVisitor;
|
|
impl<'de> Visitor<'de> for CountVisitor {
|
|
type Value = u64;
|
|
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
formatter.write_str("array")
|
|
}
|
|
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
|
|
where
|
|
A: SeqAccess<'de>,
|
|
{
|
|
let mut count = 0u64;
|
|
while seq.next_element::<IgnoredAny>()?.is_some() {
|
|
count += 1;
|
|
}
|
|
Ok(count)
|
|
}
|
|
}
|
|
deserializer.deserialize_seq(CountVisitor)
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default)]
|
|
struct PublicationPointSummary {
|
|
node_id: Option<u64>,
|
|
parent_node_id: Option<u64>,
|
|
rsync_base_uri: Option<String>,
|
|
manifest_rsync_uri: Option<String>,
|
|
publication_point_rsync_uri: Option<String>,
|
|
rrdp_notification_uri: Option<String>,
|
|
source: Option<String>,
|
|
repo_sync_source: Option<String>,
|
|
repo_sync_phase: Option<String>,
|
|
repo_sync_duration_ms: Option<u64>,
|
|
repo_sync_error: Option<String>,
|
|
repo_terminal_state: Option<String>,
|
|
this_update: Option<String>,
|
|
next_update: Option<String>,
|
|
verified_at: Option<String>,
|
|
warnings: u64,
|
|
object_stats: ObjectStats,
|
|
}
|
|
|
|
impl PublicationPointSummary {
|
|
fn to_record(&self, run_id: &str, repo_id: String, pp_id: String) -> PublicationPointRecord {
|
|
PublicationPointRecord {
|
|
schema_version: QUERY_DB_SCHEMA_VERSION,
|
|
run_id: run_id.to_string(),
|
|
pp_id,
|
|
repo_id,
|
|
node_id: self.node_id,
|
|
parent_node_id: self.parent_node_id,
|
|
rsync_base_uri: self.rsync_base_uri.clone(),
|
|
manifest_rsync_uri: self.manifest_rsync_uri.clone(),
|
|
publication_point_rsync_uri: self.publication_point_rsync_uri.clone(),
|
|
rrdp_notification_uri: self.rrdp_notification_uri.clone(),
|
|
source: self.source.clone(),
|
|
repo_sync_source: self.repo_sync_source.clone(),
|
|
repo_sync_phase: self.repo_sync_phase.clone(),
|
|
repo_sync_duration_ms: self.repo_sync_duration_ms,
|
|
repo_sync_error: self.repo_sync_error.clone(),
|
|
repo_terminal_state: self.repo_terminal_state.clone(),
|
|
this_update: self.this_update.clone(),
|
|
next_update: self.next_update.clone(),
|
|
verified_at: self.verified_at.clone(),
|
|
objects: self.object_stats.count,
|
|
rejected_objects: self.object_stats.rejected,
|
|
warnings: self.warnings,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<'de> Deserialize<'de> for PublicationPointSummary {
|
|
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
|
where
|
|
D: Deserializer<'de>,
|
|
{
|
|
deserializer.deserialize_map(PublicationPointSummaryVisitor)
|
|
}
|
|
}
|
|
|
|
struct PublicationPointSummaryVisitor;
|
|
|
|
impl<'de> Visitor<'de> for PublicationPointSummaryVisitor {
|
|
type Value = PublicationPointSummary;
|
|
|
|
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
formatter.write_str("publication point object")
|
|
}
|
|
|
|
fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
|
|
where
|
|
A: MapAccess<'de>,
|
|
{
|
|
let mut pp = PublicationPointSummary::default();
|
|
while let Some(key) = map.next_key::<String>()? {
|
|
match key.as_str() {
|
|
"node_id" => pp.node_id = map.next_value()?,
|
|
"parent_node_id" => pp.parent_node_id = map.next_value()?,
|
|
"rsync_base_uri" => pp.rsync_base_uri = map.next_value()?,
|
|
"manifest_rsync_uri" => pp.manifest_rsync_uri = map.next_value()?,
|
|
"publication_point_rsync_uri" => {
|
|
pp.publication_point_rsync_uri = map.next_value()?
|
|
}
|
|
"rrdp_notification_uri" => pp.rrdp_notification_uri = map.next_value()?,
|
|
"source" => pp.source = map.next_value()?,
|
|
"repo_sync_source" => pp.repo_sync_source = map.next_value()?,
|
|
"repo_sync_phase" => pp.repo_sync_phase = map.next_value()?,
|
|
"repo_sync_duration_ms" => pp.repo_sync_duration_ms = map.next_value()?,
|
|
"repo_sync_error" => pp.repo_sync_error = map.next_value()?,
|
|
"repo_terminal_state" => pp.repo_terminal_state = map.next_value()?,
|
|
"this_update_rfc3339_utc" => pp.this_update = map.next_value()?,
|
|
"next_update_rfc3339_utc" => pp.next_update = map.next_value()?,
|
|
"verified_at_rfc3339_utc" => pp.verified_at = map.next_value()?,
|
|
"warnings" => pp.warnings = map.next_value::<CountSeq>()?.0,
|
|
"objects" => pp.object_stats = map.next_value::<ObjectStatsSeq>()?.0,
|
|
_ => {
|
|
let _ = map.next_value::<IgnoredAny>()?;
|
|
}
|
|
}
|
|
}
|
|
Ok(pp)
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default)]
|
|
struct ObjectStats {
|
|
count: u64,
|
|
rejected: u64,
|
|
object_type_counts: BTreeMap<String, u64>,
|
|
result_counts: BTreeMap<String, u64>,
|
|
reason_counts: BTreeMap<String, u64>,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default)]
|
|
struct ObjectStatsSeq(ObjectStats);
|
|
|
|
impl<'de> Deserialize<'de> for ObjectStatsSeq {
|
|
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
|
where
|
|
D: Deserializer<'de>,
|
|
{
|
|
struct ObjectStatsVisitor;
|
|
impl<'de> Visitor<'de> for ObjectStatsVisitor {
|
|
type Value = ObjectStatsSeq;
|
|
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
formatter.write_str("object audit array")
|
|
}
|
|
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
|
|
where
|
|
A: SeqAccess<'de>,
|
|
{
|
|
let mut stats = ObjectStats::default();
|
|
while let Some(object) = seq.next_element::<ReportObjectSummary>()? {
|
|
stats.count += 1;
|
|
if object.result == "error" {
|
|
stats.rejected += 1;
|
|
if let Some(detail) = object.detail.filter(|value| !value.is_empty()) {
|
|
bump_limited_count(&mut stats.reason_counts, &detail, 512);
|
|
}
|
|
}
|
|
*stats
|
|
.object_type_counts
|
|
.entry(object.object_type)
|
|
.or_default() += 1;
|
|
*stats.result_counts.entry(object.result).or_default() += 1;
|
|
}
|
|
Ok(ObjectStatsSeq(stats))
|
|
}
|
|
}
|
|
deserializer.deserialize_seq(ObjectStatsVisitor)
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default)]
|
|
struct ReportObjectSummary {
|
|
object_type: String,
|
|
result: String,
|
|
detail: Option<String>,
|
|
}
|
|
|
|
impl<'de> Deserialize<'de> for ReportObjectSummary {
|
|
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
|
where
|
|
D: Deserializer<'de>,
|
|
{
|
|
deserializer.deserialize_map(ReportObjectSummaryVisitor)
|
|
}
|
|
}
|
|
|
|
struct ReportObjectSummaryVisitor;
|
|
|
|
impl<'de> Visitor<'de> for ReportObjectSummaryVisitor {
|
|
type Value = ReportObjectSummary;
|
|
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
formatter.write_str("object summary")
|
|
}
|
|
fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
|
|
where
|
|
A: MapAccess<'de>,
|
|
{
|
|
let mut object = ReportObjectSummary {
|
|
object_type: "other".to_string(),
|
|
result: "unknown".to_string(),
|
|
detail: None,
|
|
};
|
|
let mut uri: Option<String> = None;
|
|
while let Some(key) = map.next_key::<String>()? {
|
|
match key.as_str() {
|
|
"kind" => {
|
|
object.object_type = map
|
|
.next_value::<Option<String>>()?
|
|
.unwrap_or_else(|| "other".to_string())
|
|
}
|
|
"result" => {
|
|
object.result = map
|
|
.next_value::<Option<String>>()?
|
|
.unwrap_or_else(|| "unknown".to_string())
|
|
}
|
|
"detail" => object.detail = map.next_value()?,
|
|
"rsync_uri" | "uri" => uri = map.next_value()?,
|
|
_ => {
|
|
let _ = map.next_value::<IgnoredAny>()?;
|
|
}
|
|
}
|
|
}
|
|
if object.object_type == "other" {
|
|
if let Some(uri) = uri.as_deref() {
|
|
object.object_type = object_type_from_uri(uri);
|
|
}
|
|
}
|
|
Ok(object)
|
|
}
|
|
}
|
|
|
|
struct ObjectScanState<'a> {
|
|
run_id: &'a str,
|
|
scope: ObjectScope,
|
|
mode: ObjectScanMode,
|
|
filter: ObjectFilter,
|
|
limit: usize,
|
|
start_pp_index: u64,
|
|
start_object_index: u64,
|
|
data: Vec<ObjectInstanceRecord>,
|
|
next_cursor: Option<String>,
|
|
found: Option<ObjectInstanceRecord>,
|
|
scanned_publication_points: u64,
|
|
scanned_objects: u64,
|
|
}
|
|
|
|
impl<'a> ObjectScanState<'a> {
|
|
fn new_list(
|
|
run_id: &'a str,
|
|
scope: ObjectScope,
|
|
filter: ObjectFilter,
|
|
limit: usize,
|
|
start_pp_index: u64,
|
|
start_object_index: u64,
|
|
) -> Self {
|
|
Self {
|
|
run_id,
|
|
scope,
|
|
mode: ObjectScanMode::List,
|
|
filter,
|
|
limit,
|
|
start_pp_index,
|
|
start_object_index,
|
|
data: Vec::new(),
|
|
next_cursor: None,
|
|
found: None,
|
|
scanned_publication_points: 0,
|
|
scanned_objects: 0,
|
|
}
|
|
}
|
|
|
|
fn new_lookup(run_id: &'a str, scope: ObjectScope, lookup: ObjectLookup) -> Self {
|
|
Self {
|
|
run_id,
|
|
scope,
|
|
mode: ObjectScanMode::Lookup(lookup),
|
|
filter: ObjectFilter::default(),
|
|
limit: 1,
|
|
start_pp_index: 0,
|
|
start_object_index: 0,
|
|
data: Vec::new(),
|
|
next_cursor: None,
|
|
found: None,
|
|
scanned_publication_points: 0,
|
|
scanned_objects: 0,
|
|
}
|
|
}
|
|
|
|
/// True once a list page is full and its next cursor is fixed; everything
|
|
/// after this point cannot alter the page (cursor semantics: the cursor
|
|
/// already points at the first item of the next page).
|
|
fn list_page_complete(&self) -> bool {
|
|
matches!(self.mode, ObjectScanMode::List)
|
|
&& self.data.len() >= self.limit
|
|
&& self.next_cursor.is_some()
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
enum ObjectScanMode {
|
|
List,
|
|
Lookup(ObjectLookup),
|
|
}
|
|
|
|
struct ObjectScanSeed<'a, 's> {
|
|
state: &'s mut ObjectScanState<'a>,
|
|
}
|
|
|
|
impl<'de> DeserializeSeed<'de> for ObjectScanSeed<'_, '_> {
|
|
type Value = ();
|
|
|
|
fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
|
|
where
|
|
D: Deserializer<'de>,
|
|
{
|
|
deserializer.deserialize_map(ObjectScanReportVisitor { state: self.state })
|
|
}
|
|
}
|
|
|
|
struct ObjectScanReportVisitor<'a, 's> {
|
|
state: &'s mut ObjectScanState<'a>,
|
|
}
|
|
|
|
impl<'de> Visitor<'de> for ObjectScanReportVisitor<'_, '_> {
|
|
type Value = ();
|
|
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
formatter.write_str("audit report object")
|
|
}
|
|
fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
|
|
where
|
|
A: MapAccess<'de>,
|
|
{
|
|
while let Some(key) = map.next_key::<String>()? {
|
|
if key == "publication_points" {
|
|
map.next_value_seed(ObjectScanPpsSeed { state: self.state })?;
|
|
} else {
|
|
let _ = map.next_value::<IgnoredAny>()?;
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
struct ObjectScanPpsSeed<'a, 's> {
|
|
state: &'s mut ObjectScanState<'a>,
|
|
}
|
|
|
|
impl<'de> DeserializeSeed<'de> for ObjectScanPpsSeed<'_, '_> {
|
|
type Value = ();
|
|
fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
|
|
where
|
|
D: Deserializer<'de>,
|
|
{
|
|
deserializer.deserialize_seq(ObjectScanPpsVisitor { state: self.state })
|
|
}
|
|
}
|
|
|
|
struct ObjectScanPpsVisitor<'a, 's> {
|
|
state: &'s mut ObjectScanState<'a>,
|
|
}
|
|
|
|
impl<'de> Visitor<'de> for ObjectScanPpsVisitor<'_, '_> {
|
|
type Value = ();
|
|
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
formatter.write_str("publication point array")
|
|
}
|
|
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
|
|
where
|
|
A: SeqAccess<'de>,
|
|
{
|
|
let mut pp_index = 0u64;
|
|
loop {
|
|
if self.state.list_page_complete() {
|
|
return Err(de::Error::custom(OBJECT_SCAN_COMPLETE_MSG));
|
|
}
|
|
let Some(raw_pp) = seq.next_element::<Box<RawValue>>()? else {
|
|
break;
|
|
};
|
|
process_raw_publication_point(self.state, pp_index, raw_pp.get())
|
|
.map_err(de::Error::custom)?;
|
|
pp_index += 1;
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
fn process_raw_publication_point(
|
|
state: &mut ObjectScanState<'_>,
|
|
pp_index: u64,
|
|
raw_pp: &str,
|
|
) -> QueryDbResult<()> {
|
|
state.scanned_publication_points += 1;
|
|
if matches!(state.mode, ObjectScanMode::List) && pp_index < state.start_pp_index {
|
|
// Every object in this PP sits before the cursor, so none of them can
|
|
// be part of the page: skip materializing the subtree entirely.
|
|
return Ok(());
|
|
}
|
|
if state.scope != ObjectScope::All {
|
|
// Scope is decidable from PP-level fields alone; probe them before
|
|
// materializing the (potentially huge) objects subtree.
|
|
let probe: PublicationPointScopeProbe = serde_json::from_str(raw_pp)?;
|
|
let ctx = PpObjectContext::from_pp(state.run_id, &probe.into_summary());
|
|
if !state.scope.matches(&ctx) {
|
|
return Ok(());
|
|
}
|
|
}
|
|
let pp: PublicationPointForObjects = serde_json::from_str(raw_pp)?;
|
|
let ctx = PpObjectContext::from_pp(state.run_id, &pp.summary);
|
|
if !state.scope.matches(&ctx) {
|
|
return Ok(());
|
|
}
|
|
for (object_index, object) in pp.objects.into_iter().enumerate() {
|
|
state.scanned_objects += 1;
|
|
let record = object.to_record(state.run_id, &ctx);
|
|
match &state.mode {
|
|
ObjectScanMode::List => {
|
|
let object_index = object_index as u64;
|
|
let after_cursor = pp_index > state.start_pp_index
|
|
|| (pp_index == state.start_pp_index
|
|
&& object_index >= state.start_object_index);
|
|
if after_cursor && state.data.len() < state.limit && state.filter.matches(&record) {
|
|
state.data.push(record);
|
|
if state.data.len() == state.limit {
|
|
state.next_cursor = Some(object_cursor(pp_index, object_index + 1));
|
|
}
|
|
}
|
|
}
|
|
ObjectScanMode::Lookup(lookup) => {
|
|
if state.found.is_none() && lookup.matches(&record) {
|
|
state.found = Some(record);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default)]
|
|
struct PublicationPointForObjects {
|
|
summary: PublicationPointSummary,
|
|
objects: Vec<ReportObject>,
|
|
}
|
|
|
|
impl<'de> Deserialize<'de> for PublicationPointForObjects {
|
|
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
|
where
|
|
D: Deserializer<'de>,
|
|
{
|
|
deserializer.deserialize_map(PublicationPointForObjectsVisitor)
|
|
}
|
|
}
|
|
|
|
struct PublicationPointForObjectsVisitor;
|
|
|
|
impl<'de> Visitor<'de> for PublicationPointForObjectsVisitor {
|
|
type Value = PublicationPointForObjects;
|
|
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
formatter.write_str("publication point object for object scan")
|
|
}
|
|
fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
|
|
where
|
|
A: MapAccess<'de>,
|
|
{
|
|
let mut pp = PublicationPointForObjects::default();
|
|
while let Some(key) = map.next_key::<String>()? {
|
|
match key.as_str() {
|
|
"node_id" => pp.summary.node_id = map.next_value()?,
|
|
"parent_node_id" => pp.summary.parent_node_id = map.next_value()?,
|
|
"rsync_base_uri" => pp.summary.rsync_base_uri = map.next_value()?,
|
|
"manifest_rsync_uri" => pp.summary.manifest_rsync_uri = map.next_value()?,
|
|
"publication_point_rsync_uri" => {
|
|
pp.summary.publication_point_rsync_uri = map.next_value()?
|
|
}
|
|
"rrdp_notification_uri" => pp.summary.rrdp_notification_uri = map.next_value()?,
|
|
"source" => pp.summary.source = map.next_value()?,
|
|
"repo_sync_source" => pp.summary.repo_sync_source = map.next_value()?,
|
|
"repo_sync_phase" => pp.summary.repo_sync_phase = map.next_value()?,
|
|
"repo_sync_duration_ms" => pp.summary.repo_sync_duration_ms = map.next_value()?,
|
|
"repo_sync_error" => pp.summary.repo_sync_error = map.next_value()?,
|
|
"repo_terminal_state" => pp.summary.repo_terminal_state = map.next_value()?,
|
|
"this_update_rfc3339_utc" => pp.summary.this_update = map.next_value()?,
|
|
"next_update_rfc3339_utc" => pp.summary.next_update = map.next_value()?,
|
|
"verified_at_rfc3339_utc" => pp.summary.verified_at = map.next_value()?,
|
|
"objects" => pp.objects = map.next_value()?,
|
|
_ => {
|
|
let _ = map.next_value::<IgnoredAny>()?;
|
|
}
|
|
}
|
|
}
|
|
Ok(pp)
|
|
}
|
|
}
|
|
|
|
/// Lightweight probe of the PP-level fields needed to decide `ObjectScope`
|
|
/// before the (potentially huge) objects subtree is materialized.
|
|
#[derive(Clone, Debug, Default, Deserialize)]
|
|
struct PublicationPointScopeProbe {
|
|
node_id: Option<u64>,
|
|
rsync_base_uri: Option<String>,
|
|
manifest_rsync_uri: Option<String>,
|
|
publication_point_rsync_uri: Option<String>,
|
|
rrdp_notification_uri: Option<String>,
|
|
}
|
|
|
|
impl PublicationPointScopeProbe {
|
|
fn into_summary(self) -> PublicationPointSummary {
|
|
PublicationPointSummary {
|
|
node_id: self.node_id,
|
|
rsync_base_uri: self.rsync_base_uri,
|
|
manifest_rsync_uri: self.manifest_rsync_uri,
|
|
publication_point_rsync_uri: self.publication_point_rsync_uri,
|
|
rrdp_notification_uri: self.rrdp_notification_uri,
|
|
..PublicationPointSummary::default()
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default)]
|
|
struct ReportObject {
|
|
uri: String,
|
|
sha256: String,
|
|
object_type: String,
|
|
result: String,
|
|
detail: Option<String>,
|
|
}
|
|
|
|
impl ReportObject {
|
|
fn to_record(self, run_id: &str, ctx: &PpObjectContext) -> ObjectInstanceRecord {
|
|
let object_type = if self.object_type.is_empty() || self.object_type == "other" {
|
|
object_type_from_uri(&self.uri)
|
|
} else {
|
|
self.object_type
|
|
};
|
|
let uri_hash = stable_id(&self.uri);
|
|
let object_instance_id = stable_id(&format!(
|
|
"{run_id}\0{}\0{}\0{}",
|
|
self.uri, self.sha256, ctx.pp_id
|
|
));
|
|
let rejected = self.result == "error";
|
|
ObjectInstanceRecord {
|
|
schema_version: QUERY_DB_SCHEMA_VERSION,
|
|
run_id: run_id.to_string(),
|
|
object_instance_id,
|
|
uri: self.uri,
|
|
uri_hash,
|
|
sha256: self.sha256,
|
|
object_type,
|
|
result: self.result,
|
|
detail_summary: self.detail.clone(),
|
|
repo_id: ctx.repo_id.clone(),
|
|
pp_id: ctx.pp_id.clone(),
|
|
source_section: "report".to_string(),
|
|
rejected,
|
|
reject_reason: if rejected { self.detail } else { None },
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<'de> Deserialize<'de> for ReportObject {
|
|
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
|
where
|
|
D: Deserializer<'de>,
|
|
{
|
|
deserializer.deserialize_map(ReportObjectVisitor)
|
|
}
|
|
}
|
|
|
|
struct ReportObjectVisitor;
|
|
|
|
impl<'de> Visitor<'de> for ReportObjectVisitor {
|
|
type Value = ReportObject;
|
|
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
formatter.write_str("report object")
|
|
}
|
|
fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
|
|
where
|
|
A: MapAccess<'de>,
|
|
{
|
|
let mut object = ReportObject {
|
|
result: "unknown".to_string(),
|
|
object_type: "other".to_string(),
|
|
..ReportObject::default()
|
|
};
|
|
while let Some(key) = map.next_key::<String>()? {
|
|
match key.as_str() {
|
|
"rsync_uri" | "uri" => {
|
|
object.uri = map.next_value::<Option<String>>()?.unwrap_or_default()
|
|
}
|
|
"sha256_hex" | "sha256" => {
|
|
object.sha256 = map.next_value::<Option<String>>()?.unwrap_or_default()
|
|
}
|
|
"kind" => {
|
|
object.object_type = map
|
|
.next_value::<Option<String>>()?
|
|
.unwrap_or_else(|| "other".to_string())
|
|
}
|
|
"result" => {
|
|
object.result = map
|
|
.next_value::<Option<String>>()?
|
|
.unwrap_or_else(|| "unknown".to_string())
|
|
}
|
|
"detail" => object.detail = map.next_value()?,
|
|
_ => {
|
|
let _ = map.next_value::<IgnoredAny>()?;
|
|
}
|
|
}
|
|
}
|
|
Ok(object)
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default)]
|
|
struct PpObjectContext {
|
|
repo_id: String,
|
|
pp_id: String,
|
|
}
|
|
|
|
impl PpObjectContext {
|
|
fn from_pp(_run_id: &str, pp: &PublicationPointSummary) -> Self {
|
|
let repo_uri = repo_uri_for_pp(pp);
|
|
Self {
|
|
repo_id: stable_id(&repo_uri),
|
|
pp_id: pp_id_for(pp),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl ObjectScope {
|
|
fn matches(&self, ctx: &PpObjectContext) -> bool {
|
|
match self {
|
|
ObjectScope::All => true,
|
|
ObjectScope::Repo(repo_id) => repo_id == &ctx.repo_id,
|
|
ObjectScope::PublicationPoint(pp_id) => pp_id == &ctx.pp_id,
|
|
}
|
|
}
|
|
|
|
fn label(&self) -> String {
|
|
match self {
|
|
ObjectScope::All => "all".to_string(),
|
|
ObjectScope::Repo(_) => "repo".to_string(),
|
|
ObjectScope::PublicationPoint(_) => "publication_point".to_string(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl ObjectLookup {
|
|
fn matches(&self, object: &ObjectInstanceRecord) -> bool {
|
|
match self {
|
|
ObjectLookup::InstanceId(id) => id == &object.object_instance_id,
|
|
ObjectLookup::Uri(uri) => uri == &object.uri,
|
|
ObjectLookup::Sha256(sha256) => sha256.eq_ignore_ascii_case(&object.sha256),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug)]
|
|
struct RepoBuilder {
|
|
repo_id: String,
|
|
uri: String,
|
|
publication_points: u64,
|
|
objects: u64,
|
|
rejected_objects: u64,
|
|
sync_duration_ms_total: u64,
|
|
phases: BTreeMap<String, u64>,
|
|
terminal_states: BTreeMap<String, u64>,
|
|
}
|
|
|
|
impl RepoBuilder {
|
|
fn new(repo_id: String, uri: String) -> Self {
|
|
Self {
|
|
repo_id,
|
|
uri,
|
|
publication_points: 0,
|
|
objects: 0,
|
|
rejected_objects: 0,
|
|
sync_duration_ms_total: 0,
|
|
phases: BTreeMap::new(),
|
|
terminal_states: BTreeMap::new(),
|
|
}
|
|
}
|
|
|
|
fn add_pp(&mut self, pp: &PublicationPointRecord) {
|
|
self.publication_points += 1;
|
|
self.objects += pp.objects;
|
|
self.rejected_objects += pp.rejected_objects;
|
|
self.sync_duration_ms_total = self
|
|
.sync_duration_ms_total
|
|
.saturating_add(pp.repo_sync_duration_ms.unwrap_or(0));
|
|
if let Some(phase) = pp.repo_sync_phase.as_ref() {
|
|
*self.phases.entry(phase.clone()).or_default() += 1;
|
|
}
|
|
if let Some(state) = pp.repo_terminal_state.as_ref() {
|
|
*self.terminal_states.entry(state.clone()).or_default() += 1;
|
|
}
|
|
}
|
|
|
|
fn finish(self, run_id: &str) -> RepositoryRecord {
|
|
RepositoryRecord {
|
|
schema_version: QUERY_DB_SCHEMA_VERSION,
|
|
run_id: run_id.to_string(),
|
|
repo_id: self.repo_id,
|
|
uri: self.uri.clone(),
|
|
host: uri_host(&self.uri),
|
|
transport: infer_transport(&self.uri),
|
|
publication_points: self.publication_points,
|
|
objects: self.objects,
|
|
rejected_objects: self.rejected_objects,
|
|
download_bytes: None,
|
|
sync_duration_ms_total: self.sync_duration_ms_total,
|
|
phases: self.phases,
|
|
terminal_states: self.terminal_states,
|
|
}
|
|
}
|
|
}
|
|
|
|
pub fn stats_records_from_summary(
|
|
run_id: &str,
|
|
summary: &ReportSummary,
|
|
artifacts: Value,
|
|
) -> Vec<StatsRecord> {
|
|
vec![
|
|
stats_record(
|
|
run_id,
|
|
"overview",
|
|
"counts",
|
|
json!({
|
|
"publicationPoints": summary.publication_points.len() as u64,
|
|
"objects": summary.objects_count,
|
|
"repos": summary.repos.len() as u64,
|
|
"vrps": summary.vrps_count,
|
|
"aspas": summary.aspas_count,
|
|
}),
|
|
),
|
|
stats_record(
|
|
run_id,
|
|
"objects",
|
|
"by_type",
|
|
json!(summary.stats.object_type_counts),
|
|
),
|
|
stats_record(
|
|
run_id,
|
|
"validation",
|
|
"by_result",
|
|
json!(summary.stats.result_counts),
|
|
),
|
|
stats_record(
|
|
run_id,
|
|
"objects",
|
|
"by_source",
|
|
json!(summary.stats.source_counts),
|
|
),
|
|
stats_record(
|
|
run_id,
|
|
"validation",
|
|
"reasons",
|
|
json!(summary.stats.reason_counts),
|
|
),
|
|
stats_record(
|
|
run_id,
|
|
"validation_events",
|
|
"manifest",
|
|
summary.query_audit.clone().unwrap_or_else(|| json!({})),
|
|
),
|
|
stats_record(run_id, "validation_events", "by_type", json!({})),
|
|
stats_record(run_id, "validation_events", "by_result", json!({})),
|
|
stats_record(run_id, "validation_events", "reasons", json!({})),
|
|
stats_record(
|
|
run_id,
|
|
"downloads",
|
|
"summary",
|
|
summary.download_stats.clone().unwrap_or_else(|| json!({})),
|
|
),
|
|
stats_record(run_id, "artifacts", "manifest", artifacts),
|
|
]
|
|
}
|
|
|
|
fn stats_record(run_id: &str, scope: &str, name: &str, value: Value) -> StatsRecord {
|
|
StatsRecord {
|
|
schema_version: QUERY_DB_SCHEMA_VERSION,
|
|
run_id: run_id.to_string(),
|
|
scope: scope.to_string(),
|
|
name: name.to_string(),
|
|
value,
|
|
}
|
|
}
|
|
|
|
fn merge_counts(target: &mut BTreeMap<String, u64>, source: BTreeMap<String, u64>) {
|
|
for (key, value) in source {
|
|
*target.entry(key).or_default() += value;
|
|
}
|
|
}
|
|
|
|
fn bump_limited_count(counts: &mut BTreeMap<String, u64>, key: &str, limit: usize) {
|
|
if counts.contains_key(key) || counts.len() < limit {
|
|
*counts.entry(key.to_string()).or_default() += 1;
|
|
} else {
|
|
*counts.entry("__other__".to_string()).or_default() += 1;
|
|
}
|
|
}
|
|
|
|
fn repo_uri_for_pp(pp: &PublicationPointSummary) -> String {
|
|
pp.rrdp_notification_uri
|
|
.as_deref()
|
|
.or(pp.rsync_base_uri.as_deref())
|
|
.or(pp.publication_point_rsync_uri.as_deref())
|
|
.or(pp.manifest_rsync_uri.as_deref())
|
|
.unwrap_or("unknown")
|
|
.to_string()
|
|
}
|
|
|
|
fn pp_id_for(pp: &PublicationPointSummary) -> String {
|
|
if let Some(node_id) = pp.node_id {
|
|
return format!("node_{node_id}");
|
|
}
|
|
stable_id(
|
|
pp.manifest_rsync_uri
|
|
.as_deref()
|
|
.or(pp.publication_point_rsync_uri.as_deref())
|
|
.unwrap_or("unknown"),
|
|
)
|
|
}
|
|
|
|
fn infer_transport(uri: &str) -> String {
|
|
if uri.starts_with("http://") || uri.starts_with("https://") {
|
|
"rrdp".to_string()
|
|
} else if uri.starts_with("rsync://") {
|
|
"rsync".to_string()
|
|
} else {
|
|
"unknown".to_string()
|
|
}
|
|
}
|
|
|
|
fn uri_host(uri: &str) -> String {
|
|
let without_scheme = uri.split_once("://").map(|(_, rest)| rest).unwrap_or(uri);
|
|
without_scheme
|
|
.split('/')
|
|
.next()
|
|
.filter(|s| !s.is_empty())
|
|
.unwrap_or("unknown")
|
|
.to_string()
|
|
}
|
|
|
|
fn object_type_from_uri(uri: &str) -> String {
|
|
let lower = uri.to_ascii_lowercase();
|
|
for (suffix, kind) in [
|
|
(".mft", "manifest"),
|
|
(".crl", "crl"),
|
|
(".cer", "certificate"),
|
|
(".roa", "roa"),
|
|
(".asa", "aspa"),
|
|
(".gbr", "gbr"),
|
|
] {
|
|
if lower.ends_with(suffix) {
|
|
return kind.to_string();
|
|
}
|
|
}
|
|
"other".to_string()
|
|
}
|
|
|
|
fn stable_id(value: &str) -> String {
|
|
let digest = Sha256::digest(value.as_bytes());
|
|
hex::encode(&digest[..12])
|
|
}
|
|
|
|
fn json_str<'a>(value: &'a Value, path: &[&str]) -> Option<&'a str> {
|
|
let mut current = value;
|
|
for key in path {
|
|
current = current.get(*key)?;
|
|
}
|
|
current.as_str()
|
|
}
|
|
|
|
fn json_array_len(value: &Value, path: &[&str]) -> Option<u64> {
|
|
let mut current = value;
|
|
for key in path {
|
|
current = current.get(*key)?;
|
|
}
|
|
current.as_array().map(|items| items.len() as u64)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::fs;
|
|
|
|
#[test]
|
|
fn cursor_round_trip_and_rejects_bad_input() {
|
|
assert_eq!(parse_object_cursor(None).unwrap(), (0, 0));
|
|
let cursor = object_cursor(3, 9);
|
|
assert_eq!(parse_object_cursor(Some(&cursor)).unwrap(), (3, 9));
|
|
assert!(parse_object_cursor(Some("bad")).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn summary_and_object_listing_stream_report() {
|
|
let temp = tempfile::tempdir().expect("tempdir");
|
|
let report_path = temp.path().join("report.json");
|
|
fs::write(
|
|
&report_path,
|
|
r#"{
|
|
"meta":{"validation_time_rfc3339_utc":"2026-06-16T00:00:00Z"},
|
|
"tree":{"warnings":[{}]},
|
|
"publication_points":[{
|
|
"node_id":1,
|
|
"rsync_base_uri":"rsync://repo.example/rpki/",
|
|
"manifest_rsync_uri":"rsync://repo.example/rpki/m.mft",
|
|
"publication_point_rsync_uri":"rsync://repo.example/rpki/",
|
|
"rrdp_notification_uri":"https://repo.example/rrdp/notification.xml",
|
|
"source":"rrdp",
|
|
"repo_sync_source":"rrdp",
|
|
"repo_sync_phase":"rrdp_delta",
|
|
"repo_sync_duration_ms":7,
|
|
"repo_terminal_state":"fresh",
|
|
"warnings":[],
|
|
"objects":[
|
|
{"rsync_uri":"rsync://repo.example/rpki/m.mft","sha256_hex":"11","kind":"manifest","result":"ok","detail":"restored child CA instance from publication-point cache"},
|
|
{"rsync_uri":"rsync://repo.example/rpki/a.roa","sha256_hex":"22","kind":"roa","result":"error","detail":"bad roa"}
|
|
]
|
|
}],
|
|
"vrps":[
|
|
{"asn":64496,"prefix":"192.0.2.0/24","max_length":24},
|
|
{"asn":64497,"prefix":"2001:db8::/32","max_length":48}
|
|
],
|
|
"aspas":[{}],
|
|
"download_stats":{"eventsTotal":0},
|
|
"queryAudit":{"status":"complete","eventsPath":"validation-events.jsonl"}
|
|
}"#,
|
|
)
|
|
.expect("report");
|
|
let summary = summarize_report(&report_path, "run_0001").expect("summary");
|
|
assert_eq!(
|
|
summary.validation_time.as_deref(),
|
|
Some("2026-06-16T00:00:00Z")
|
|
);
|
|
assert_eq!(summary.publication_points.len(), 1);
|
|
assert_eq!(summary.repos.len(), 1);
|
|
assert_eq!(summary.objects_count, 2);
|
|
assert_eq!(summary.vrps_count, 2);
|
|
assert_eq!(summary.aspas_count, 1);
|
|
assert_eq!(summary.stats.object_type_counts["roa"], 1);
|
|
assert_eq!(summary.stats.reason_counts["bad roa"], 1);
|
|
// #133: informational detail on non-error objects must not be counted
|
|
// as reject reasons.
|
|
assert_eq!(summary.stats.reason_counts.len(), 1);
|
|
assert!(!summary
|
|
.stats
|
|
.reason_counts
|
|
.contains_key("restored child CA instance from publication-point cache"));
|
|
|
|
let page =
|
|
list_report_objects(&report_path, "run_0001", ObjectScope::All, 1, None).expect("page");
|
|
assert_eq!(page.data.len(), 1);
|
|
assert!(page.next_cursor.is_some());
|
|
let page2 = list_report_objects(
|
|
&report_path,
|
|
"run_0001",
|
|
ObjectScope::All,
|
|
10,
|
|
page.next_cursor.as_deref(),
|
|
)
|
|
.expect("page2");
|
|
assert_eq!(page2.data.len(), 1);
|
|
assert_eq!(page2.data[0].object_type, "roa");
|
|
|
|
let found = lookup_report_object(
|
|
&report_path,
|
|
"run_0001",
|
|
ObjectScope::All,
|
|
ObjectLookup::Uri("rsync://repo.example/rpki/a.roa".to_string()),
|
|
)
|
|
.expect("lookup")
|
|
.expect("found");
|
|
assert_eq!(found.object.result, "error");
|
|
assert_eq!(found.resolution.mode, "report_scan");
|
|
}
|
|
|
|
/// Three PPs with two objects each; used to anchor paging and scope
|
|
/// behavior of the early-stop object scan.
|
|
fn write_multi_pp_report(report_path: &std::path::Path) {
|
|
fs::write(
|
|
report_path,
|
|
r#"{
|
|
"meta":{"validation_time_rfc3339_utc":"2026-06-16T00:00:00Z"},
|
|
"tree":{"warnings":[]},
|
|
"publication_points":[
|
|
{
|
|
"node_id":1,
|
|
"rsync_base_uri":"rsync://repo-a.example/rpki/",
|
|
"manifest_rsync_uri":"rsync://repo-a.example/rpki/m.mft",
|
|
"publication_point_rsync_uri":"rsync://repo-a.example/rpki/",
|
|
"rrdp_notification_uri":"https://repo-a.example/rrdp/notification.xml",
|
|
"objects":[
|
|
{"rsync_uri":"rsync://repo-a.example/rpki/m.mft","sha256_hex":"a1","kind":"manifest","result":"ok"},
|
|
{"rsync_uri":"rsync://repo-a.example/rpki/a.roa","sha256_hex":"a2","kind":"roa","result":"ok"}
|
|
]
|
|
},
|
|
{
|
|
"node_id":2,
|
|
"rsync_base_uri":"rsync://repo-b.example/rpki/",
|
|
"manifest_rsync_uri":"rsync://repo-b.example/rpki/m.mft",
|
|
"publication_point_rsync_uri":"rsync://repo-b.example/rpki/",
|
|
"objects":[
|
|
{"rsync_uri":"rsync://repo-b.example/rpki/m.mft","sha256_hex":"b1","kind":"manifest","result":"ok"},
|
|
{"rsync_uri":"rsync://repo-b.example/rpki/b.roa","sha256_hex":"b2","kind":"roa","result":"error","detail":"bad roa"}
|
|
]
|
|
},
|
|
{
|
|
"node_id":3,
|
|
"rsync_base_uri":"rsync://repo-c.example/rpki/",
|
|
"manifest_rsync_uri":"rsync://repo-c.example/rpki/m.mft",
|
|
"publication_point_rsync_uri":"rsync://repo-c.example/rpki/",
|
|
"rrdp_notification_uri":"https://repo-c.example/rrdp/notification.xml",
|
|
"objects":[
|
|
{"rsync_uri":"rsync://repo-c.example/rpki/m.mft","sha256_hex":"c1","kind":"manifest","result":"ok"},
|
|
{"rsync_uri":"rsync://repo-c.example/rpki/c.roa","sha256_hex":"c2","kind":"roa","result":"ok"}
|
|
]
|
|
}
|
|
]
|
|
}"#,
|
|
)
|
|
.expect("report");
|
|
}
|
|
|
|
#[test]
|
|
fn object_listing_pages_across_publication_points_with_stable_cursors() {
|
|
let temp = tempfile::tempdir().expect("tempdir");
|
|
let report_path = temp.path().join("report.json");
|
|
write_multi_pp_report(&report_path);
|
|
|
|
// Page 1: fills inside PP 0; the cursor must resume at PP 0 object 2.
|
|
let page1 = list_report_objects(&report_path, "run_0001", ObjectScope::All, 2, None)
|
|
.expect("page1");
|
|
assert_eq!(page1.data.len(), 2);
|
|
assert_eq!(page1.data[0].uri, "rsync://repo-a.example/rpki/m.mft");
|
|
assert_eq!(page1.data[1].uri, "rsync://repo-a.example/rpki/a.roa");
|
|
assert_eq!(page1.next_cursor.as_deref(), Some("r1:0:2"));
|
|
|
|
// Page 2: PP 0 is exhausted; the two PP 1 objects fill the page.
|
|
let page2 = list_report_objects(
|
|
&report_path,
|
|
"run_0001",
|
|
ObjectScope::All,
|
|
2,
|
|
page1.next_cursor.as_deref(),
|
|
)
|
|
.expect("page2");
|
|
assert_eq!(page2.data.len(), 2);
|
|
assert_eq!(page2.data[0].uri, "rsync://repo-b.example/rpki/m.mft");
|
|
assert_eq!(page2.data[1].uri, "rsync://repo-b.example/rpki/b.roa");
|
|
assert_eq!(page2.next_cursor.as_deref(), Some("r1:1:2"));
|
|
|
|
// Page 3: last PP, page fills at the final object.
|
|
let page3 = list_report_objects(
|
|
&report_path,
|
|
"run_0001",
|
|
ObjectScope::All,
|
|
2,
|
|
page2.next_cursor.as_deref(),
|
|
)
|
|
.expect("page3");
|
|
assert_eq!(page3.data.len(), 2);
|
|
assert_eq!(page3.next_cursor.as_deref(), Some("r1:2:2"));
|
|
|
|
// Page 4: nothing left.
|
|
let page4 = list_report_objects(
|
|
&report_path,
|
|
"run_0001",
|
|
ObjectScope::All,
|
|
2,
|
|
page3.next_cursor.as_deref(),
|
|
)
|
|
.expect("page4");
|
|
assert!(page4.data.is_empty());
|
|
assert_eq!(page4.next_cursor, None);
|
|
|
|
// A page larger than the remaining tail must not produce a cursor.
|
|
let tail = list_report_objects(&report_path, "run_0001", ObjectScope::All, 10, None)
|
|
.expect("tail");
|
|
assert_eq!(tail.data.len(), 6);
|
|
assert_eq!(tail.next_cursor, None);
|
|
}
|
|
|
|
#[test]
|
|
fn object_listing_scope_skips_non_matching_publication_points() {
|
|
let temp = tempfile::tempdir().expect("tempdir");
|
|
let report_path = temp.path().join("report.json");
|
|
write_multi_pp_report(&report_path);
|
|
|
|
// PP scope: only PP 1 (node_id 2) objects, cursor resumes within PP 1.
|
|
let page1 = list_report_objects(
|
|
&report_path,
|
|
"run_0001",
|
|
ObjectScope::PublicationPoint("node_2".to_string()),
|
|
1,
|
|
None,
|
|
)
|
|
.expect("page1");
|
|
assert_eq!(page1.data.len(), 1);
|
|
assert_eq!(page1.data[0].uri, "rsync://repo-b.example/rpki/m.mft");
|
|
assert_eq!(page1.next_cursor.as_deref(), Some("r1:1:1"));
|
|
let page2 = list_report_objects(
|
|
&report_path,
|
|
"run_0001",
|
|
ObjectScope::PublicationPoint("node_2".to_string()),
|
|
10,
|
|
page1.next_cursor.as_deref(),
|
|
)
|
|
.expect("page2");
|
|
assert_eq!(page2.data.len(), 1);
|
|
assert_eq!(page2.data[0].uri, "rsync://repo-b.example/rpki/b.roa");
|
|
assert_eq!(page2.next_cursor, None);
|
|
|
|
// Repo scope derived from the PP without rrdp uri (repo-b).
|
|
let probe_pp = list_report_objects(
|
|
&report_path,
|
|
"run_0001",
|
|
ObjectScope::PublicationPoint("node_2".to_string()),
|
|
10,
|
|
None,
|
|
)
|
|
.expect("probe");
|
|
let repo_id = probe_pp.data[0].repo_id.clone();
|
|
let repo_page = list_report_objects(
|
|
&report_path,
|
|
"run_0001",
|
|
ObjectScope::Repo(repo_id),
|
|
10,
|
|
None,
|
|
)
|
|
.expect("repo page");
|
|
assert_eq!(repo_page.data.len(), 2);
|
|
assert!(
|
|
repo_page
|
|
.data
|
|
.iter()
|
|
.all(|object| object.uri.contains("repo-b.example"))
|
|
);
|
|
|
|
// Unknown scope id: empty page, no cursor, no error.
|
|
let empty = list_report_objects(
|
|
&report_path,
|
|
"run_0001",
|
|
ObjectScope::PublicationPoint("node_999".to_string()),
|
|
10,
|
|
None,
|
|
)
|
|
.expect("empty");
|
|
assert!(empty.data.is_empty());
|
|
assert_eq!(empty.next_cursor, None);
|
|
|
|
// Filtered listing keeps working across skipped PPs.
|
|
let mut filter = ObjectFilter::default();
|
|
filter.query = Some("c.roa".to_string());
|
|
let filtered = list_report_objects_filtered(
|
|
&report_path,
|
|
"run_0001",
|
|
ObjectScope::All,
|
|
&filter,
|
|
10,
|
|
None,
|
|
)
|
|
.expect("filtered");
|
|
assert_eq!(filtered.data.len(), 1);
|
|
assert_eq!(filtered.data[0].uri, "rsync://repo-c.example/rpki/c.roa");
|
|
assert_eq!(filtered.next_cursor, None);
|
|
}
|
|
}
|