use std::collections::BTreeMap; use std::fs::File; use std::io::BufReader; use std::path::Path; use serde::de::{self, DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor}; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::value::RawValue; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; use crate::query::vrp::VrpRecord; use crate::query_db::{ ObjectInstanceRecord, PublicationPointRecord, QUERY_DB_SCHEMA_VERSION, QueryDbError, QueryDbResult, QueryPage, RepositoryRecord, StatsRecord, }; #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ReportSummary { pub validation_time: Option, pub publication_points: Vec, pub repos: Vec, pub stats: ReportStatsSummary, pub query_audit: Option, pub download_stats: Option, pub vrps_count: u64, pub vrps: Vec, pub aspas_count: u64, pub warnings_count: u64, pub objects_count: u64, } #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ReportStatsSummary { pub object_type_counts: BTreeMap, pub result_counts: BTreeMap, pub source_counts: BTreeMap, pub reason_counts: BTreeMap, } #[derive(Clone, Debug, PartialEq, Eq)] pub enum ObjectScope { All, Repo(String), PublicationPoint(String), } #[derive(Clone, Debug, PartialEq, Eq)] pub enum ObjectLookup { InstanceId(String), Uri(String), Sha256(String), } /// Optional predicates applied while streaming report objects. `reason` and /// `query` must be pre-lowercased by the caller; they are matched as /// case-insensitive substrings. The other fields are exact matches. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ObjectFilter { pub object_type: Option, pub result: Option, pub rejected: Option, pub reason: Option, pub query: Option, pub repo_id: Option, pub pp_id: Option, pub issues_only: bool, } impl ObjectFilter { pub fn matches(&self, object: &ObjectInstanceRecord) -> bool { if self.issues_only && !(object.rejected || object.result == "error") { return false; } if let Some(rejected) = self.rejected && object.rejected != rejected { return false; } if let Some(result) = self.result.as_deref() && !object.result.eq_ignore_ascii_case(result) { return false; } if let Some(object_type) = self.object_type.as_deref() && !object.object_type.eq_ignore_ascii_case(object_type) { return false; } if let Some(repo_id) = self.repo_id.as_deref() && object.repo_id != repo_id { return false; } if let Some(pp_id) = self.pp_id.as_deref() && object.pp_id != pp_id { return false; } if let Some(reason) = self.reason.as_deref() { let Some(reject_reason) = object.reject_reason.as_deref() else { return false; }; if !reject_reason.to_lowercase().contains(reason) { return false; } } if let Some(query) = self.query.as_deref() && !object.uri.to_lowercase().contains(query) { return false; } true } } #[derive(Clone, Debug, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] pub struct ObjectResolutionMeta { pub mode: String, pub cache_hit: bool, pub scan_scope: String, pub scanned_publication_points: u64, pub scanned_objects: u64, } #[derive(Clone, Debug, PartialEq, Eq)] pub struct ObjectLookupResult { pub object: ObjectInstanceRecord, pub resolution: ObjectResolutionMeta, } pub fn summarize_report(report_path: &Path, run_id: &str) -> QueryDbResult { let file = File::open(report_path)?; let reader = BufReader::new(file); let mut deserializer = serde_json::Deserializer::from_reader(reader); ReportSummarySeed { run_id } .deserialize(&mut deserializer) .map_err(QueryDbError::from) } pub fn list_report_objects( report_path: &Path, run_id: &str, scope: ObjectScope, limit: usize, cursor: Option<&str>, ) -> QueryDbResult> { list_report_objects_filtered( report_path, run_id, scope, &ObjectFilter::default(), limit, cursor, ) } pub fn list_report_objects_filtered( report_path: &Path, run_id: &str, scope: ObjectScope, filter: &ObjectFilter, limit: usize, cursor: Option<&str>, ) -> QueryDbResult> { let (start_pp_index, start_object_index) = parse_object_cursor(cursor)?; let file = File::open(report_path)?; let reader = BufReader::new(file); let mut deserializer = serde_json::Deserializer::from_reader(reader); let mut state = ObjectScanState::new_list( run_id, scope, filter.clone(), limit, start_pp_index, start_object_index, ); let scan = ObjectScanSeed { state: &mut state }.deserialize(&mut deserializer); match scan { Ok(()) => {} // A full page plus its next cursor is all the caller needs; the // visitor aborts the remaining scan with a sentinel error. Err(err) if is_object_scan_complete(&err) => {} Err(err) => return Err(QueryDbError::from(err)), } Ok(QueryPage { data: state.data, next_cursor: state.next_cursor, limit, }) } pub fn lookup_report_object( report_path: &Path, run_id: &str, scope: ObjectScope, lookup: ObjectLookup, ) -> QueryDbResult> { let file = File::open(report_path)?; let reader = BufReader::new(file); let mut deserializer = serde_json::Deserializer::from_reader(reader); let mut state = ObjectScanState::new_lookup(run_id, scope, lookup); ObjectScanSeed { state: &mut state } .deserialize(&mut deserializer) .map_err(QueryDbError::from)?; Ok(state.found.map(|object| ObjectLookupResult { object, resolution: ObjectResolutionMeta { mode: "report_scan".to_string(), cache_hit: false, scan_scope: state.scope.label(), scanned_publication_points: state.scanned_publication_points, scanned_objects: state.scanned_objects, }, })) } pub fn object_cursor(pp_index: u64, object_index: u64) -> String { format!("r1:{pp_index}:{object_index}") } /// Sentinel message used to abort the streaming object scan early once a list /// page (plus its next cursor) has been collected. The deserializer is left /// mid-document and dropped, which is fine because the caller only needs the /// state accumulated so far. const OBJECT_SCAN_COMPLETE_MSG: &str = "rpki report object scan complete: page collected"; fn is_object_scan_complete(err: &serde_json::Error) -> bool { err.is_data() && err.to_string().starts_with(OBJECT_SCAN_COMPLETE_MSG) } fn parse_object_cursor(cursor: Option<&str>) -> QueryDbResult<(u64, u64)> { let Some(cursor) = cursor else { return Ok((0, 0)); }; let mut parts = cursor.split(':'); match (parts.next(), parts.next(), parts.next(), parts.next()) { (Some("r1"), Some(pp), Some(object), None) => { let pp = pp.parse::().map_err(|_| { QueryDbError::InvalidArtifact(format!("invalid object cursor: {cursor}")) })?; let object = object.parse::().map_err(|_| { QueryDbError::InvalidArtifact(format!("invalid object cursor: {cursor}")) })?; Ok((pp, object)) } _ => Err(QueryDbError::InvalidArtifact(format!( "invalid object cursor: {cursor}" ))), } } struct ReportSummarySeed<'a> { run_id: &'a str, } impl<'de> DeserializeSeed<'de> for ReportSummarySeed<'_> { type Value = ReportSummary; fn deserialize(self, deserializer: D) -> Result where D: Deserializer<'de>, { deserializer.deserialize_map(ReportSummaryVisitor { run_id: self.run_id, }) } } struct ReportSummaryVisitor<'a> { run_id: &'a str, } impl<'de> Visitor<'de> for ReportSummaryVisitor<'_> { type Value = ReportSummary; fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { formatter.write_str("audit report object") } fn visit_map(self, mut map: A) -> Result where A: MapAccess<'de>, { let mut out = ReportSummary::default(); while let Some(key) = map.next_key::()? { match key.as_str() { "meta" => { let meta: Value = map.next_value()?; out.validation_time = json_str(&meta, &["validation_time_rfc3339_utc"]).map(str::to_string); } "tree" => { let tree: Value = map.next_value()?; out.warnings_count = json_array_len(&tree, &["warnings"]).unwrap_or(0); } "publication_points" => { let pps = map.next_value_seed(PublicationPointsSummarySeed { run_id: self.run_id, })?; out.objects_count = pps.objects_count; out.publication_points = pps.publication_points; out.repos = pps.repos; out.stats = pps.stats; } "vrps" => { let vrps = map.next_value_seed(VrpSequenceSeed { run_id: self.run_id, })?; out.vrps_count = vrps.count; out.vrps = vrps.records; } "aspas" => out.aspas_count = map.next_value::()?.0, "download_stats" => out.download_stats = Some(map.next_value()?), "queryAudit" => out.query_audit = Some(map.next_value()?), _ => { let _ = map.next_value::()?; } } } Ok(out) } } struct PublicationPointsSummarySeed<'a> { run_id: &'a str, } impl<'de> DeserializeSeed<'de> for PublicationPointsSummarySeed<'_> { type Value = PublicationPointsSummary; fn deserialize(self, deserializer: D) -> Result where D: Deserializer<'de>, { deserializer.deserialize_seq(PublicationPointsSummaryVisitor { run_id: self.run_id, }) } } #[derive(Default)] struct PublicationPointsSummary { publication_points: Vec, repos: Vec, stats: ReportStatsSummary, objects_count: u64, } #[derive(Deserialize)] struct RawVrpRecord { asn: u32, prefix: String, max_length: u8, } struct VrpSequenceSeed<'a> { run_id: &'a str, } struct VrpSequence { count: u64, records: Vec, } impl<'de> DeserializeSeed<'de> for VrpSequenceSeed<'_> { type Value = VrpSequence; fn deserialize(self, deserializer: D) -> Result where D: Deserializer<'de>, { deserializer.deserialize_seq(VrpSequenceVisitor { run_id: self.run_id, }) } } struct VrpSequenceVisitor<'a> { run_id: &'a str, } impl<'de> Visitor<'de> for VrpSequenceVisitor<'_> { type Value = VrpSequence; fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { formatter.write_str("VRP array") } fn visit_seq(self, mut seq: A) -> Result where A: SeqAccess<'de>, { let mut count = 0u64; let mut records = Vec::new(); while let Some(raw) = seq.next_element::()? { let record = VrpRecord::from_report(self.run_id, raw.asn, &raw.prefix, raw.max_length) .map_err(de::Error::custom)?; count += 1; records.push(record); } Ok(VrpSequence { count, records }) } } struct PublicationPointsSummaryVisitor<'a> { run_id: &'a str, } impl<'de> Visitor<'de> for PublicationPointsSummaryVisitor<'_> { type Value = PublicationPointsSummary; fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { formatter.write_str("publication point array") } fn visit_seq(self, mut seq: A) -> Result where A: SeqAccess<'de>, { let mut out = PublicationPointsSummary::default(); let mut repos = BTreeMap::::new(); while let Some(pp) = seq.next_element::()? { let repo_uri = repo_uri_for_pp(&pp); let repo_id = stable_id(&repo_uri); let pp_id = pp_id_for(&pp); let pp_record = pp.to_record(self.run_id, repo_id.clone(), pp_id); repos .entry(repo_id.clone()) .or_insert_with(|| RepoBuilder::new(repo_id, repo_uri)) .add_pp(&pp_record); out.objects_count += pp_record.objects; merge_counts( &mut out.stats.object_type_counts, pp.object_stats.object_type_counts, ); merge_counts(&mut out.stats.result_counts, pp.object_stats.result_counts); merge_counts(&mut out.stats.reason_counts, pp.object_stats.reason_counts); *out.stats .source_counts .entry("report".to_string()) .or_default() += pp_record.objects; out.publication_points.push(pp_record); } out.repos = repos .into_values() .map(|builder| builder.finish(self.run_id)) .collect(); Ok(out) } } #[derive(Default, Deserialize)] struct CountSeq(#[serde(deserialize_with = "count_seq")] u64); fn count_seq<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, { struct CountVisitor; impl<'de> Visitor<'de> for CountVisitor { type Value = u64; fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { formatter.write_str("array") } fn visit_seq(self, mut seq: A) -> Result where A: SeqAccess<'de>, { let mut count = 0u64; while seq.next_element::()?.is_some() { count += 1; } Ok(count) } } deserializer.deserialize_seq(CountVisitor) } #[derive(Clone, Debug, Default)] struct PublicationPointSummary { node_id: Option, parent_node_id: Option, rsync_base_uri: Option, manifest_rsync_uri: Option, publication_point_rsync_uri: Option, rrdp_notification_uri: Option, source: Option, repo_sync_source: Option, repo_sync_phase: Option, repo_sync_duration_ms: Option, repo_sync_error: Option, repo_terminal_state: Option, this_update: Option, next_update: Option, verified_at: Option, warnings: u64, object_stats: ObjectStats, } impl PublicationPointSummary { fn to_record(&self, run_id: &str, repo_id: String, pp_id: String) -> PublicationPointRecord { PublicationPointRecord { schema_version: QUERY_DB_SCHEMA_VERSION, run_id: run_id.to_string(), pp_id, repo_id, node_id: self.node_id, parent_node_id: self.parent_node_id, rsync_base_uri: self.rsync_base_uri.clone(), manifest_rsync_uri: self.manifest_rsync_uri.clone(), publication_point_rsync_uri: self.publication_point_rsync_uri.clone(), rrdp_notification_uri: self.rrdp_notification_uri.clone(), source: self.source.clone(), repo_sync_source: self.repo_sync_source.clone(), repo_sync_phase: self.repo_sync_phase.clone(), repo_sync_duration_ms: self.repo_sync_duration_ms, repo_sync_error: self.repo_sync_error.clone(), repo_terminal_state: self.repo_terminal_state.clone(), this_update: self.this_update.clone(), next_update: self.next_update.clone(), verified_at: self.verified_at.clone(), objects: self.object_stats.count, rejected_objects: self.object_stats.rejected, warnings: self.warnings, } } } impl<'de> Deserialize<'de> for PublicationPointSummary { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserializer.deserialize_map(PublicationPointSummaryVisitor) } } struct PublicationPointSummaryVisitor; impl<'de> Visitor<'de> for PublicationPointSummaryVisitor { type Value = PublicationPointSummary; fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { formatter.write_str("publication point object") } fn visit_map(self, mut map: A) -> Result where A: MapAccess<'de>, { let mut pp = PublicationPointSummary::default(); while let Some(key) = map.next_key::()? { match key.as_str() { "node_id" => pp.node_id = map.next_value()?, "parent_node_id" => pp.parent_node_id = map.next_value()?, "rsync_base_uri" => pp.rsync_base_uri = map.next_value()?, "manifest_rsync_uri" => pp.manifest_rsync_uri = map.next_value()?, "publication_point_rsync_uri" => { pp.publication_point_rsync_uri = map.next_value()? } "rrdp_notification_uri" => pp.rrdp_notification_uri = map.next_value()?, "source" => pp.source = map.next_value()?, "repo_sync_source" => pp.repo_sync_source = map.next_value()?, "repo_sync_phase" => pp.repo_sync_phase = map.next_value()?, "repo_sync_duration_ms" => pp.repo_sync_duration_ms = map.next_value()?, "repo_sync_error" => pp.repo_sync_error = map.next_value()?, "repo_terminal_state" => pp.repo_terminal_state = map.next_value()?, "this_update_rfc3339_utc" => pp.this_update = map.next_value()?, "next_update_rfc3339_utc" => pp.next_update = map.next_value()?, "verified_at_rfc3339_utc" => pp.verified_at = map.next_value()?, "warnings" => pp.warnings = map.next_value::()?.0, "objects" => pp.object_stats = map.next_value::()?.0, _ => { let _ = map.next_value::()?; } } } Ok(pp) } } #[derive(Clone, Debug, Default)] struct ObjectStats { count: u64, rejected: u64, object_type_counts: BTreeMap, result_counts: BTreeMap, reason_counts: BTreeMap, } #[derive(Clone, Debug, Default)] struct ObjectStatsSeq(ObjectStats); impl<'de> Deserialize<'de> for ObjectStatsSeq { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { struct ObjectStatsVisitor; impl<'de> Visitor<'de> for ObjectStatsVisitor { type Value = ObjectStatsSeq; fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { formatter.write_str("object audit array") } fn visit_seq(self, mut seq: A) -> Result where A: SeqAccess<'de>, { let mut stats = ObjectStats::default(); while let Some(object) = seq.next_element::()? { stats.count += 1; if object.result == "error" { stats.rejected += 1; if let Some(detail) = object.detail.filter(|value| !value.is_empty()) { bump_limited_count(&mut stats.reason_counts, &detail, 512); } } *stats .object_type_counts .entry(object.object_type) .or_default() += 1; *stats.result_counts.entry(object.result).or_default() += 1; } Ok(ObjectStatsSeq(stats)) } } deserializer.deserialize_seq(ObjectStatsVisitor) } } #[derive(Clone, Debug, Default)] struct ReportObjectSummary { object_type: String, result: String, detail: Option, } impl<'de> Deserialize<'de> for ReportObjectSummary { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserializer.deserialize_map(ReportObjectSummaryVisitor) } } struct ReportObjectSummaryVisitor; impl<'de> Visitor<'de> for ReportObjectSummaryVisitor { type Value = ReportObjectSummary; fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { formatter.write_str("object summary") } fn visit_map(self, mut map: A) -> Result where A: MapAccess<'de>, { let mut object = ReportObjectSummary { object_type: "other".to_string(), result: "unknown".to_string(), detail: None, }; let mut uri: Option = None; while let Some(key) = map.next_key::()? { match key.as_str() { "kind" => { object.object_type = map .next_value::>()? .unwrap_or_else(|| "other".to_string()) } "result" => { object.result = map .next_value::>()? .unwrap_or_else(|| "unknown".to_string()) } "detail" => object.detail = map.next_value()?, "rsync_uri" | "uri" => uri = map.next_value()?, _ => { let _ = map.next_value::()?; } } } if object.object_type == "other" { if let Some(uri) = uri.as_deref() { object.object_type = object_type_from_uri(uri); } } Ok(object) } } struct ObjectScanState<'a> { run_id: &'a str, scope: ObjectScope, mode: ObjectScanMode, filter: ObjectFilter, limit: usize, start_pp_index: u64, start_object_index: u64, data: Vec, next_cursor: Option, found: Option, scanned_publication_points: u64, scanned_objects: u64, } impl<'a> ObjectScanState<'a> { fn new_list( run_id: &'a str, scope: ObjectScope, filter: ObjectFilter, limit: usize, start_pp_index: u64, start_object_index: u64, ) -> Self { Self { run_id, scope, mode: ObjectScanMode::List, filter, limit, start_pp_index, start_object_index, data: Vec::new(), next_cursor: None, found: None, scanned_publication_points: 0, scanned_objects: 0, } } fn new_lookup(run_id: &'a str, scope: ObjectScope, lookup: ObjectLookup) -> Self { Self { run_id, scope, mode: ObjectScanMode::Lookup(lookup), filter: ObjectFilter::default(), limit: 1, start_pp_index: 0, start_object_index: 0, data: Vec::new(), next_cursor: None, found: None, scanned_publication_points: 0, scanned_objects: 0, } } /// True once a list page is full and its next cursor is fixed; everything /// after this point cannot alter the page (cursor semantics: the cursor /// already points at the first item of the next page). fn list_page_complete(&self) -> bool { matches!(self.mode, ObjectScanMode::List) && self.data.len() >= self.limit && self.next_cursor.is_some() } } #[derive(Clone, Debug, PartialEq, Eq)] enum ObjectScanMode { List, Lookup(ObjectLookup), } struct ObjectScanSeed<'a, 's> { state: &'s mut ObjectScanState<'a>, } impl<'de> DeserializeSeed<'de> for ObjectScanSeed<'_, '_> { type Value = (); fn deserialize(self, deserializer: D) -> Result where D: Deserializer<'de>, { deserializer.deserialize_map(ObjectScanReportVisitor { state: self.state }) } } struct ObjectScanReportVisitor<'a, 's> { state: &'s mut ObjectScanState<'a>, } impl<'de> Visitor<'de> for ObjectScanReportVisitor<'_, '_> { type Value = (); fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { formatter.write_str("audit report object") } fn visit_map(self, mut map: A) -> Result where A: MapAccess<'de>, { while let Some(key) = map.next_key::()? { if key == "publication_points" { map.next_value_seed(ObjectScanPpsSeed { state: self.state })?; } else { let _ = map.next_value::()?; } } Ok(()) } } struct ObjectScanPpsSeed<'a, 's> { state: &'s mut ObjectScanState<'a>, } impl<'de> DeserializeSeed<'de> for ObjectScanPpsSeed<'_, '_> { type Value = (); fn deserialize(self, deserializer: D) -> Result where D: Deserializer<'de>, { deserializer.deserialize_seq(ObjectScanPpsVisitor { state: self.state }) } } struct ObjectScanPpsVisitor<'a, 's> { state: &'s mut ObjectScanState<'a>, } impl<'de> Visitor<'de> for ObjectScanPpsVisitor<'_, '_> { type Value = (); fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { formatter.write_str("publication point array") } fn visit_seq(self, mut seq: A) -> Result where A: SeqAccess<'de>, { let mut pp_index = 0u64; loop { if self.state.list_page_complete() { return Err(de::Error::custom(OBJECT_SCAN_COMPLETE_MSG)); } let Some(raw_pp) = seq.next_element::>()? else { break; }; process_raw_publication_point(self.state, pp_index, raw_pp.get()) .map_err(de::Error::custom)?; pp_index += 1; } Ok(()) } } fn process_raw_publication_point( state: &mut ObjectScanState<'_>, pp_index: u64, raw_pp: &str, ) -> QueryDbResult<()> { state.scanned_publication_points += 1; if matches!(state.mode, ObjectScanMode::List) && pp_index < state.start_pp_index { // Every object in this PP sits before the cursor, so none of them can // be part of the page: skip materializing the subtree entirely. return Ok(()); } if state.scope != ObjectScope::All { // Scope is decidable from PP-level fields alone; probe them before // materializing the (potentially huge) objects subtree. let probe: PublicationPointScopeProbe = serde_json::from_str(raw_pp)?; let ctx = PpObjectContext::from_pp(state.run_id, &probe.into_summary()); if !state.scope.matches(&ctx) { return Ok(()); } } let pp: PublicationPointForObjects = serde_json::from_str(raw_pp)?; let ctx = PpObjectContext::from_pp(state.run_id, &pp.summary); if !state.scope.matches(&ctx) { return Ok(()); } for (object_index, object) in pp.objects.into_iter().enumerate() { state.scanned_objects += 1; let record = object.to_record(state.run_id, &ctx); match &state.mode { ObjectScanMode::List => { let object_index = object_index as u64; let after_cursor = pp_index > state.start_pp_index || (pp_index == state.start_pp_index && object_index >= state.start_object_index); if after_cursor && state.data.len() < state.limit && state.filter.matches(&record) { state.data.push(record); if state.data.len() == state.limit { state.next_cursor = Some(object_cursor(pp_index, object_index + 1)); } } } ObjectScanMode::Lookup(lookup) => { if state.found.is_none() && lookup.matches(&record) { state.found = Some(record); } } } } Ok(()) } #[derive(Clone, Debug, Default)] struct PublicationPointForObjects { summary: PublicationPointSummary, objects: Vec, } impl<'de> Deserialize<'de> for PublicationPointForObjects { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserializer.deserialize_map(PublicationPointForObjectsVisitor) } } struct PublicationPointForObjectsVisitor; impl<'de> Visitor<'de> for PublicationPointForObjectsVisitor { type Value = PublicationPointForObjects; fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { formatter.write_str("publication point object for object scan") } fn visit_map(self, mut map: A) -> Result where A: MapAccess<'de>, { let mut pp = PublicationPointForObjects::default(); while let Some(key) = map.next_key::()? { match key.as_str() { "node_id" => pp.summary.node_id = map.next_value()?, "parent_node_id" => pp.summary.parent_node_id = map.next_value()?, "rsync_base_uri" => pp.summary.rsync_base_uri = map.next_value()?, "manifest_rsync_uri" => pp.summary.manifest_rsync_uri = map.next_value()?, "publication_point_rsync_uri" => { pp.summary.publication_point_rsync_uri = map.next_value()? } "rrdp_notification_uri" => pp.summary.rrdp_notification_uri = map.next_value()?, "source" => pp.summary.source = map.next_value()?, "repo_sync_source" => pp.summary.repo_sync_source = map.next_value()?, "repo_sync_phase" => pp.summary.repo_sync_phase = map.next_value()?, "repo_sync_duration_ms" => pp.summary.repo_sync_duration_ms = map.next_value()?, "repo_sync_error" => pp.summary.repo_sync_error = map.next_value()?, "repo_terminal_state" => pp.summary.repo_terminal_state = map.next_value()?, "this_update_rfc3339_utc" => pp.summary.this_update = map.next_value()?, "next_update_rfc3339_utc" => pp.summary.next_update = map.next_value()?, "verified_at_rfc3339_utc" => pp.summary.verified_at = map.next_value()?, "objects" => pp.objects = map.next_value()?, _ => { let _ = map.next_value::()?; } } } Ok(pp) } } /// Lightweight probe of the PP-level fields needed to decide `ObjectScope` /// before the (potentially huge) objects subtree is materialized. #[derive(Clone, Debug, Default, Deserialize)] struct PublicationPointScopeProbe { node_id: Option, rsync_base_uri: Option, manifest_rsync_uri: Option, publication_point_rsync_uri: Option, rrdp_notification_uri: Option, } impl PublicationPointScopeProbe { fn into_summary(self) -> PublicationPointSummary { PublicationPointSummary { node_id: self.node_id, rsync_base_uri: self.rsync_base_uri, manifest_rsync_uri: self.manifest_rsync_uri, publication_point_rsync_uri: self.publication_point_rsync_uri, rrdp_notification_uri: self.rrdp_notification_uri, ..PublicationPointSummary::default() } } } #[derive(Clone, Debug, Default)] struct ReportObject { uri: String, sha256: String, object_type: String, result: String, detail: Option, } impl ReportObject { fn to_record(self, run_id: &str, ctx: &PpObjectContext) -> ObjectInstanceRecord { let object_type = if self.object_type.is_empty() || self.object_type == "other" { object_type_from_uri(&self.uri) } else { self.object_type }; let uri_hash = stable_id(&self.uri); let object_instance_id = stable_id(&format!( "{run_id}\0{}\0{}\0{}", self.uri, self.sha256, ctx.pp_id )); let rejected = self.result == "error"; ObjectInstanceRecord { schema_version: QUERY_DB_SCHEMA_VERSION, run_id: run_id.to_string(), object_instance_id, uri: self.uri, uri_hash, sha256: self.sha256, object_type, result: self.result, detail_summary: self.detail.clone(), repo_id: ctx.repo_id.clone(), pp_id: ctx.pp_id.clone(), source_section: "report".to_string(), rejected, reject_reason: if rejected { self.detail } else { None }, } } } impl<'de> Deserialize<'de> for ReportObject { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { deserializer.deserialize_map(ReportObjectVisitor) } } struct ReportObjectVisitor; impl<'de> Visitor<'de> for ReportObjectVisitor { type Value = ReportObject; fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { formatter.write_str("report object") } fn visit_map(self, mut map: A) -> Result where A: MapAccess<'de>, { let mut object = ReportObject { result: "unknown".to_string(), object_type: "other".to_string(), ..ReportObject::default() }; while let Some(key) = map.next_key::()? { match key.as_str() { "rsync_uri" | "uri" => { object.uri = map.next_value::>()?.unwrap_or_default() } "sha256_hex" | "sha256" => { object.sha256 = map.next_value::>()?.unwrap_or_default() } "kind" => { object.object_type = map .next_value::>()? .unwrap_or_else(|| "other".to_string()) } "result" => { object.result = map .next_value::>()? .unwrap_or_else(|| "unknown".to_string()) } "detail" => object.detail = map.next_value()?, _ => { let _ = map.next_value::()?; } } } Ok(object) } } #[derive(Clone, Debug, Default)] struct PpObjectContext { repo_id: String, pp_id: String, } impl PpObjectContext { fn from_pp(_run_id: &str, pp: &PublicationPointSummary) -> Self { let repo_uri = repo_uri_for_pp(pp); Self { repo_id: stable_id(&repo_uri), pp_id: pp_id_for(pp), } } } impl ObjectScope { fn matches(&self, ctx: &PpObjectContext) -> bool { match self { ObjectScope::All => true, ObjectScope::Repo(repo_id) => repo_id == &ctx.repo_id, ObjectScope::PublicationPoint(pp_id) => pp_id == &ctx.pp_id, } } fn label(&self) -> String { match self { ObjectScope::All => "all".to_string(), ObjectScope::Repo(_) => "repo".to_string(), ObjectScope::PublicationPoint(_) => "publication_point".to_string(), } } } impl ObjectLookup { fn matches(&self, object: &ObjectInstanceRecord) -> bool { match self { ObjectLookup::InstanceId(id) => id == &object.object_instance_id, ObjectLookup::Uri(uri) => uri == &object.uri, ObjectLookup::Sha256(sha256) => sha256.eq_ignore_ascii_case(&object.sha256), } } } #[derive(Clone, Debug)] struct RepoBuilder { repo_id: String, uri: String, publication_points: u64, objects: u64, rejected_objects: u64, sync_duration_ms_total: u64, phases: BTreeMap, terminal_states: BTreeMap, } impl RepoBuilder { fn new(repo_id: String, uri: String) -> Self { Self { repo_id, uri, publication_points: 0, objects: 0, rejected_objects: 0, sync_duration_ms_total: 0, phases: BTreeMap::new(), terminal_states: BTreeMap::new(), } } fn add_pp(&mut self, pp: &PublicationPointRecord) { self.publication_points += 1; self.objects += pp.objects; self.rejected_objects += pp.rejected_objects; self.sync_duration_ms_total = self .sync_duration_ms_total .saturating_add(pp.repo_sync_duration_ms.unwrap_or(0)); if let Some(phase) = pp.repo_sync_phase.as_ref() { *self.phases.entry(phase.clone()).or_default() += 1; } if let Some(state) = pp.repo_terminal_state.as_ref() { *self.terminal_states.entry(state.clone()).or_default() += 1; } } fn finish(self, run_id: &str) -> RepositoryRecord { RepositoryRecord { schema_version: QUERY_DB_SCHEMA_VERSION, run_id: run_id.to_string(), repo_id: self.repo_id, uri: self.uri.clone(), host: uri_host(&self.uri), transport: infer_transport(&self.uri), publication_points: self.publication_points, objects: self.objects, rejected_objects: self.rejected_objects, download_bytes: None, sync_duration_ms_total: self.sync_duration_ms_total, phases: self.phases, terminal_states: self.terminal_states, } } } pub fn stats_records_from_summary( run_id: &str, summary: &ReportSummary, artifacts: Value, ) -> Vec { vec![ stats_record( run_id, "overview", "counts", json!({ "publicationPoints": summary.publication_points.len() as u64, "objects": summary.objects_count, "repos": summary.repos.len() as u64, "vrps": summary.vrps_count, "aspas": summary.aspas_count, }), ), stats_record( run_id, "objects", "by_type", json!(summary.stats.object_type_counts), ), stats_record( run_id, "validation", "by_result", json!(summary.stats.result_counts), ), stats_record( run_id, "objects", "by_source", json!(summary.stats.source_counts), ), stats_record( run_id, "validation", "reasons", json!(summary.stats.reason_counts), ), stats_record( run_id, "validation_events", "manifest", summary.query_audit.clone().unwrap_or_else(|| json!({})), ), stats_record(run_id, "validation_events", "by_type", json!({})), stats_record(run_id, "validation_events", "by_result", json!({})), stats_record(run_id, "validation_events", "reasons", json!({})), stats_record( run_id, "downloads", "summary", summary.download_stats.clone().unwrap_or_else(|| json!({})), ), stats_record(run_id, "artifacts", "manifest", artifacts), ] } fn stats_record(run_id: &str, scope: &str, name: &str, value: Value) -> StatsRecord { StatsRecord { schema_version: QUERY_DB_SCHEMA_VERSION, run_id: run_id.to_string(), scope: scope.to_string(), name: name.to_string(), value, } } fn merge_counts(target: &mut BTreeMap, source: BTreeMap) { for (key, value) in source { *target.entry(key).or_default() += value; } } fn bump_limited_count(counts: &mut BTreeMap, key: &str, limit: usize) { if counts.contains_key(key) || counts.len() < limit { *counts.entry(key.to_string()).or_default() += 1; } else { *counts.entry("__other__".to_string()).or_default() += 1; } } fn repo_uri_for_pp(pp: &PublicationPointSummary) -> String { pp.rrdp_notification_uri .as_deref() .or(pp.rsync_base_uri.as_deref()) .or(pp.publication_point_rsync_uri.as_deref()) .or(pp.manifest_rsync_uri.as_deref()) .unwrap_or("unknown") .to_string() } fn pp_id_for(pp: &PublicationPointSummary) -> String { if let Some(node_id) = pp.node_id { return format!("node_{node_id}"); } stable_id( pp.manifest_rsync_uri .as_deref() .or(pp.publication_point_rsync_uri.as_deref()) .unwrap_or("unknown"), ) } fn infer_transport(uri: &str) -> String { if uri.starts_with("http://") || uri.starts_with("https://") { "rrdp".to_string() } else if uri.starts_with("rsync://") { "rsync".to_string() } else { "unknown".to_string() } } fn uri_host(uri: &str) -> String { let without_scheme = uri.split_once("://").map(|(_, rest)| rest).unwrap_or(uri); without_scheme .split('/') .next() .filter(|s| !s.is_empty()) .unwrap_or("unknown") .to_string() } fn object_type_from_uri(uri: &str) -> String { let lower = uri.to_ascii_lowercase(); for (suffix, kind) in [ (".mft", "manifest"), (".crl", "crl"), (".cer", "certificate"), (".roa", "roa"), (".asa", "aspa"), (".gbr", "gbr"), ] { if lower.ends_with(suffix) { return kind.to_string(); } } "other".to_string() } fn stable_id(value: &str) -> String { let digest = Sha256::digest(value.as_bytes()); hex::encode(&digest[..12]) } fn json_str<'a>(value: &'a Value, path: &[&str]) -> Option<&'a str> { let mut current = value; for key in path { current = current.get(*key)?; } current.as_str() } fn json_array_len(value: &Value, path: &[&str]) -> Option { let mut current = value; for key in path { current = current.get(*key)?; } current.as_array().map(|items| items.len() as u64) } #[cfg(test)] mod tests { use super::*; use std::fs; #[test] fn cursor_round_trip_and_rejects_bad_input() { assert_eq!(parse_object_cursor(None).unwrap(), (0, 0)); let cursor = object_cursor(3, 9); assert_eq!(parse_object_cursor(Some(&cursor)).unwrap(), (3, 9)); assert!(parse_object_cursor(Some("bad")).is_err()); } #[test] fn summary_and_object_listing_stream_report() { let temp = tempfile::tempdir().expect("tempdir"); let report_path = temp.path().join("report.json"); fs::write( &report_path, r#"{ "meta":{"validation_time_rfc3339_utc":"2026-06-16T00:00:00Z"}, "tree":{"warnings":[{}]}, "publication_points":[{ "node_id":1, "rsync_base_uri":"rsync://repo.example/rpki/", "manifest_rsync_uri":"rsync://repo.example/rpki/m.mft", "publication_point_rsync_uri":"rsync://repo.example/rpki/", "rrdp_notification_uri":"https://repo.example/rrdp/notification.xml", "source":"rrdp", "repo_sync_source":"rrdp", "repo_sync_phase":"rrdp_delta", "repo_sync_duration_ms":7, "repo_terminal_state":"fresh", "warnings":[], "objects":[ {"rsync_uri":"rsync://repo.example/rpki/m.mft","sha256_hex":"11","kind":"manifest","result":"ok","detail":"restored child CA instance from publication-point cache"}, {"rsync_uri":"rsync://repo.example/rpki/a.roa","sha256_hex":"22","kind":"roa","result":"error","detail":"bad roa"} ] }], "vrps":[ {"asn":64496,"prefix":"192.0.2.0/24","max_length":24}, {"asn":64497,"prefix":"2001:db8::/32","max_length":48} ], "aspas":[{}], "download_stats":{"eventsTotal":0}, "queryAudit":{"status":"complete","eventsPath":"validation-events.jsonl"} }"#, ) .expect("report"); let summary = summarize_report(&report_path, "run_0001").expect("summary"); assert_eq!( summary.validation_time.as_deref(), Some("2026-06-16T00:00:00Z") ); assert_eq!(summary.publication_points.len(), 1); assert_eq!(summary.repos.len(), 1); assert_eq!(summary.objects_count, 2); assert_eq!(summary.vrps_count, 2); assert_eq!(summary.aspas_count, 1); assert_eq!(summary.stats.object_type_counts["roa"], 1); assert_eq!(summary.stats.reason_counts["bad roa"], 1); // #133: informational detail on non-error objects must not be counted // as reject reasons. assert_eq!(summary.stats.reason_counts.len(), 1); assert!(!summary .stats .reason_counts .contains_key("restored child CA instance from publication-point cache")); let page = list_report_objects(&report_path, "run_0001", ObjectScope::All, 1, None).expect("page"); assert_eq!(page.data.len(), 1); assert!(page.next_cursor.is_some()); let page2 = list_report_objects( &report_path, "run_0001", ObjectScope::All, 10, page.next_cursor.as_deref(), ) .expect("page2"); assert_eq!(page2.data.len(), 1); assert_eq!(page2.data[0].object_type, "roa"); let found = lookup_report_object( &report_path, "run_0001", ObjectScope::All, ObjectLookup::Uri("rsync://repo.example/rpki/a.roa".to_string()), ) .expect("lookup") .expect("found"); assert_eq!(found.object.result, "error"); assert_eq!(found.resolution.mode, "report_scan"); } /// Three PPs with two objects each; used to anchor paging and scope /// behavior of the early-stop object scan. fn write_multi_pp_report(report_path: &std::path::Path) { fs::write( report_path, r#"{ "meta":{"validation_time_rfc3339_utc":"2026-06-16T00:00:00Z"}, "tree":{"warnings":[]}, "publication_points":[ { "node_id":1, "rsync_base_uri":"rsync://repo-a.example/rpki/", "manifest_rsync_uri":"rsync://repo-a.example/rpki/m.mft", "publication_point_rsync_uri":"rsync://repo-a.example/rpki/", "rrdp_notification_uri":"https://repo-a.example/rrdp/notification.xml", "objects":[ {"rsync_uri":"rsync://repo-a.example/rpki/m.mft","sha256_hex":"a1","kind":"manifest","result":"ok"}, {"rsync_uri":"rsync://repo-a.example/rpki/a.roa","sha256_hex":"a2","kind":"roa","result":"ok"} ] }, { "node_id":2, "rsync_base_uri":"rsync://repo-b.example/rpki/", "manifest_rsync_uri":"rsync://repo-b.example/rpki/m.mft", "publication_point_rsync_uri":"rsync://repo-b.example/rpki/", "objects":[ {"rsync_uri":"rsync://repo-b.example/rpki/m.mft","sha256_hex":"b1","kind":"manifest","result":"ok"}, {"rsync_uri":"rsync://repo-b.example/rpki/b.roa","sha256_hex":"b2","kind":"roa","result":"error","detail":"bad roa"} ] }, { "node_id":3, "rsync_base_uri":"rsync://repo-c.example/rpki/", "manifest_rsync_uri":"rsync://repo-c.example/rpki/m.mft", "publication_point_rsync_uri":"rsync://repo-c.example/rpki/", "rrdp_notification_uri":"https://repo-c.example/rrdp/notification.xml", "objects":[ {"rsync_uri":"rsync://repo-c.example/rpki/m.mft","sha256_hex":"c1","kind":"manifest","result":"ok"}, {"rsync_uri":"rsync://repo-c.example/rpki/c.roa","sha256_hex":"c2","kind":"roa","result":"ok"} ] } ] }"#, ) .expect("report"); } #[test] fn object_listing_pages_across_publication_points_with_stable_cursors() { let temp = tempfile::tempdir().expect("tempdir"); let report_path = temp.path().join("report.json"); write_multi_pp_report(&report_path); // Page 1: fills inside PP 0; the cursor must resume at PP 0 object 2. let page1 = list_report_objects(&report_path, "run_0001", ObjectScope::All, 2, None) .expect("page1"); assert_eq!(page1.data.len(), 2); assert_eq!(page1.data[0].uri, "rsync://repo-a.example/rpki/m.mft"); assert_eq!(page1.data[1].uri, "rsync://repo-a.example/rpki/a.roa"); assert_eq!(page1.next_cursor.as_deref(), Some("r1:0:2")); // Page 2: PP 0 is exhausted; the two PP 1 objects fill the page. let page2 = list_report_objects( &report_path, "run_0001", ObjectScope::All, 2, page1.next_cursor.as_deref(), ) .expect("page2"); assert_eq!(page2.data.len(), 2); assert_eq!(page2.data[0].uri, "rsync://repo-b.example/rpki/m.mft"); assert_eq!(page2.data[1].uri, "rsync://repo-b.example/rpki/b.roa"); assert_eq!(page2.next_cursor.as_deref(), Some("r1:1:2")); // Page 3: last PP, page fills at the final object. let page3 = list_report_objects( &report_path, "run_0001", ObjectScope::All, 2, page2.next_cursor.as_deref(), ) .expect("page3"); assert_eq!(page3.data.len(), 2); assert_eq!(page3.next_cursor.as_deref(), Some("r1:2:2")); // Page 4: nothing left. let page4 = list_report_objects( &report_path, "run_0001", ObjectScope::All, 2, page3.next_cursor.as_deref(), ) .expect("page4"); assert!(page4.data.is_empty()); assert_eq!(page4.next_cursor, None); // A page larger than the remaining tail must not produce a cursor. let tail = list_report_objects(&report_path, "run_0001", ObjectScope::All, 10, None) .expect("tail"); assert_eq!(tail.data.len(), 6); assert_eq!(tail.next_cursor, None); } #[test] fn object_listing_scope_skips_non_matching_publication_points() { let temp = tempfile::tempdir().expect("tempdir"); let report_path = temp.path().join("report.json"); write_multi_pp_report(&report_path); // PP scope: only PP 1 (node_id 2) objects, cursor resumes within PP 1. let page1 = list_report_objects( &report_path, "run_0001", ObjectScope::PublicationPoint("node_2".to_string()), 1, None, ) .expect("page1"); assert_eq!(page1.data.len(), 1); assert_eq!(page1.data[0].uri, "rsync://repo-b.example/rpki/m.mft"); assert_eq!(page1.next_cursor.as_deref(), Some("r1:1:1")); let page2 = list_report_objects( &report_path, "run_0001", ObjectScope::PublicationPoint("node_2".to_string()), 10, page1.next_cursor.as_deref(), ) .expect("page2"); assert_eq!(page2.data.len(), 1); assert_eq!(page2.data[0].uri, "rsync://repo-b.example/rpki/b.roa"); assert_eq!(page2.next_cursor, None); // Repo scope derived from the PP without rrdp uri (repo-b). let probe_pp = list_report_objects( &report_path, "run_0001", ObjectScope::PublicationPoint("node_2".to_string()), 10, None, ) .expect("probe"); let repo_id = probe_pp.data[0].repo_id.clone(); let repo_page = list_report_objects( &report_path, "run_0001", ObjectScope::Repo(repo_id), 10, None, ) .expect("repo page"); assert_eq!(repo_page.data.len(), 2); assert!( repo_page .data .iter() .all(|object| object.uri.contains("repo-b.example")) ); // Unknown scope id: empty page, no cursor, no error. let empty = list_report_objects( &report_path, "run_0001", ObjectScope::PublicationPoint("node_999".to_string()), 10, None, ) .expect("empty"); assert!(empty.data.is_empty()); assert_eq!(empty.next_cursor, None); // Filtered listing keeps working across skipped PPs. let mut filter = ObjectFilter::default(); filter.query = Some("c.roa".to_string()); let filtered = list_report_objects_filtered( &report_path, "run_0001", ObjectScope::All, &filter, 10, None, ) .expect("filtered"); assert_eq!(filtered.data.len(), 1); assert_eq!(filtered.data[0].uri, "rsync://repo-c.example/rpki/c.roa"); assert_eq!(filtered.next_cursor, None); } }