20260626 优化PP缓存索引与证书缓存长尾
This commit is contained in:
parent
af25316d68
commit
9e339e63e7
@ -785,6 +785,71 @@
|
||||
],
|
||||
"title": "State DB File Count Over Time",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "Prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none",
|
||||
"decimals": 0,
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 48,
|
||||
"w": 24,
|
||||
"h": 8
|
||||
},
|
||||
"id": 18,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"fresh\"})",
|
||||
"legendFormat": "fresh pp",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"roa\"})",
|
||||
"legendFormat": "fresh roa",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"manifest\"})",
|
||||
"legendFormat": "fresh mft",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"certificate\"})",
|
||||
"legendFormat": "fresh crt",
|
||||
"refId": "D"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"crl\"})",
|
||||
"legendFormat": "fresh crl",
|
||||
"refId": "E"
|
||||
}
|
||||
],
|
||||
"title": "Fresh PP / Object Counts by Run",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "5s",
|
||||
|
||||
@ -71,6 +71,10 @@ RPKI_PROGRESS_PP_CONTROL_SLOW_MS=100
|
||||
# 是否在运行前尝试禁用 rpki-client timer 并杀掉竞争 RP 进程。
|
||||
DISABLE_COMPETING_RPS=1
|
||||
|
||||
# 是否启用实验性 child CRT 验证缓存。独立于 ROA cache / PP cache；默认关闭。
|
||||
# 开启后会额外传入 --enable-child-certificate-validation-cache。
|
||||
ENABLE_CHILD_CERTIFICATE_VALIDATION_CACHE=0
|
||||
|
||||
# 传给 rpki 子进程的额外参数。多个参数用空格分隔。
|
||||
# 示例：RPKI_EXTRA_ARGS="--enable-roa-validation-cache"
|
||||
# 实验性 transport 预热：RPKI_EXTRA_ARGS="--enable-transport-request-prefetch --enable-roa-validation-cache"
|
||||
|
||||
@ -30,6 +30,7 @@ RPKI_PROGRESS_PP_CONTROL_SLOW_MS="${RPKI_PROGRESS_PP_CONTROL_SLOW_MS:-100}"
|
||||
RPKI_PROGRESS_PP_CACHE_SLOW_MS="${RPKI_PROGRESS_PP_CACHE_SLOW_MS:-50}"
|
||||
RPKI_PROGRESS_CONTROL_LOOP_SLOW_MS="${RPKI_PROGRESS_CONTROL_LOOP_SLOW_MS:-1000}"
|
||||
DISABLE_COMPETING_RPS="${DISABLE_COMPETING_RPS:-1}"
|
||||
ENABLE_CHILD_CERTIFICATE_VALIDATION_CACHE="${ENABLE_CHILD_CERTIFICATE_VALIDATION_CACHE:-0}"
|
||||
RPKI_EXTRA_ARGS="${RPKI_EXTRA_ARGS:-}"
|
||||
RPKI_ANALYZE="${RPKI_ANALYZE:-0}"
|
||||
|
||||
@ -571,6 +572,9 @@ build_child_args() {
|
||||
--vaps-csv-out "{run_out}/vaps.csv"
|
||||
--compare-view-trust-anchor "$(compare_view_trust_anchor)"
|
||||
)
|
||||
if is_true "$ENABLE_CHILD_CERTIFICATE_VALIDATION_CACHE"; then
|
||||
CHILD_ARGS+=(--enable-child-certificate-validation-cache)
|
||||
fi
|
||||
if [[ -n "$RPKI_EXTRA_ARGS" ]]; then
|
||||
# shellcheck disable=SC2206
|
||||
local extra_args=( $RPKI_EXTRA_ARGS )
|
||||
|
||||
23
src/cli.rs
23
src/cli.rs
@ -52,6 +52,7 @@ use std::sync::Arc;
|
||||
struct RunStageTiming {
|
||||
validation_ms: u64,
|
||||
enable_roa_validation_cache: bool,
|
||||
enable_child_certificate_validation_cache: bool,
|
||||
publication_point_cache_observe_only: bool,
|
||||
enable_publication_point_validation_cache: bool,
|
||||
enable_transport_request_prefetch: bool,
|
||||
@ -78,6 +79,7 @@ struct RunStageTiming {
|
||||
analysis_phases: HashMap<String, DurationStats>,
|
||||
analysis_top_publication_points: Vec<TopDurationEntry>,
|
||||
analysis_top_publication_point_steps: Vec<TopDurationEntry>,
|
||||
analysis_top_publication_point_cache_steps: Vec<TopDurationEntry>,
|
||||
vcir_storage_summary_ms: Option<u64>,
|
||||
vcir_storage: Option<VcirStorageSummary>,
|
||||
publication_point_cache_index_load: Option<crate::storage::PpCacheIndexLoadStats>,
|
||||
@ -133,6 +135,7 @@ pub struct CliArgs {
|
||||
pub skip_report_build: bool,
|
||||
pub skip_vcir_persist: bool,
|
||||
pub enable_roa_validation_cache: bool,
|
||||
pub enable_child_certificate_validation_cache: bool,
|
||||
pub publication_point_cache_observe_only: bool,
|
||||
pub enable_publication_point_validation_cache: bool,
|
||||
pub enable_transport_request_prefetch: bool,
|
||||
@ -192,6 +195,8 @@ Options:
|
||||
--skip-vcir-persist Skip VCIR persistence/projection building for compare-only runs
|
||||
--enable-roa-validation-cache
|
||||
Reuse accepted ROA validation outputs from previous VCIR records (default: off)
|
||||
--enable-child-certificate-validation-cache
|
||||
Experimental: reuse validated child certificate discovery results
|
||||
--publication-point-cache-observe-only
|
||||
Evaluate publication-point cache eligibility without changing results
|
||||
--enable-publication-point-validation-cache
|
||||
@ -278,6 +283,7 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
||||
let mut skip_report_build: bool = false;
|
||||
let mut skip_vcir_persist: bool = false;
|
||||
let mut enable_roa_validation_cache: bool = false;
|
||||
let mut enable_child_certificate_validation_cache: bool = false;
|
||||
let mut publication_point_cache_observe_only: bool = false;
|
||||
let mut enable_publication_point_validation_cache: bool = false;
|
||||
let mut enable_transport_request_prefetch: bool = false;
|
||||
@ -486,6 +492,9 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
||||
"--enable-roa-validation-cache" => {
|
||||
enable_roa_validation_cache = true;
|
||||
}
|
||||
"--enable-child-certificate-validation-cache" => {
|
||||
enable_child_certificate_validation_cache = true;
|
||||
}
|
||||
"--publication-point-cache-observe-only" => {
|
||||
publication_point_cache_observe_only = true;
|
||||
}
|
||||
@ -925,6 +934,7 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
||||
skip_report_build,
|
||||
skip_vcir_persist,
|
||||
enable_roa_validation_cache,
|
||||
enable_child_certificate_validation_cache,
|
||||
publication_point_cache_observe_only,
|
||||
enable_publication_point_validation_cache,
|
||||
enable_transport_request_prefetch,
|
||||
@ -1992,6 +2002,7 @@ pub fn run(argv: &[String]) -> Result<(), String> {
|
||||
persist_vcir: !args.skip_vcir_persist,
|
||||
build_ccr_accumulator: args.ccr_out_path.is_some(),
|
||||
enable_roa_validation_cache: args.enable_roa_validation_cache,
|
||||
enable_child_certificate_validation_cache: args.enable_child_certificate_validation_cache,
|
||||
publication_point_cache_observe_only: args.publication_point_cache_observe_only,
|
||||
enable_publication_point_validation_cache: args.enable_publication_point_validation_cache,
|
||||
enable_transport_request_prefetch: args.enable_transport_request_prefetch,
|
||||
@ -2487,6 +2498,7 @@ pub fn run(argv: &[String]) -> Result<(), String> {
|
||||
let stage_timing = RunStageTiming {
|
||||
validation_ms,
|
||||
enable_roa_validation_cache: args.enable_roa_validation_cache,
|
||||
enable_child_certificate_validation_cache: args.enable_child_certificate_validation_cache,
|
||||
publication_point_cache_observe_only: args.publication_point_cache_observe_only,
|
||||
enable_publication_point_validation_cache: args.enable_publication_point_validation_cache,
|
||||
enable_transport_request_prefetch: args.enable_transport_request_prefetch,
|
||||
@ -2525,6 +2537,17 @@ pub fn run(argv: &[String]) -> Result<(), String> {
|
||||
.as_ref()
|
||||
.map(|report| report.top_publication_point_steps.clone())
|
||||
.unwrap_or_default(),
|
||||
analysis_top_publication_point_cache_steps: timing_report_snapshot
|
||||
.as_ref()
|
||||
.map(|report| {
|
||||
report
|
||||
.top_publication_point_steps
|
||||
.iter()
|
||||
.filter(|entry| entry.key.contains("::publication_point_cache_"))
|
||||
.cloned()
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
vcir_storage_summary_ms,
|
||||
vcir_storage,
|
||||
publication_point_cache_index_load,
|
||||
|
||||
@ -147,6 +147,22 @@ fn parse_accepts_enable_roa_validation_cache() {
|
||||
assert!(args.enable_roa_validation_cache);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_accepts_enable_child_certificate_validation_cache() {
|
||||
let argv = vec![
|
||||
"rpki".to_string(),
|
||||
"--db".to_string(),
|
||||
"db".to_string(),
|
||||
"--tal-path".to_string(),
|
||||
"x.tal".to_string(),
|
||||
"--ta-path".to_string(),
|
||||
"x.cer".to_string(),
|
||||
"--enable-child-certificate-validation-cache".to_string(),
|
||||
];
|
||||
let args = parse_args(&argv).expect("parse args");
|
||||
assert!(args.enable_child_certificate_validation_cache);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_accepts_publication_point_cache_flags() {
|
||||
let argv = vec![
|
||||
@ -194,6 +210,7 @@ fn parse_disables_roa_validation_cache_by_default() {
|
||||
];
|
||||
let args = parse_args(&argv).expect("parse args");
|
||||
assert!(!args.enable_roa_validation_cache);
|
||||
assert!(!args.enable_child_certificate_validation_cache);
|
||||
assert!(!args.publication_point_cache_observe_only);
|
||||
assert!(!args.enable_publication_point_validation_cache);
|
||||
assert!(!args.enable_transport_request_prefetch);
|
||||
@ -1602,6 +1619,7 @@ fn run_report_task_and_stage_timing_work() {
|
||||
let stage_timing = RunStageTiming {
|
||||
validation_ms: 1,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
@ -1628,6 +1646,7 @@ fn run_report_task_and_stage_timing_work() {
|
||||
analysis_phases: std::collections::HashMap::new(),
|
||||
analysis_top_publication_points: Vec::new(),
|
||||
analysis_top_publication_point_steps: Vec::new(),
|
||||
analysis_top_publication_point_cache_steps: Vec::new(),
|
||||
vcir_storage_summary_ms: Some(16),
|
||||
vcir_storage: Some(VcirStorageSummary {
|
||||
entry_count: 2,
|
||||
@ -1706,6 +1725,7 @@ fn stage_timing_serializes_memory_telemetry() {
|
||||
let stage_timing = RunStageTiming {
|
||||
validation_ms: 1,
|
||||
enable_roa_validation_cache: true,
|
||||
enable_child_certificate_validation_cache: true,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: true,
|
||||
@ -1738,6 +1758,7 @@ fn stage_timing_serializes_memory_telemetry() {
|
||||
analysis_phases: std::collections::HashMap::new(),
|
||||
analysis_top_publication_points: Vec::new(),
|
||||
analysis_top_publication_point_steps: Vec::new(),
|
||||
analysis_top_publication_point_cache_steps: Vec::new(),
|
||||
vcir_storage_summary_ms: None,
|
||||
vcir_storage: None,
|
||||
publication_point_cache_index_load: None,
|
||||
|
||||
@ -710,14 +710,14 @@ mod tests {
|
||||
};
|
||||
use crate::policy::SyncPreference;
|
||||
use crate::report::Warning;
|
||||
use crate::validation::tree::{CaInstanceHandle, DiscoveredChildCaInstance};
|
||||
use crate::validation::tree::{CaCertificateRef, CaInstanceHandle, DiscoveredChildCaInstance};
|
||||
|
||||
fn sample_ca(manifest: &str) -> CaInstanceHandle {
|
||||
CaInstanceHandle {
|
||||
depth: 0,
|
||||
tal_id: "arin".to_string(),
|
||||
parent_manifest_rsync_uri: None,
|
||||
ca_certificate_der: vec![1, 2, 3],
|
||||
ca_certificate: CaCertificateRef::inline_der(vec![1, 2, 3]),
|
||||
ca_certificate_rsync_uri: None,
|
||||
effective_ip_resources: None,
|
||||
effective_as_resources: None,
|
||||
|
||||
662
src/storage.rs
662
src/storage.rs
@ -16,17 +16,18 @@ use crate::data_model::rc::{AsResourceSet, IpResourceSet};
|
||||
|
||||
use config::*;
|
||||
pub use config::{
|
||||
ALL_COLUMN_FAMILY_NAMES, CF_MANIFEST_REPLAY_META, CF_PUBLICATION_POINT_CACHE_PROJECTION,
|
||||
CF_RAW_BY_HASH, CF_REPOSITORY_VIEW, CF_ROA_CACHE_PROJECTION, CF_RRDP_SOURCE,
|
||||
CF_RRDP_SOURCE_MEMBER, CF_RRDP_URI_OWNER, CF_TRANSPORT_PREFETCH, CF_VCIR,
|
||||
column_family_descriptors,
|
||||
ALL_COLUMN_FAMILY_NAMES, CF_CHILD_CERTIFICATE_CACHE_PROJECTION, CF_MANIFEST_REPLAY_META,
|
||||
CF_PUBLICATION_POINT_CACHE_PROJECTION, CF_RAW_BY_HASH, CF_REPOSITORY_VIEW,
|
||||
CF_ROA_CACHE_PROJECTION, CF_RRDP_SOURCE, CF_RRDP_SOURCE_MEMBER, CF_RRDP_URI_OWNER,
|
||||
CF_TRANSPORT_PREFETCH, CF_VCIR, column_family_descriptors,
|
||||
};
|
||||
use keys::*;
|
||||
use pack::compute_sha256_32;
|
||||
pub use pack::{PackBytes, PackFile, PackTime};
|
||||
pub use pp_cache_index::{PpCacheIndexLoadStats, PpCacheIndexRefreshStats};
|
||||
use pp_cache_index::{
|
||||
PpCacheMmapIndexSet, default_pp_cache_index_dir, load_pp_cache_mmap_index_set,
|
||||
PpCacheIndexLookup, PpCacheMmapIndexSet, compact_pp_cache_index, default_pp_cache_index_dir,
|
||||
load_pp_cache_mmap_index, load_pp_cache_mmap_index_set, pp_cache_index_directory_stats,
|
||||
write_pp_cache_index_atomic, write_pp_cache_index_segment,
|
||||
};
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
@ -52,11 +53,20 @@ pub enum StorageError {
|
||||
|
||||
pub type StorageResult<T> = Result<T, StorageError>;
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
|
||||
pub struct ChildCertificateCacheMmapLookup {
|
||||
pub projections: Vec<Option<ChildCertificateCacheProjection>>,
|
||||
pub hits: usize,
|
||||
pub misses: usize,
|
||||
pub file_bytes: u64,
|
||||
}
|
||||
|
||||
pub struct RocksStore {
|
||||
db: DB,
|
||||
external_raw_store: Option<ExternalRawStoreDb>,
|
||||
external_repo_bytes: Option<ExternalRepoBytesDb>,
|
||||
publication_point_cache_index_dir: PathBuf,
|
||||
child_certificate_cache_index_dir: PathBuf,
|
||||
publication_point_cache_projection_index: Mutex<PublicationPointCacheProjectionIndexState>,
|
||||
}
|
||||
|
||||
@ -95,6 +105,21 @@ fn process_vm_rss_kb() -> Option<u64> {
|
||||
})
|
||||
}
|
||||
|
||||
fn default_child_certificate_cache_index_dir(db_path: &Path) -> PathBuf {
|
||||
let file_name = db_path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("work-db");
|
||||
db_path.with_file_name(format!("{file_name}.child-cert-cache-index"))
|
||||
}
|
||||
|
||||
fn child_certificate_cache_segment_file_name(manifest_rsync_uri: &str) -> String {
|
||||
format!(
|
||||
"{}.idx",
|
||||
hex::encode(compute_sha256_32(manifest_rsync_uri.as_bytes()))
|
||||
)
|
||||
}
|
||||
|
||||
const ROCKSDB_MEMORY_PROPERTY_NAMES: &[(&str, &str)] = &[
|
||||
("cur_size_all_mem_tables", "rocksdb.cur-size-all-mem-tables"),
|
||||
("size_all_mem_tables", "rocksdb.size-all-mem-tables"),
|
||||
@ -116,6 +141,8 @@ const PP_CACHE_RAW_INDEX_ENV: &str = "RPKI_PP_CACHE_RAW_INDEX";
|
||||
const PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_ENV: &str =
|
||||
"RPKI_PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_BYTES";
|
||||
const DEFAULT_PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_BYTES: usize = 32 * 1024 * 1024;
|
||||
const PP_CACHE_INDEX_COMPACTION_SEGMENT_THRESHOLD: usize = 16;
|
||||
const PP_CACHE_INDEX_COMPACTION_BYTES_THRESHOLD: u64 = 1_610_612_736;
|
||||
|
||||
fn pp_cache_raw_index_enabled() -> bool {
|
||||
match std::env::var(PP_CACHE_RAW_INDEX_ENV) {
|
||||
@ -1179,6 +1206,216 @@ pub struct RoaCacheProjection {
|
||||
pub entries: Vec<RoaCacheObjectProjection>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ChildCertificateCacheRouterKeyProjection {
|
||||
#[serde(rename = "a")]
|
||||
pub as_id: u32,
|
||||
#[serde(rename = "s")]
|
||||
#[serde(with = "serde_byte_vec")]
|
||||
pub ski: Vec<u8>,
|
||||
#[serde(rename = "p")]
|
||||
#[serde(with = "serde_byte_vec")]
|
||||
pub spki_der: Vec<u8>,
|
||||
#[serde(rename = "e")]
|
||||
pub item_effective_until: PackTime,
|
||||
}
|
||||
|
||||
impl ChildCertificateCacheRouterKeyProjection {
|
||||
pub fn validate_internal(&self) -> StorageResult<()> {
|
||||
if self.ski.is_empty() {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: "child_certificate_cache_projection.router_keys[].ski",
|
||||
detail: "must not be empty".to_string(),
|
||||
});
|
||||
}
|
||||
if self.spki_der.is_empty() {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: "child_certificate_cache_projection.router_keys[].spki_der",
|
||||
detail: "must not be empty".to_string(),
|
||||
});
|
||||
}
|
||||
parse_time(
|
||||
"child_certificate_cache_projection.router_keys[].item_effective_until",
|
||||
&self.item_effective_until,
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ChildCertificateCachePayload {
|
||||
ChildCa {
|
||||
#[serde(rename = "cm")]
|
||||
child_manifest_rsync_uri: String,
|
||||
#[serde(rename = "ski")]
|
||||
child_ski: String,
|
||||
#[serde(rename = "rb")]
|
||||
child_rsync_base_uri: String,
|
||||
#[serde(rename = "pp")]
|
||||
child_publication_point_rsync_uri: String,
|
||||
#[serde(rename = "rn")]
|
||||
child_rrdp_notification_uri: Option<String>,
|
||||
#[serde(rename = "ip")]
|
||||
child_effective_ip_resources: Option<IpResourceSet>,
|
||||
#[serde(rename = "as")]
|
||||
child_effective_as_resources: Option<AsResourceSet>,
|
||||
},
|
||||
Router {
|
||||
#[serde(rename = "r")]
|
||||
router_keys: Vec<ChildCertificateCacheRouterKeyProjection>,
|
||||
},
|
||||
}
|
||||
|
||||
impl ChildCertificateCachePayload {
|
||||
pub fn validate_internal(&self) -> StorageResult<()> {
|
||||
match self {
|
||||
Self::ChildCa {
|
||||
child_manifest_rsync_uri,
|
||||
child_ski,
|
||||
child_rsync_base_uri,
|
||||
child_publication_point_rsync_uri,
|
||||
child_rrdp_notification_uri,
|
||||
..
|
||||
} => {
|
||||
validate_non_empty(
|
||||
"child_certificate_cache_projection.child_manifest_rsync_uri",
|
||||
child_manifest_rsync_uri,
|
||||
)?;
|
||||
validate_non_empty("child_certificate_cache_projection.child_ski", child_ski)?;
|
||||
validate_non_empty(
|
||||
"child_certificate_cache_projection.child_rsync_base_uri",
|
||||
child_rsync_base_uri,
|
||||
)?;
|
||||
validate_non_empty(
|
||||
"child_certificate_cache_projection.child_publication_point_rsync_uri",
|
||||
child_publication_point_rsync_uri,
|
||||
)?;
|
||||
if let Some(uri) = child_rrdp_notification_uri {
|
||||
validate_non_empty(
|
||||
"child_certificate_cache_projection.child_rrdp_notification_uri",
|
||||
uri,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Self::Router { router_keys } => {
|
||||
if router_keys.is_empty() {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: "child_certificate_cache_projection.router_keys",
|
||||
detail: "must not be empty".to_string(),
|
||||
});
|
||||
}
|
||||
for router_key in router_keys {
|
||||
router_key.validate_internal()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ChildCertificateCacheProjection {
|
||||
#[serde(rename = "sv")]
|
||||
pub schema_version: u32,
|
||||
#[serde(rename = "av")]
|
||||
pub algorithm_version: u32,
|
||||
#[serde(rename = "k")]
|
||||
pub cache_key_sha256_hex: String,
|
||||
#[serde(rename = "cu")]
|
||||
pub child_cert_uri: String,
|
||||
#[serde(rename = "ch")]
|
||||
pub child_cert_sha256_hex: String,
|
||||
#[serde(rename = "cs")]
|
||||
#[serde(with = "serde_byte_vec")]
|
||||
pub child_cert_serial: Vec<u8>,
|
||||
#[serde(rename = "ih")]
|
||||
pub issuer_ca_sha256_hex: String,
|
||||
#[serde(rename = "cru")]
|
||||
pub issuer_crl_uri: String,
|
||||
#[serde(rename = "crh")]
|
||||
pub issuer_crl_sha256_hex: String,
|
||||
#[serde(rename = "pc")]
|
||||
#[serde(with = "serde_bytes_32")]
|
||||
pub parent_context_digest: [u8; 32],
|
||||
#[serde(rename = "pf")]
|
||||
#[serde(with = "serde_bytes_32")]
|
||||
pub validation_policy_fingerprint: [u8; 32],
|
||||
#[serde(rename = "nb")]
|
||||
pub effective_not_before: PackTime,
|
||||
#[serde(rename = "nu")]
|
||||
pub effective_until: PackTime,
|
||||
#[serde(rename = "p")]
|
||||
pub payload: ChildCertificateCachePayload,
|
||||
}
|
||||
|
||||
pub const CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION: u32 = 2;
|
||||
pub const CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION: u32 = 3;
|
||||
|
||||
impl ChildCertificateCacheProjection {
|
||||
pub fn validate_internal(&self) -> StorageResult<()> {
|
||||
if self.schema_version != CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: "child_certificate_cache_projection.schema_version",
|
||||
detail: format!("unsupported schema_version {}", self.schema_version),
|
||||
});
|
||||
}
|
||||
if self.algorithm_version != CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: "child_certificate_cache_projection.algorithm_version",
|
||||
detail: format!("unsupported algorithm_version {}", self.algorithm_version),
|
||||
});
|
||||
}
|
||||
validate_sha256_hex(
|
||||
"child_certificate_cache_projection.cache_key_sha256_hex",
|
||||
&self.cache_key_sha256_hex,
|
||||
)?;
|
||||
validate_non_empty(
|
||||
"child_certificate_cache_projection.child_cert_uri",
|
||||
&self.child_cert_uri,
|
||||
)?;
|
||||
validate_sha256_hex(
|
||||
"child_certificate_cache_projection.child_cert_sha256_hex",
|
||||
&self.child_cert_sha256_hex,
|
||||
)?;
|
||||
if self.child_cert_serial.is_empty() {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: "child_certificate_cache_projection.child_cert_serial",
|
||||
detail: "must not be empty".to_string(),
|
||||
});
|
||||
}
|
||||
validate_sha256_hex(
|
||||
"child_certificate_cache_projection.issuer_ca_sha256_hex",
|
||||
&self.issuer_ca_sha256_hex,
|
||||
)?;
|
||||
validate_non_empty(
|
||||
"child_certificate_cache_projection.issuer_crl_uri",
|
||||
&self.issuer_crl_uri,
|
||||
)?;
|
||||
validate_sha256_hex(
|
||||
"child_certificate_cache_projection.issuer_crl_sha256_hex",
|
||||
&self.issuer_crl_sha256_hex,
|
||||
)?;
|
||||
let effective_not_before = parse_time(
|
||||
"child_certificate_cache_projection.effective_not_before",
|
||||
&self.effective_not_before,
|
||||
)?;
|
||||
let effective_until = parse_time(
|
||||
"child_certificate_cache_projection.effective_until",
|
||||
&self.effective_until,
|
||||
)?;
|
||||
if effective_not_before >= effective_until {
|
||||
return Err(StorageError::InvalidData {
|
||||
entity: "child_certificate_cache_projection.effective_window",
|
||||
detail: "effective_not_before must be before effective_until".to_string(),
|
||||
});
|
||||
}
|
||||
self.payload.validate_internal()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub const PUBLICATION_POINT_CACHE_SCHEMA_VERSION: u32 = 1;
|
||||
pub const PUBLICATION_POINT_CACHE_ALGORITHM_VERSION: u32 = 1;
|
||||
|
||||
@ -2584,6 +2821,7 @@ impl RocksStore {
|
||||
external_raw_store: None,
|
||||
external_repo_bytes: None,
|
||||
publication_point_cache_index_dir: default_pp_cache_index_dir(path),
|
||||
child_certificate_cache_index_dir: default_child_certificate_cache_index_dir(path),
|
||||
publication_point_cache_projection_index: Mutex::new(
|
||||
PublicationPointCacheProjectionIndexState::Uninitialized,
|
||||
),
|
||||
@ -2658,6 +2896,59 @@ impl RocksStore {
|
||||
}
|
||||
}
|
||||
|
||||
fn try_load_publication_point_cache_mmap_index_for_update(
|
||||
&self,
|
||||
reason: &'static str,
|
||||
) -> StorageResult<()> {
|
||||
if !pp_cache_raw_index_enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut guard = self
|
||||
.publication_point_cache_projection_index
|
||||
.lock()
|
||||
.map_err(|e| {
|
||||
StorageError::RocksDb(format!("publication point cache index lock poisoned: {e}"))
|
||||
})?;
|
||||
if !matches!(
|
||||
*guard,
|
||||
PublicationPointCacheProjectionIndexState::Uninitialized
|
||||
) {
|
||||
return Ok(());
|
||||
}
|
||||
match load_pp_cache_mmap_index_set(&self.publication_point_cache_index_dir) {
|
||||
Ok((mmap, stats)) => {
|
||||
crate::progress_log::emit(
|
||||
"publication_point_cache_mmap_index_load",
|
||||
serde_json::json!({
|
||||
"state": "loaded",
|
||||
"reason": reason,
|
||||
"entries": stats.entries,
|
||||
"bytes": stats.bytes,
|
||||
"file_bytes": stats.file_bytes,
|
||||
"load_ms": stats.load_ms,
|
||||
}),
|
||||
);
|
||||
*guard = PublicationPointCacheProjectionIndexState::LoadedMmap {
|
||||
mmap,
|
||||
dirty: HashMap::new(),
|
||||
dirty_bytes: 0,
|
||||
load_stats: stats,
|
||||
};
|
||||
}
|
||||
Err(e) => {
|
||||
crate::progress_log::emit(
|
||||
"publication_point_cache_mmap_index_load",
|
||||
serde_json::json!({
|
||||
"state": "deferred_fallback_scan",
|
||||
"reason": reason,
|
||||
"error": e.to_string(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cf(&self, name: &'static str) -> StorageResult<&ColumnFamily> {
|
||||
self.db
|
||||
.cf_handle(name)
|
||||
@ -3210,6 +3501,226 @@ impl RocksStore {
|
||||
Ok(Some(projection))
|
||||
}
|
||||
|
||||
pub fn put_child_certificate_cache_projection(
|
||||
&self,
|
||||
projection: &ChildCertificateCacheProjection,
|
||||
) -> StorageResult<()> {
|
||||
projection.validate_internal()?;
|
||||
let cf = self.cf(CF_CHILD_CERTIFICATE_CACHE_PROJECTION)?;
|
||||
let key = child_certificate_cache_projection_key(&projection.cache_key_sha256_hex);
|
||||
let value = encode_cbor(projection, "child_certificate_cache_projection")?;
|
||||
self.db
|
||||
.put_cf(cf, key.as_bytes(), value)
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_child_certificate_cache_projection(
|
||||
&self,
|
||||
cache_key_sha256_hex: &str,
|
||||
) -> StorageResult<Option<ChildCertificateCacheProjection>> {
|
||||
validate_sha256_hex(
|
||||
"child_certificate_cache_projection.cache_key_sha256_hex",
|
||||
cache_key_sha256_hex,
|
||||
)?;
|
||||
let cf = self.cf(CF_CHILD_CERTIFICATE_CACHE_PROJECTION)?;
|
||||
let key = child_certificate_cache_projection_key(cache_key_sha256_hex);
|
||||
let Some(bytes) = self
|
||||
.db
|
||||
.get_cf(cf, key.as_bytes())
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let projection = decode_cbor::<ChildCertificateCacheProjection>(
|
||||
&bytes,
|
||||
"child_certificate_cache_projection",
|
||||
)?;
|
||||
projection.validate_internal()?;
|
||||
Ok(Some(projection))
|
||||
}
|
||||
|
||||
pub fn get_child_certificate_cache_projections_batch(
|
||||
&self,
|
||||
cache_key_sha256_hexes: &[String],
|
||||
) -> StorageResult<Vec<Option<ChildCertificateCacheProjection>>> {
|
||||
if cache_key_sha256_hexes.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
for cache_key_sha256_hex in cache_key_sha256_hexes {
|
||||
validate_sha256_hex(
|
||||
"child_certificate_cache_projection.cache_key_sha256_hex",
|
||||
cache_key_sha256_hex,
|
||||
)?;
|
||||
}
|
||||
|
||||
let cf = self.cf(CF_CHILD_CERTIFICATE_CACHE_PROJECTION)?;
|
||||
let keys: Vec<String> = cache_key_sha256_hexes
|
||||
.iter()
|
||||
.map(|key| child_certificate_cache_projection_key(key))
|
||||
.collect();
|
||||
self.db
|
||||
.multi_get_cf(keys.iter().map(|key| (cf, key.as_bytes())))
|
||||
.into_iter()
|
||||
.map(|res| {
|
||||
let Some(bytes) = res.map_err(|e| StorageError::RocksDb(e.to_string()))? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let projection = decode_cbor::<ChildCertificateCacheProjection>(
|
||||
&bytes,
|
||||
"child_certificate_cache_projection",
|
||||
)?;
|
||||
projection.validate_internal()?;
|
||||
Ok(Some(projection))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn child_certificate_cache_segment_path(&self, manifest_rsync_uri: &str) -> PathBuf {
|
||||
self.child_certificate_cache_index_dir
|
||||
.join(child_certificate_cache_segment_file_name(
|
||||
manifest_rsync_uri,
|
||||
))
|
||||
}
|
||||
|
||||
pub fn get_child_certificate_cache_projections_mmap_segment(
|
||||
&self,
|
||||
manifest_rsync_uri: &str,
|
||||
cache_key_sha256_hexes: &[String],
|
||||
) -> StorageResult<Option<ChildCertificateCacheMmapLookup>> {
|
||||
if cache_key_sha256_hexes.is_empty() {
|
||||
return Ok(Some(ChildCertificateCacheMmapLookup::default()));
|
||||
}
|
||||
validate_non_empty(
|
||||
"child_certificate_cache_mmap_segment.manifest_rsync_uri",
|
||||
manifest_rsync_uri,
|
||||
)?;
|
||||
for cache_key_sha256_hex in cache_key_sha256_hexes {
|
||||
validate_sha256_hex(
|
||||
"child_certificate_cache_projection.cache_key_sha256_hex",
|
||||
cache_key_sha256_hex,
|
||||
)?;
|
||||
}
|
||||
|
||||
let path = self.child_certificate_cache_segment_path(manifest_rsync_uri);
|
||||
if !path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let (index, _) = load_pp_cache_mmap_index(&path)?;
|
||||
let file_bytes = index.file_bytes();
|
||||
let mut hits = 0usize;
|
||||
let mut misses = 0usize;
|
||||
let mut projections = Vec::with_capacity(cache_key_sha256_hexes.len());
|
||||
for cache_key_sha256_hex in cache_key_sha256_hexes {
|
||||
match index.lookup(cache_key_sha256_hex) {
|
||||
Some(PpCacheIndexLookup::Hit(bytes)) => {
|
||||
let projection = decode_cbor::<ChildCertificateCacheProjection>(
|
||||
bytes,
|
||||
"child_certificate_cache_projection_mmap_segment",
|
||||
)?;
|
||||
projection.validate_internal()?;
|
||||
hits = hits.saturating_add(1);
|
||||
projections.push(Some(projection));
|
||||
}
|
||||
Some(PpCacheIndexLookup::Deleted) | None => {
|
||||
misses = misses.saturating_add(1);
|
||||
projections.push(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Some(ChildCertificateCacheMmapLookup {
|
||||
projections,
|
||||
hits,
|
||||
misses,
|
||||
file_bytes,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn write_child_certificate_cache_mmap_segment(
|
||||
&self,
|
||||
manifest_rsync_uri: &str,
|
||||
projections: &[ChildCertificateCacheProjection],
|
||||
) -> StorageResult<PpCacheIndexRefreshStats> {
|
||||
validate_non_empty(
|
||||
"child_certificate_cache_mmap_segment.manifest_rsync_uri",
|
||||
manifest_rsync_uri,
|
||||
)?;
|
||||
let mut entries = Vec::with_capacity(projections.len());
|
||||
for projection in projections {
|
||||
projection.validate_internal()?;
|
||||
entries.push((
|
||||
projection.cache_key_sha256_hex.clone(),
|
||||
encode_cbor(
|
||||
projection,
|
||||
"child_certificate_cache_projection_mmap_segment",
|
||||
)?,
|
||||
));
|
||||
}
|
||||
let path = self.child_certificate_cache_segment_path(manifest_rsync_uri);
|
||||
write_pp_cache_index_atomic(&path, entries)
|
||||
}
|
||||
|
||||
pub fn write_child_certificate_cache_mmap_segment_overlay(
|
||||
&self,
|
||||
manifest_rsync_uri: &str,
|
||||
cache_key_sha256_hexes: &[String],
|
||||
projections: &[ChildCertificateCacheProjection],
|
||||
) -> StorageResult<PpCacheIndexRefreshStats> {
|
||||
validate_non_empty(
|
||||
"child_certificate_cache_mmap_segment.manifest_rsync_uri",
|
||||
manifest_rsync_uri,
|
||||
)?;
|
||||
let mut dirty = HashMap::<String, Vec<u8>>::with_capacity(projections.len());
|
||||
for projection in projections {
|
||||
projection.validate_internal()?;
|
||||
dirty.insert(
|
||||
projection.cache_key_sha256_hex.clone(),
|
||||
encode_cbor(
|
||||
projection,
|
||||
"child_certificate_cache_projection_mmap_segment",
|
||||
)?,
|
||||
);
|
||||
}
|
||||
|
||||
let path = self.child_certificate_cache_segment_path(manifest_rsync_uri);
|
||||
let existing = if path.exists() {
|
||||
Some(load_pp_cache_mmap_index(&path)?.0)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut entries = Vec::with_capacity(cache_key_sha256_hexes.len());
|
||||
let mut emitted = HashSet::<String>::new();
|
||||
for cache_key_sha256_hex in cache_key_sha256_hexes {
|
||||
validate_sha256_hex(
|
||||
"child_certificate_cache_projection.cache_key_sha256_hex",
|
||||
cache_key_sha256_hex,
|
||||
)?;
|
||||
if !emitted.insert(cache_key_sha256_hex.clone()) {
|
||||
continue;
|
||||
}
|
||||
if let Some(value) = dirty.remove(cache_key_sha256_hex) {
|
||||
entries.push((cache_key_sha256_hex.clone(), value));
|
||||
continue;
|
||||
}
|
||||
if let Some(existing) = existing.as_ref() {
|
||||
if let Some(PpCacheIndexLookup::Hit(bytes)) = existing.lookup(cache_key_sha256_hex)
|
||||
{
|
||||
entries.push((cache_key_sha256_hex.clone(), bytes.to_vec()));
|
||||
}
|
||||
}
|
||||
}
|
||||
for (key, value) in dirty {
|
||||
if emitted.insert(key.clone()) {
|
||||
entries.push((key, value));
|
||||
}
|
||||
}
|
||||
|
||||
write_pp_cache_index_atomic(&path, entries)
|
||||
}
|
||||
|
||||
pub fn get_publication_point_cache_projection(
|
||||
&self,
|
||||
manifest_rsync_uri: &str,
|
||||
@ -3322,8 +3833,13 @@ impl RocksStore {
|
||||
}
|
||||
PublicationPointCacheProjectionIndexState::LoadedMmap { mmap, dirty, .. } => {
|
||||
if let Some(bytes) = dirty.get(manifest_rsync_uri).cloned() {
|
||||
if bytes.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
owned_bytes = Some(bytes);
|
||||
} else if let Some(bytes) = mmap.get(manifest_rsync_uri) {
|
||||
} else if let Some(lookup) = mmap.lookup(manifest_rsync_uri) {
|
||||
match lookup {
|
||||
PpCacheIndexLookup::Hit(bytes) => {
|
||||
let projection = decode_cbor::<PublicationPointCacheProjection>(
|
||||
bytes,
|
||||
"publication_point_cache_projection",
|
||||
@ -3331,6 +3847,9 @@ impl RocksStore {
|
||||
projection.validate_internal()?;
|
||||
return Ok(Some(projection));
|
||||
}
|
||||
PpCacheIndexLookup::Deleted => return Ok(None),
|
||||
}
|
||||
}
|
||||
}
|
||||
PublicationPointCacheProjectionIndexState::Disabled
|
||||
| PublicationPointCacheProjectionIndexState::Uninitialized => {}
|
||||
@ -3426,6 +3945,7 @@ impl RocksStore {
|
||||
&self,
|
||||
projection: &PublicationPointCacheProjection,
|
||||
) -> StorageResult<()> {
|
||||
self.try_load_publication_point_cache_mmap_index_for_update("write")?;
|
||||
let mut guard = self
|
||||
.publication_point_cache_projection_index
|
||||
.lock()
|
||||
@ -3485,15 +4005,12 @@ impl RocksStore {
|
||||
&self,
|
||||
manifest_rsync_uri: &str,
|
||||
) -> StorageResult<()> {
|
||||
let mut invalidate_mmap_files = false;
|
||||
{
|
||||
self.try_load_publication_point_cache_mmap_index_for_update("delete")?;
|
||||
let mut guard = self
|
||||
.publication_point_cache_projection_index
|
||||
.lock()
|
||||
.map_err(|e| {
|
||||
StorageError::RocksDb(format!(
|
||||
"publication point cache index lock poisoned: {e}"
|
||||
))
|
||||
StorageError::RocksDb(format!("publication point cache index lock poisoned: {e}"))
|
||||
})?;
|
||||
match &mut *guard {
|
||||
PublicationPointCacheProjectionIndexState::Loaded {
|
||||
@ -3513,49 +4030,18 @@ impl RocksStore {
|
||||
*total_bytes = total_bytes.saturating_sub(previous.len());
|
||||
}
|
||||
}
|
||||
PublicationPointCacheProjectionIndexState::LoadedMmap { .. } => {
|
||||
*guard = PublicationPointCacheProjectionIndexState::Disabled;
|
||||
invalidate_mmap_files = true;
|
||||
PublicationPointCacheProjectionIndexState::LoadedMmap {
|
||||
dirty, dirty_bytes, ..
|
||||
} => {
|
||||
if let Some(previous) =
|
||||
dirty.insert(manifest_rsync_uri.to_string(), Arc::<[u8]>::from([]))
|
||||
{
|
||||
*dirty_bytes = dirty_bytes.saturating_sub(previous.len());
|
||||
}
|
||||
}
|
||||
PublicationPointCacheProjectionIndexState::Uninitialized
|
||||
| PublicationPointCacheProjectionIndexState::Disabled => {}
|
||||
}
|
||||
}
|
||||
|
||||
if invalidate_mmap_files {
|
||||
self.remove_publication_point_cache_mmap_index_files()?;
|
||||
let mut guard = self
|
||||
.publication_point_cache_projection_index
|
||||
.lock()
|
||||
.map_err(|e| {
|
||||
StorageError::RocksDb(format!(
|
||||
"publication point cache index lock poisoned: {e}"
|
||||
))
|
||||
})?;
|
||||
if matches!(*guard, PublicationPointCacheProjectionIndexState::Disabled) {
|
||||
*guard = PublicationPointCacheProjectionIndexState::Uninitialized;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn remove_publication_point_cache_mmap_index_files(&self) -> StorageResult<()> {
|
||||
let dir = &self.publication_point_cache_index_dir;
|
||||
if !dir.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
for entry in std::fs::read_dir(dir).map_err(|e| StorageError::RocksDb(e.to_string()))? {
|
||||
let entry = entry.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
let path = entry.path();
|
||||
let Some(file_name) = path.file_name().and_then(|name| name.to_str()) else {
|
||||
continue;
|
||||
};
|
||||
if file_name == "current.idx"
|
||||
|| (file_name.starts_with("segment-") && file_name.ends_with(".idx"))
|
||||
{
|
||||
std::fs::remove_file(&path).map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@ -3565,6 +4051,7 @@ impl RocksStore {
|
||||
if !pp_cache_raw_index_enabled() {
|
||||
return Ok(None);
|
||||
}
|
||||
self.try_load_publication_point_cache_mmap_index_for_update("refresh")?;
|
||||
enum RefreshAction {
|
||||
Entries {
|
||||
entries: Vec<(String, Vec<u8>)>,
|
||||
@ -3647,6 +4134,81 @@ impl RocksStore {
|
||||
"write_ms": stats.write_ms,
|
||||
}),
|
||||
);
|
||||
let directory_stats =
|
||||
pp_cache_index_directory_stats(&self.publication_point_cache_index_dir)?;
|
||||
let compaction_reason = if directory_stats.segment_count
|
||||
>= PP_CACHE_INDEX_COMPACTION_SEGMENT_THRESHOLD
|
||||
{
|
||||
Some(format!(
|
||||
"segment_count>={PP_CACHE_INDEX_COMPACTION_SEGMENT_THRESHOLD}"
|
||||
))
|
||||
} else if directory_stats.total_file_bytes >= PP_CACHE_INDEX_COMPACTION_BYTES_THRESHOLD {
|
||||
Some(format!(
|
||||
"total_file_bytes>={PP_CACHE_INDEX_COMPACTION_BYTES_THRESHOLD}"
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(reason) = compaction_reason {
|
||||
crate::progress_log::emit(
|
||||
"publication_point_cache_mmap_index_compaction",
|
||||
serde_json::json!({
|
||||
"state": "started",
|
||||
"reason": reason,
|
||||
"segment_count": directory_stats.segment_count,
|
||||
"total_file_bytes": directory_stats.total_file_bytes,
|
||||
}),
|
||||
);
|
||||
match compact_pp_cache_index(&self.publication_point_cache_index_dir) {
|
||||
Ok(mut compact_stats) => {
|
||||
compact_stats.compaction_reason = Some(reason);
|
||||
compact_stats.old_entries = stats.old_entries;
|
||||
compact_stats.dirty_entries = stats.dirty_entries;
|
||||
crate::progress_log::emit(
|
||||
"publication_point_cache_mmap_index_compaction",
|
||||
serde_json::json!({
|
||||
"state": "completed",
|
||||
"reason": compact_stats.compaction_reason,
|
||||
"segments_before": compact_stats.compaction_segments_before,
|
||||
"total_file_bytes_before": compact_stats.compaction_total_file_bytes_before,
|
||||
"live_entries": compact_stats.compaction_live_entries,
|
||||
"file_bytes": compact_stats.compaction_file_bytes,
|
||||
"reclaimed_bytes": compact_stats.compaction_reclaimed_bytes,
|
||||
"deleted_segments": compact_stats.compaction_deleted_segments,
|
||||
"compaction_ms": compact_stats.compaction_ms,
|
||||
}),
|
||||
);
|
||||
stats.compaction_triggered = compact_stats.compaction_triggered;
|
||||
stats.compaction_reason = compact_stats.compaction_reason;
|
||||
stats.compaction_segments_before = compact_stats.compaction_segments_before;
|
||||
stats.compaction_total_file_bytes_before =
|
||||
compact_stats.compaction_total_file_bytes_before;
|
||||
stats.compaction_live_entries = compact_stats.compaction_live_entries;
|
||||
stats.compaction_file_bytes = compact_stats.compaction_file_bytes;
|
||||
stats.compaction_reclaimed_bytes = compact_stats.compaction_reclaimed_bytes;
|
||||
stats.compaction_ms = compact_stats.compaction_ms;
|
||||
stats.compaction_deleted_segments = compact_stats.compaction_deleted_segments;
|
||||
}
|
||||
Err(e) => {
|
||||
let error = e.to_string();
|
||||
crate::progress_log::emit(
|
||||
"publication_point_cache_mmap_index_compaction",
|
||||
serde_json::json!({
|
||||
"state": "failed",
|
||||
"reason": reason,
|
||||
"segment_count": directory_stats.segment_count,
|
||||
"total_file_bytes": directory_stats.total_file_bytes,
|
||||
"error": error,
|
||||
}),
|
||||
);
|
||||
stats.compaction_triggered = true;
|
||||
stats.compaction_reason = Some(reason);
|
||||
stats.compaction_segments_before = directory_stats.segment_count;
|
||||
stats.compaction_total_file_bytes_before = directory_stats.total_file_bytes;
|
||||
stats.compaction_error = Some(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut guard = self
|
||||
.publication_point_cache_projection_index
|
||||
.lock()
|
||||
|
||||
@ -7,6 +7,7 @@ pub const CF_VCIR: &str = "vcir";
|
||||
pub const CF_MANIFEST_REPLAY_META: &str = "manifest_replay_meta";
|
||||
pub const CF_ROA_CACHE_PROJECTION: &str = "roa_cache_projection";
|
||||
pub const CF_PUBLICATION_POINT_CACHE_PROJECTION: &str = "publication_point_cache_projection";
|
||||
pub const CF_CHILD_CERTIFICATE_CACHE_PROJECTION: &str = "child_certificate_cache_projection";
|
||||
pub const CF_RRDP_SOURCE: &str = "rrdp_source";
|
||||
pub const CF_RRDP_SOURCE_MEMBER: &str = "rrdp_source_member";
|
||||
pub const CF_RRDP_URI_OWNER: &str = "rrdp_uri_owner";
|
||||
@ -20,6 +21,7 @@ pub const ALL_COLUMN_FAMILY_NAMES: &[&str] = &[
|
||||
CF_MANIFEST_REPLAY_META,
|
||||
CF_ROA_CACHE_PROJECTION,
|
||||
CF_PUBLICATION_POINT_CACHE_PROJECTION,
|
||||
CF_CHILD_CERTIFICATE_CACHE_PROJECTION,
|
||||
CF_RRDP_SOURCE,
|
||||
CF_RRDP_SOURCE_MEMBER,
|
||||
CF_RRDP_URI_OWNER,
|
||||
@ -34,6 +36,8 @@ pub(super) const MANIFEST_REPLAY_META_KEY_PREFIX: &str = "manifest_replay_meta:"
|
||||
pub(super) const ROA_CACHE_PROJECTION_KEY_PREFIX: &str = "roa_cache_projection:";
|
||||
pub(super) const PUBLICATION_POINT_CACHE_PROJECTION_KEY_PREFIX: &str =
|
||||
"publication_point_cache_projection:";
|
||||
pub(super) const CHILD_CERTIFICATE_CACHE_PROJECTION_KEY_PREFIX: &str =
|
||||
"child_certificate_cache_projection:";
|
||||
pub(super) const RRDP_SOURCE_KEY_PREFIX: &str = "rrdp_source:";
|
||||
pub(super) const RRDP_SOURCE_MEMBER_KEY_PREFIX: &str = "rrdp_source_member:";
|
||||
pub(super) const RRDP_URI_OWNER_KEY_PREFIX: &str = "rrdp_uri_owner:";
|
||||
|
||||
@ -38,6 +38,10 @@ pub(super) fn publication_point_cache_projection_key(manifest_rsync_uri: &str) -
|
||||
format!("{PUBLICATION_POINT_CACHE_PROJECTION_KEY_PREFIX}{manifest_rsync_uri}")
|
||||
}
|
||||
|
||||
pub(super) fn child_certificate_cache_projection_key(cache_key_sha256_hex: &str) -> String {
|
||||
format!("{CHILD_CERTIFICATE_CACHE_PROJECTION_KEY_PREFIX}{cache_key_sha256_hex}")
|
||||
}
|
||||
|
||||
pub(super) fn publication_point_cache_projection_key_manifest_uri(key: &[u8]) -> Option<String> {
|
||||
let key = std::str::from_utf8(key).ok()?;
|
||||
key.strip_prefix(PUBLICATION_POINT_CACHE_PROJECTION_KEY_PREFIX)
|
||||
|
||||
@ -32,6 +32,22 @@ pub struct PpCacheIndexRefreshStats {
|
||||
pub new_entries: usize,
|
||||
pub file_bytes: u64,
|
||||
pub write_ms: u64,
|
||||
pub compaction_triggered: bool,
|
||||
pub compaction_reason: Option<String>,
|
||||
pub compaction_segments_before: usize,
|
||||
pub compaction_total_file_bytes_before: u64,
|
||||
pub compaction_live_entries: usize,
|
||||
pub compaction_file_bytes: u64,
|
||||
pub compaction_reclaimed_bytes: u64,
|
||||
pub compaction_ms: u64,
|
||||
pub compaction_deleted_segments: usize,
|
||||
pub compaction_error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
|
||||
pub struct PpCacheIndexDirectoryStats {
|
||||
pub segment_count: usize,
|
||||
pub total_file_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
@ -40,6 +56,12 @@ struct PpCacheIndexEntry {
|
||||
value_len: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum PpCacheIndexLookup<'a> {
|
||||
Hit(&'a [u8]),
|
||||
Deleted,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct PpCacheMmapIndex {
|
||||
mmap: Arc<Mmap>,
|
||||
@ -126,12 +148,23 @@ impl PpCacheMmapIndex {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get(&self, manifest_rsync_uri: &str) -> Option<&[u8]> {
|
||||
pub fn lookup(&self, manifest_rsync_uri: &str) -> Option<PpCacheIndexLookup<'_>> {
|
||||
let entry = self.entries.get(manifest_rsync_uri)?;
|
||||
if entry.value_len == 0 {
|
||||
return Some(PpCacheIndexLookup::Deleted);
|
||||
}
|
||||
let blob_offset = read_u64(self.mmap.as_ref(), 40).ok()? as usize;
|
||||
let start = blob_offset + entry.value_offset;
|
||||
let end = start + entry.value_len;
|
||||
Some(&self.mmap[start..end])
|
||||
Some(PpCacheIndexLookup::Hit(&self.mmap[start..end]))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn get(&self, manifest_rsync_uri: &str) -> Option<&[u8]> {
|
||||
match self.lookup(manifest_rsync_uri)? {
|
||||
PpCacheIndexLookup::Hit(bytes) => Some(bytes),
|
||||
PpCacheIndexLookup::Deleted => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn entries(&self) -> usize {
|
||||
@ -156,10 +189,18 @@ pub struct PpCacheMmapIndexSet {
|
||||
}
|
||||
|
||||
impl PpCacheMmapIndexSet {
|
||||
pub fn get(&self, manifest_rsync_uri: &str) -> Option<&[u8]> {
|
||||
pub fn lookup(&self, manifest_rsync_uri: &str) -> Option<PpCacheIndexLookup<'_>> {
|
||||
self.indexes
|
||||
.iter()
|
||||
.find_map(|index| index.get(manifest_rsync_uri))
|
||||
.find_map(|index| index.lookup(manifest_rsync_uri))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn get(&self, manifest_rsync_uri: &str) -> Option<&[u8]> {
|
||||
match self.lookup(manifest_rsync_uri)? {
|
||||
PpCacheIndexLookup::Hit(bytes) => Some(bytes),
|
||||
PpCacheIndexLookup::Deleted => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn entries(&self) -> usize {
|
||||
@ -259,6 +300,84 @@ pub fn load_pp_cache_mmap_index_set(
|
||||
Ok((set, stats))
|
||||
}
|
||||
|
||||
pub fn pp_cache_index_directory_stats(dir: &Path) -> StorageResult<PpCacheIndexDirectoryStats> {
|
||||
let mut stats = PpCacheIndexDirectoryStats::default();
|
||||
if !dir.exists() {
|
||||
return Ok(stats);
|
||||
}
|
||||
for entry in fs::read_dir(dir).map_err(|e| StorageError::RocksDb(e.to_string()))? {
|
||||
let path = entry
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?
|
||||
.path();
|
||||
let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
|
||||
continue;
|
||||
};
|
||||
let is_index =
|
||||
name == "current.idx" || (name.starts_with("segment-") && name.ends_with(".idx"));
|
||||
if !is_index {
|
||||
continue;
|
||||
}
|
||||
let len = fs::metadata(&path)
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?
|
||||
.len();
|
||||
stats.total_file_bytes = stats.total_file_bytes.saturating_add(len);
|
||||
if name.starts_with("segment-") && name.ends_with(".idx") {
|
||||
stats.segment_count += 1;
|
||||
}
|
||||
}
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
pub fn compact_pp_cache_index(dir: &Path) -> StorageResult<PpCacheIndexRefreshStats> {
|
||||
let started = Instant::now();
|
||||
fs::create_dir_all(dir).map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
let before = pp_cache_index_directory_stats(dir)?;
|
||||
let (index_set, _) = load_pp_cache_mmap_index_set(dir)?;
|
||||
let mut compact_entries = Vec::with_capacity(index_set.entries());
|
||||
let mut seen = HashMap::<String, ()>::with_capacity(index_set.entries());
|
||||
for index in &index_set.indexes {
|
||||
for (key, entry) in &index.entries {
|
||||
if seen.insert(key.clone(), ()).is_some() {
|
||||
continue;
|
||||
}
|
||||
if entry.value_len == 0 {
|
||||
continue;
|
||||
}
|
||||
let blob_offset = read_u64(index.mmap.as_ref(), 40)? as usize;
|
||||
let start = blob_offset + entry.value_offset;
|
||||
let end = start + entry.value_len;
|
||||
compact_entries.push((key.clone(), index.mmap[start..end].to_vec()));
|
||||
}
|
||||
}
|
||||
|
||||
let current = dir.join("current.idx");
|
||||
let mut stats = write_pp_cache_index_atomic(¤t, compact_entries)?;
|
||||
stats.state = "compacted".to_string();
|
||||
stats.compaction_triggered = true;
|
||||
stats.compaction_segments_before = before.segment_count;
|
||||
stats.compaction_total_file_bytes_before = before.total_file_bytes;
|
||||
stats.compaction_live_entries = stats.new_entries;
|
||||
stats.compaction_file_bytes = stats.file_bytes;
|
||||
stats.compaction_reclaimed_bytes = before.total_file_bytes.saturating_sub(stats.file_bytes);
|
||||
|
||||
let mut deleted_segments = 0usize;
|
||||
for entry in fs::read_dir(dir).map_err(|e| StorageError::RocksDb(e.to_string()))? {
|
||||
let path = entry
|
||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?
|
||||
.path();
|
||||
let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
|
||||
continue;
|
||||
};
|
||||
if name.starts_with("segment-") && name.ends_with(".idx") {
|
||||
fs::remove_file(&path).map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||
deleted_segments += 1;
|
||||
}
|
||||
}
|
||||
stats.compaction_deleted_segments = deleted_segments;
|
||||
stats.compaction_ms = started.elapsed().as_millis() as u64;
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
pub fn write_pp_cache_index_segment<I>(
|
||||
dir: &Path,
|
||||
entries: I,
|
||||
@ -316,6 +435,7 @@ where
|
||||
new_entries: ordered.len(),
|
||||
file_bytes,
|
||||
write_ms: started.elapsed().as_millis() as u64,
|
||||
..PpCacheIndexRefreshStats::default()
|
||||
})
|
||||
}
|
||||
|
||||
@ -465,4 +585,87 @@ mod tests {
|
||||
assert_eq!(stats.entries, 2);
|
||||
assert_eq!(set.get("rsync://example.test/a.mft"), Some(&b"new"[..]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pp_cache_index_set_tombstone_shadows_older_entry() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let current = dir.path().join("current.idx");
|
||||
write_pp_cache_index_atomic(
|
||||
¤t,
|
||||
vec![("rsync://example.test/a.mft".to_string(), b"old".to_vec())],
|
||||
)
|
||||
.expect("write current index");
|
||||
write_pp_cache_index_segment(
|
||||
dir.path(),
|
||||
vec![("rsync://example.test/a.mft".to_string(), Vec::new())],
|
||||
)
|
||||
.expect("write tombstone segment index");
|
||||
|
||||
let (set, stats) = load_pp_cache_mmap_index_set(dir.path()).expect("load index set");
|
||||
assert_eq!(stats.entries, 2);
|
||||
assert_eq!(
|
||||
set.lookup("rsync://example.test/a.mft"),
|
||||
Some(PpCacheIndexLookup::Deleted)
|
||||
);
|
||||
assert_eq!(set.get("rsync://example.test/a.mft"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pp_cache_index_compaction_keeps_latest_values_and_removes_segments() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let current = dir.path().join("current.idx");
|
||||
write_pp_cache_index_atomic(
|
||||
¤t,
|
||||
vec![
|
||||
("rsync://example.test/a.mft".to_string(), b"old-a".to_vec()),
|
||||
("rsync://example.test/b.mft".to_string(), b"old-b".to_vec()),
|
||||
("rsync://example.test/c.mft".to_string(), b"old-c".to_vec()),
|
||||
],
|
||||
)
|
||||
.expect("write current index");
|
||||
write_pp_cache_index_segment(
|
||||
dir.path(),
|
||||
vec![
|
||||
("rsync://example.test/a.mft".to_string(), b"new-a".to_vec()),
|
||||
("rsync://example.test/b.mft".to_string(), Vec::new()),
|
||||
],
|
||||
)
|
||||
.expect("write first segment");
|
||||
write_pp_cache_index_segment(
|
||||
dir.path(),
|
||||
vec![("rsync://example.test/d.mft".to_string(), b"new-d".to_vec())],
|
||||
)
|
||||
.expect("write second segment");
|
||||
|
||||
let before = pp_cache_index_directory_stats(dir.path()).expect("stats before");
|
||||
assert_eq!(before.segment_count, 2);
|
||||
|
||||
let stats = compact_pp_cache_index(dir.path()).expect("compact");
|
||||
assert!(stats.compaction_triggered);
|
||||
assert_eq!(stats.compaction_segments_before, 2);
|
||||
assert_eq!(stats.compaction_live_entries, 3);
|
||||
assert_eq!(stats.compaction_deleted_segments, 2);
|
||||
assert!(stats.compaction_reclaimed_bytes > 0);
|
||||
|
||||
let after = pp_cache_index_directory_stats(dir.path()).expect("stats after");
|
||||
assert_eq!(after.segment_count, 0);
|
||||
let segments = fs::read_dir(dir.path())
|
||||
.expect("read dir")
|
||||
.filter_map(Result::ok)
|
||||
.filter(|entry| {
|
||||
entry
|
||||
.file_name()
|
||||
.to_str()
|
||||
.is_some_and(|name| name.starts_with("segment-"))
|
||||
})
|
||||
.count();
|
||||
assert_eq!(segments, 0);
|
||||
|
||||
let (index, _) = load_pp_cache_mmap_index(¤t).expect("load compacted current");
|
||||
assert_eq!(index.entries(), 3);
|
||||
assert_eq!(index.get("rsync://example.test/a.mft"), Some(&b"new-a"[..]));
|
||||
assert_eq!(index.get("rsync://example.test/b.mft"), None);
|
||||
assert_eq!(index.get("rsync://example.test/c.mft"), Some(&b"old-c"[..]));
|
||||
assert_eq!(index.get("rsync://example.test/d.mft"), Some(&b"new-d"[..]));
|
||||
}
|
||||
}
|
||||
|
||||
@ -14,6 +14,37 @@ fn sha256_32(input: &[u8]) -> [u8; 32] {
|
||||
compute_sha256_32(input)
|
||||
}
|
||||
|
||||
fn sample_child_certificate_cache_projection(
|
||||
cache_key_sha256_hex: String,
|
||||
child_cert_uri: &str,
|
||||
child_cert_sha256_hex: &str,
|
||||
) -> ChildCertificateCacheProjection {
|
||||
ChildCertificateCacheProjection {
|
||||
schema_version: CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION,
|
||||
algorithm_version: CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION,
|
||||
cache_key_sha256_hex,
|
||||
child_cert_uri: child_cert_uri.to_string(),
|
||||
child_cert_sha256_hex: child_cert_sha256_hex.to_string(),
|
||||
child_cert_serial: vec![1],
|
||||
issuer_ca_sha256_hex: sha256_hex(b"issuer-ca"),
|
||||
issuer_crl_uri: "rsync://example.test/repo/issuer.crl".to_string(),
|
||||
issuer_crl_sha256_hex: sha256_hex(b"issuer-crl"),
|
||||
parent_context_digest: sha256_32(b"parent-context"),
|
||||
validation_policy_fingerprint: sha256_32(b"policy"),
|
||||
effective_not_before: pack_time(0),
|
||||
effective_until: pack_time(24),
|
||||
payload: ChildCertificateCachePayload::ChildCa {
|
||||
child_manifest_rsync_uri: format!("{child_cert_uri}.mft"),
|
||||
child_ski: "11".repeat(20),
|
||||
child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(),
|
||||
child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(),
|
||||
child_rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()),
|
||||
child_effective_ip_resources: None,
|
||||
child_effective_as_resources: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_work_db_blob_mode_accepts_supported_values() {
|
||||
assert_eq!(default_work_db_blob_mode(), WorkDbBlobMode::Disabled);
|
||||
@ -565,6 +596,128 @@ fn publication_point_cache_mmap_index_dirty_overlay_wins_and_refreshes_segment()
|
||||
assert_eq!(got.manifest_sha256, sha256_32(b"manifest-new"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publication_point_cache_mmap_index_write_before_read_refreshes_segment() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let db_path = td.path().join("work-db");
|
||||
let vcir = sample_vcir("rsync://example.test/repo/current.mft");
|
||||
let mut projection = PublicationPointCacheProjection::from_vcir_with_context(
|
||||
&vcir,
|
||||
"rsync://example.test/repo/".to_string(),
|
||||
Some("rsync://example.test/repo/ca.cer".to_string()),
|
||||
sha256_32(b"ca-cert"),
|
||||
sha256_32(b"manifest-old"),
|
||||
sha256_32(b"ta-context"),
|
||||
sha256_32(b"parent-context"),
|
||||
sha256_32(b"policy"),
|
||||
)
|
||||
.expect("build publication point projection");
|
||||
|
||||
{
|
||||
let store = RocksStore::open(&db_path).expect("open rocksdb");
|
||||
store
|
||||
.put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection))
|
||||
.expect("put old projection");
|
||||
let stats = store
|
||||
.refresh_publication_point_cache_mmap_index()
|
||||
.expect("refresh base mmap index")
|
||||
.expect("refresh stats");
|
||||
assert_eq!(stats.state, "written");
|
||||
}
|
||||
|
||||
{
|
||||
let store = RocksStore::open(&db_path).expect("reopen rocksdb");
|
||||
projection.manifest_sha256 = sha256_32(b"manifest-new");
|
||||
store
|
||||
.put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection))
|
||||
.expect("put new projection before any cached read");
|
||||
let stats = store
|
||||
.refresh_publication_point_cache_mmap_index()
|
||||
.expect("refresh dirty mmap segment")
|
||||
.expect("refresh stats");
|
||||
assert_eq!(stats.state, "segment_written");
|
||||
assert_eq!(stats.dirty_entries, 1);
|
||||
assert_eq!(stats.new_entries, 1);
|
||||
}
|
||||
|
||||
let store = RocksStore::open(&db_path).expect("reopen rocksdb after segment");
|
||||
let got = store
|
||||
.get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri)
|
||||
.expect("get refreshed projection")
|
||||
.expect("projection exists");
|
||||
assert_eq!(got.manifest_sha256, sha256_32(b"manifest-new"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publication_point_cache_mmap_index_delete_before_read_refreshes_tombstone_segment() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let db_path = td.path().join("work-db");
|
||||
let vcir = sample_vcir("rsync://example.test/repo/current.mft");
|
||||
let projection = PublicationPointCacheProjection::from_vcir_with_context(
|
||||
&vcir,
|
||||
"rsync://example.test/repo/".to_string(),
|
||||
Some("rsync://example.test/repo/ca.cer".to_string()),
|
||||
sha256_32(b"ca-cert"),
|
||||
sha256_32(b"manifest"),
|
||||
sha256_32(b"ta-context"),
|
||||
sha256_32(b"parent-context"),
|
||||
sha256_32(b"policy"),
|
||||
)
|
||||
.expect("build publication point projection");
|
||||
|
||||
{
|
||||
let store = RocksStore::open(&db_path).expect("open rocksdb");
|
||||
store
|
||||
.put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection))
|
||||
.expect("put projection");
|
||||
let stats = store
|
||||
.refresh_publication_point_cache_mmap_index()
|
||||
.expect("refresh base mmap index")
|
||||
.expect("refresh stats");
|
||||
assert_eq!(stats.state, "written");
|
||||
}
|
||||
|
||||
{
|
||||
let store = RocksStore::open(&db_path).expect("reopen rocksdb");
|
||||
store
|
||||
.replace_vcir_manifest_replay_meta_and_projection_action(
|
||||
&vcir,
|
||||
None,
|
||||
PublicationPointCacheProjectionWriteAction::Delete {
|
||||
manifest_rsync_uri: &vcir.manifest_rsync_uri,
|
||||
},
|
||||
)
|
||||
.expect("delete projection before any cached read");
|
||||
assert!(
|
||||
store
|
||||
.get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri)
|
||||
.expect("dirty tombstone lookup")
|
||||
.is_none()
|
||||
);
|
||||
let stats = store
|
||||
.refresh_publication_point_cache_mmap_index()
|
||||
.expect("refresh tombstone mmap segment")
|
||||
.expect("refresh stats");
|
||||
assert_eq!(stats.state, "segment_written");
|
||||
assert_eq!(stats.dirty_entries, 1);
|
||||
assert_eq!(stats.new_entries, 1);
|
||||
}
|
||||
|
||||
let store = RocksStore::open(&db_path).expect("reopen rocksdb after tombstone segment");
|
||||
assert!(
|
||||
store
|
||||
.get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri)
|
||||
.expect("tombstone shadows old current index")
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_publication_point_cache_projection(&vcir.manifest_rsync_uri)
|
||||
.expect("direct db lookup")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publication_point_cache_projection_rejects_version_mismatch() {
|
||||
let vcir = sample_vcir("rsync://example.test/repo/current.mft");
|
||||
@ -1364,6 +1517,196 @@ fn replace_vcir_and_manifest_replay_meta_replaces_current_entry() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_child_certificate_cache_projections_batch_preserves_order_and_misses() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
|
||||
let first_key = sha256_hex(b"child-cache-key-first");
|
||||
let missing_key = sha256_hex(b"child-cache-key-missing");
|
||||
let second_key = sha256_hex(b"child-cache-key-second");
|
||||
let first_hash = sha256_hex(b"first-child-cert");
|
||||
let second_hash = sha256_hex(b"second-child-cert");
|
||||
let first = sample_child_certificate_cache_projection(
|
||||
first_key.clone(),
|
||||
"rsync://example.test/repo/first.cer",
|
||||
&first_hash,
|
||||
);
|
||||
let second = sample_child_certificate_cache_projection(
|
||||
second_key.clone(),
|
||||
"rsync://example.test/repo/second.cer",
|
||||
&second_hash,
|
||||
);
|
||||
|
||||
store
|
||||
.put_child_certificate_cache_projection(&first)
|
||||
.expect("put first projection");
|
||||
store
|
||||
.put_child_certificate_cache_projection(&second)
|
||||
.expect("put second projection");
|
||||
|
||||
let got = store
|
||||
.get_child_certificate_cache_projections_batch(&[
|
||||
second_key.clone(),
|
||||
missing_key,
|
||||
first_key.clone(),
|
||||
])
|
||||
.expect("batch get child projections");
|
||||
|
||||
assert_eq!(got.len(), 3);
|
||||
assert_eq!(
|
||||
got[0]
|
||||
.as_ref()
|
||||
.expect("second projection")
|
||||
.child_cert_sha256_hex,
|
||||
second_hash
|
||||
);
|
||||
assert!(got[1].is_none());
|
||||
assert_eq!(
|
||||
got[2]
|
||||
.as_ref()
|
||||
.expect("first projection")
|
||||
.child_cert_sha256_hex,
|
||||
first_hash
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn child_certificate_cache_mmap_segment_preserves_order_and_misses() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
|
||||
let manifest_uri = "rsync://example.test/repo/parent.mft";
|
||||
let first_key = sha256_hex(b"child-cache-mmap-key-first");
|
||||
let missing_key = sha256_hex(b"child-cache-mmap-key-missing");
|
||||
let second_key = sha256_hex(b"child-cache-mmap-key-second");
|
||||
let first_hash = sha256_hex(b"first-child-cert-mmap");
|
||||
let second_hash = sha256_hex(b"second-child-cert-mmap");
|
||||
let first = sample_child_certificate_cache_projection(
|
||||
first_key.clone(),
|
||||
"rsync://example.test/repo/first.cer",
|
||||
&first_hash,
|
||||
);
|
||||
let second = sample_child_certificate_cache_projection(
|
||||
second_key.clone(),
|
||||
"rsync://example.test/repo/second.cer",
|
||||
&second_hash,
|
||||
);
|
||||
|
||||
assert!(
|
||||
store
|
||||
.get_child_certificate_cache_projections_mmap_segment(
|
||||
manifest_uri,
|
||||
&[first_key.clone()]
|
||||
)
|
||||
.expect("missing segment lookup")
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let write_stats = store
|
||||
.write_child_certificate_cache_mmap_segment(manifest_uri, &[first.clone(), second.clone()])
|
||||
.expect("write child projection segment");
|
||||
assert_eq!(write_stats.new_entries, 2);
|
||||
assert!(write_stats.file_bytes > 0);
|
||||
|
||||
let got = store
|
||||
.get_child_certificate_cache_projections_mmap_segment(
|
||||
manifest_uri,
|
||||
&[second_key.clone(), missing_key, first_key.clone()],
|
||||
)
|
||||
.expect("lookup child projection segment")
|
||||
.expect("segment exists");
|
||||
|
||||
assert_eq!(got.hits, 2);
|
||||
assert_eq!(got.misses, 1);
|
||||
assert!(got.file_bytes > 0);
|
||||
assert_eq!(got.projections.len(), 3);
|
||||
assert_eq!(
|
||||
got.projections[0]
|
||||
.as_ref()
|
||||
.expect("second projection")
|
||||
.child_cert_sha256_hex,
|
||||
second_hash
|
||||
);
|
||||
assert!(got.projections[1].is_none());
|
||||
assert_eq!(
|
||||
got.projections[2]
|
||||
.as_ref()
|
||||
.expect("first projection")
|
||||
.child_cert_sha256_hex,
|
||||
first_hash
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn child_certificate_cache_mmap_segment_overlay_preserves_existing_values() {
|
||||
let td = tempfile::tempdir().expect("tempdir");
|
||||
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||
|
||||
let manifest_uri = "rsync://example.test/repo/parent-overlay.mft";
|
||||
let first_key = sha256_hex(b"child-cache-overlay-key-first");
|
||||
let second_key = sha256_hex(b"child-cache-overlay-key-second");
|
||||
let missing_key = sha256_hex(b"child-cache-overlay-key-missing");
|
||||
let first_hash = sha256_hex(b"first-child-cert-overlay");
|
||||
let old_second_hash = sha256_hex(b"old-second-child-cert-overlay");
|
||||
let new_second_hash = sha256_hex(b"new-second-child-cert-overlay");
|
||||
let first = sample_child_certificate_cache_projection(
|
||||
first_key.clone(),
|
||||
"rsync://example.test/repo/first-overlay.cer",
|
||||
&first_hash,
|
||||
);
|
||||
let old_second = sample_child_certificate_cache_projection(
|
||||
second_key.clone(),
|
||||
"rsync://example.test/repo/second-overlay.cer",
|
||||
&old_second_hash,
|
||||
);
|
||||
let new_second = sample_child_certificate_cache_projection(
|
||||
second_key.clone(),
|
||||
"rsync://example.test/repo/second-overlay.cer",
|
||||
&new_second_hash,
|
||||
);
|
||||
|
||||
store
|
||||
.write_child_certificate_cache_mmap_segment(
|
||||
manifest_uri,
|
||||
&[first.clone(), old_second.clone()],
|
||||
)
|
||||
.expect("write initial segment");
|
||||
let stats = store
|
||||
.write_child_certificate_cache_mmap_segment_overlay(
|
||||
manifest_uri,
|
||||
&[first_key.clone(), second_key.clone(), missing_key.clone()],
|
||||
std::slice::from_ref(&new_second),
|
||||
)
|
||||
.expect("write segment overlay");
|
||||
assert_eq!(stats.new_entries, 2);
|
||||
|
||||
let got = store
|
||||
.get_child_certificate_cache_projections_mmap_segment(
|
||||
manifest_uri,
|
||||
&[first_key.clone(), second_key.clone(), missing_key],
|
||||
)
|
||||
.expect("lookup segment")
|
||||
.expect("segment exists");
|
||||
assert_eq!(got.hits, 2);
|
||||
assert_eq!(got.misses, 1);
|
||||
assert_eq!(
|
||||
got.projections[0]
|
||||
.as_ref()
|
||||
.expect("first projection")
|
||||
.child_cert_sha256_hex,
|
||||
first_hash
|
||||
);
|
||||
assert_eq!(
|
||||
got.projections[1]
|
||||
.as_ref()
|
||||
.expect("updated second projection")
|
||||
.child_cert_sha256_hex,
|
||||
new_second_hash
|
||||
);
|
||||
assert!(got.projections[2].is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn storage_helpers_cover_optional_validation_paths() {
|
||||
let withdrawn = RepositoryViewEntry {
|
||||
|
||||
@ -213,6 +213,9 @@ pub enum ManifestFreshError {
|
||||
|
||||
#[error("manifest file hash mismatch: {rsync_uri} (RFC 9286 §6.5; RFC 9286 §6.6)")]
|
||||
HashMismatch { rsync_uri: String },
|
||||
|
||||
#[error("issuer CA certificate bytes unavailable: {detail} (RFC 6487 §4; RFC 9286 §6.2)")]
|
||||
IssuerCaLoadFailed { detail: String },
|
||||
}
|
||||
|
||||
impl ManifestFreshError {
|
||||
|
||||
@ -6,7 +6,7 @@ use crate::storage::RocksStore;
|
||||
use crate::sync::rrdp::Fetcher as HttpFetcher;
|
||||
use crate::validation::manifest::PublicationPointSource;
|
||||
use crate::validation::objects::ObjectsOutput;
|
||||
use crate::validation::tree::{CaInstanceHandle, PublicationPointRunner};
|
||||
use crate::validation::tree::{CaCertificateRef, CaInstanceHandle, PublicationPointRunner};
|
||||
use crate::validation::tree_runner::Rpkiv1PublicationPointRunner;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
@ -49,7 +49,7 @@ pub fn run_publication_point_once(
|
||||
depth: 0,
|
||||
tal_id: "single-publication-point".to_string(),
|
||||
parent_manifest_rsync_uri: None,
|
||||
ca_certificate_der: issuer_ca_der.to_vec(),
|
||||
ca_certificate: CaCertificateRef::inline_der(issuer_ca_der.to_vec()),
|
||||
ca_certificate_rsync_uri: issuer_ca_rsync_uri.map(str::to_string),
|
||||
effective_ip_resources: issuer_effective_ip.cloned(),
|
||||
effective_as_resources: issuer_effective_as.cloned(),
|
||||
@ -80,6 +80,7 @@ pub fn run_publication_point_once(
|
||||
ccr_accumulator: None,
|
||||
persist_vcir: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
};
|
||||
|
||||
@ -33,6 +33,7 @@ use crate::validation::from_tal::{
|
||||
discover_root_ca_instance_from_tal_with_fetchers_strict_name,
|
||||
};
|
||||
use crate::validation::objects::ParallelRoaWorkerPool;
|
||||
use crate::validation::tree::CaCertificateRef;
|
||||
use crate::validation::tree::{
|
||||
CaInstanceHandle, TreeRunAuditOutput, TreeRunConfig, TreeRunError, TreeRunOutput,
|
||||
run_tree_serial, run_tree_serial_audit, run_tree_serial_audit_multi_root,
|
||||
@ -136,6 +137,7 @@ fn make_live_runner<'a>(
|
||||
ccr_accumulator: Option<CcrAccumulator>,
|
||||
persist_vcir: bool,
|
||||
enable_roa_validation_cache: bool,
|
||||
enable_child_certificate_validation_cache: bool,
|
||||
publication_point_cache_observe_only: bool,
|
||||
enable_publication_point_validation_cache: bool,
|
||||
) -> Rpkiv1PublicationPointRunner<'a> {
|
||||
@ -163,6 +165,7 @@ fn make_live_runner<'a>(
|
||||
ccr_accumulator: ccr_accumulator.map(Mutex::new),
|
||||
persist_vcir,
|
||||
enable_roa_validation_cache,
|
||||
enable_child_certificate_validation_cache,
|
||||
publication_point_cache_observe_only,
|
||||
enable_publication_point_validation_cache,
|
||||
}
|
||||
@ -537,7 +540,7 @@ pub fn root_handle_from_trust_anchor(
|
||||
depth: 0,
|
||||
tal_id,
|
||||
parent_manifest_rsync_uri: None,
|
||||
ca_certificate_der: trust_anchor.ta_certificate.raw_der.clone(),
|
||||
ca_certificate: CaCertificateRef::inline_der(trust_anchor.ta_certificate.raw_der.clone()),
|
||||
ca_certificate_rsync_uri,
|
||||
effective_ip_resources: ta_rc.tbs.extensions.ip_resources.clone(),
|
||||
effective_as_resources: ta_rc.tbs.extensions.as_resources.clone(),
|
||||
@ -578,6 +581,7 @@ pub fn run_tree_from_tal_url_serial(
|
||||
None,
|
||||
config.persist_vcir,
|
||||
config.enable_roa_validation_cache,
|
||||
config.enable_child_certificate_validation_cache,
|
||||
config.publication_point_cache_observe_only,
|
||||
config.enable_publication_point_validation_cache,
|
||||
);
|
||||
@ -624,6 +628,7 @@ pub fn run_tree_from_tal_url_serial_audit(
|
||||
None,
|
||||
config.persist_vcir,
|
||||
config.enable_roa_validation_cache,
|
||||
config.enable_child_certificate_validation_cache,
|
||||
config.publication_point_cache_observe_only,
|
||||
config.enable_publication_point_validation_cache,
|
||||
);
|
||||
@ -692,6 +697,7 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing(
|
||||
None,
|
||||
config.persist_vcir,
|
||||
config.enable_roa_validation_cache,
|
||||
config.enable_child_certificate_validation_cache,
|
||||
config.publication_point_cache_observe_only,
|
||||
config.enable_publication_point_validation_cache,
|
||||
);
|
||||
@ -782,6 +788,7 @@ where
|
||||
.then(|| CcrAccumulator::new(vec![discovery.trust_anchor.clone()])),
|
||||
config.persist_vcir,
|
||||
config.enable_roa_validation_cache,
|
||||
config.enable_child_certificate_validation_cache,
|
||||
config.publication_point_cache_observe_only,
|
||||
config.enable_publication_point_validation_cache,
|
||||
);
|
||||
@ -910,6 +917,7 @@ where
|
||||
}),
|
||||
config.persist_vcir,
|
||||
config.enable_roa_validation_cache,
|
||||
config.enable_child_certificate_validation_cache,
|
||||
config.publication_point_cache_observe_only,
|
||||
config.enable_publication_point_validation_cache,
|
||||
);
|
||||
@ -1350,6 +1358,7 @@ pub fn run_tree_from_tal_and_ta_der_serial(
|
||||
ccr_accumulator: None,
|
||||
persist_vcir: true,
|
||||
enable_roa_validation_cache: config.enable_roa_validation_cache,
|
||||
enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache,
|
||||
publication_point_cache_observe_only: config.publication_point_cache_observe_only,
|
||||
enable_publication_point_validation_cache: config.enable_publication_point_validation_cache,
|
||||
};
|
||||
@ -1405,6 +1414,7 @@ pub fn run_tree_from_tal_bytes_serial_audit(
|
||||
ccr_accumulator: None,
|
||||
persist_vcir: true,
|
||||
enable_roa_validation_cache: config.enable_roa_validation_cache,
|
||||
enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache,
|
||||
publication_point_cache_observe_only: config.publication_point_cache_observe_only,
|
||||
enable_publication_point_validation_cache: config.enable_publication_point_validation_cache,
|
||||
};
|
||||
@ -1482,6 +1492,7 @@ pub fn run_tree_from_tal_bytes_serial_audit_with_timing(
|
||||
ccr_accumulator: None,
|
||||
persist_vcir: true,
|
||||
enable_roa_validation_cache: config.enable_roa_validation_cache,
|
||||
enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache,
|
||||
publication_point_cache_observe_only: config.publication_point_cache_observe_only,
|
||||
enable_publication_point_validation_cache: config.enable_publication_point_validation_cache,
|
||||
};
|
||||
@ -1558,6 +1569,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit(
|
||||
ccr_accumulator: None,
|
||||
persist_vcir: true,
|
||||
enable_roa_validation_cache: config.enable_roa_validation_cache,
|
||||
enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache,
|
||||
publication_point_cache_observe_only: config.publication_point_cache_observe_only,
|
||||
enable_publication_point_validation_cache: config.enable_publication_point_validation_cache,
|
||||
};
|
||||
@ -1635,6 +1647,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing(
|
||||
ccr_accumulator: None,
|
||||
persist_vcir: true,
|
||||
enable_roa_validation_cache: config.enable_roa_validation_cache,
|
||||
enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache,
|
||||
publication_point_cache_observe_only: config.publication_point_cache_observe_only,
|
||||
enable_publication_point_validation_cache: config.enable_publication_point_validation_cache,
|
||||
};
|
||||
@ -1719,6 +1732,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial(
|
||||
ccr_accumulator: None,
|
||||
persist_vcir: true,
|
||||
enable_roa_validation_cache: config.enable_roa_validation_cache,
|
||||
enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache,
|
||||
publication_point_cache_observe_only: config.publication_point_cache_observe_only,
|
||||
enable_publication_point_validation_cache: config.enable_publication_point_validation_cache,
|
||||
};
|
||||
@ -1784,6 +1798,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit(
|
||||
ccr_accumulator: None,
|
||||
persist_vcir: true,
|
||||
enable_roa_validation_cache: config.enable_roa_validation_cache,
|
||||
enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache,
|
||||
publication_point_cache_observe_only: config.publication_point_cache_observe_only,
|
||||
enable_publication_point_validation_cache: config.enable_publication_point_validation_cache,
|
||||
};
|
||||
@ -1871,6 +1886,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing(
|
||||
ccr_accumulator: None,
|
||||
persist_vcir: true,
|
||||
enable_roa_validation_cache: config.enable_roa_validation_cache,
|
||||
enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache,
|
||||
publication_point_cache_observe_only: config.publication_point_cache_observe_only,
|
||||
enable_publication_point_validation_cache: config.enable_publication_point_validation_cache,
|
||||
};
|
||||
@ -1938,6 +1954,7 @@ fn build_payload_replay_runner<'a>(
|
||||
ccr_accumulator: None,
|
||||
persist_vcir: true,
|
||||
enable_roa_validation_cache,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
}
|
||||
@ -1975,6 +1992,7 @@ fn build_payload_delta_replay_runner<'a>(
|
||||
ccr_accumulator: None,
|
||||
persist_vcir: true,
|
||||
enable_roa_validation_cache,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
}
|
||||
@ -2012,6 +2030,7 @@ fn build_payload_delta_replay_current_store_runner<'a>(
|
||||
ccr_accumulator: None,
|
||||
persist_vcir: true,
|
||||
enable_roa_validation_cache,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
}
|
||||
@ -2577,6 +2596,7 @@ mod replay_api_tests {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
@ -2616,6 +2636,7 @@ mod replay_api_tests {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
@ -2665,6 +2686,7 @@ mod replay_api_tests {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
@ -2715,6 +2737,7 @@ mod replay_api_tests {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
@ -2775,6 +2798,7 @@ mod replay_api_tests {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
@ -2811,6 +2835,7 @@ mod replay_api_tests {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
@ -2867,6 +2892,7 @@ mod replay_api_tests {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
@ -2932,6 +2958,7 @@ mod replay_api_tests {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
|
||||
@ -7,6 +7,8 @@ use crate::validation::objects::{
|
||||
AspaAttestation, ObjectsOutput, RoaValidationCacheStats, RouterKeyPayload, Vrp,
|
||||
};
|
||||
use crate::validation::publication_point::PublicationPointSnapshot;
|
||||
use std::borrow::Cow;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TreeRunConfig {
|
||||
@ -22,6 +24,8 @@ pub struct TreeRunConfig {
|
||||
pub build_ccr_accumulator: bool,
|
||||
/// Reuse accepted ROA validation outputs from previous VCIR when explicitly enabled.
|
||||
pub enable_roa_validation_cache: bool,
|
||||
/// Reuse validated child certificate discovery results when explicitly enabled.
|
||||
pub enable_child_certificate_validation_cache: bool,
|
||||
/// Evaluate publication-point cache eligibility without changing validation results.
|
||||
pub publication_point_cache_observe_only: bool,
|
||||
/// Reuse complete publication-point validation projections when explicitly enabled.
|
||||
@ -39,6 +43,7 @@ impl Default for TreeRunConfig {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
@ -46,13 +51,85 @@ impl Default for TreeRunConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum CaCertificateRef {
|
||||
InlineDer(Vec<u8>),
|
||||
RepoBytes {
|
||||
sha256_hex: String,
|
||||
cached_der: Arc<OnceLock<Arc<[u8]>>>,
|
||||
},
|
||||
}
|
||||
|
||||
impl PartialEq for CaCertificateRef {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
match (self, other) {
|
||||
(Self::InlineDer(left), Self::InlineDer(right)) => left == right,
|
||||
(
|
||||
Self::RepoBytes {
|
||||
sha256_hex: left, ..
|
||||
},
|
||||
Self::RepoBytes {
|
||||
sha256_hex: right, ..
|
||||
},
|
||||
) => left == right,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for CaCertificateRef {}
|
||||
|
||||
impl CaCertificateRef {
|
||||
pub fn inline_der(bytes: Vec<u8>) -> Self {
|
||||
Self::InlineDer(bytes)
|
||||
}
|
||||
|
||||
pub fn repo_bytes(sha256_hex: String) -> Self {
|
||||
Self::RepoBytes {
|
||||
sha256_hex,
|
||||
cached_der: Arc::new(OnceLock::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn sha256_hex(&self) -> Option<&str> {
|
||||
match self {
|
||||
Self::InlineDer(_) => None,
|
||||
Self::RepoBytes { sha256_hex, .. } => Some(sha256_hex.as_str()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn der<'a>(&'a self, store: &crate::storage::RocksStore) -> Result<Cow<'a, [u8]>, String> {
|
||||
match self {
|
||||
Self::InlineDer(bytes) => Ok(Cow::Borrowed(bytes.as_slice())),
|
||||
Self::RepoBytes {
|
||||
sha256_hex,
|
||||
cached_der,
|
||||
} => {
|
||||
if cached_der.get().is_none() {
|
||||
let bytes = store
|
||||
.get_blob_bytes(sha256_hex)
|
||||
.map_err(|e| format!("load CA certificate bytes failed: {e}"))?
|
||||
.ok_or_else(|| {
|
||||
format!("missing CA certificate repo bytes for sha256={sha256_hex}")
|
||||
})?;
|
||||
let _ = cached_der.set(Arc::from(bytes));
|
||||
}
|
||||
let bytes = cached_der.get().ok_or_else(|| {
|
||||
format!("missing cached CA certificate bytes for sha256={sha256_hex}")
|
||||
})?;
|
||||
Ok(Cow::Borrowed(bytes.as_ref()))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct CaInstanceHandle {
|
||||
pub depth: usize,
|
||||
pub tal_id: String,
|
||||
pub parent_manifest_rsync_uri: Option<String>,
|
||||
/// DER bytes of the CA certificate for this CA instance.
|
||||
pub ca_certificate_der: Vec<u8>,
|
||||
/// CA certificate bytes or a lazy repo-bytes reference for this CA instance.
|
||||
pub ca_certificate: CaCertificateRef,
|
||||
/// rsync URI of this CA certificate object (where it is published).
|
||||
///
|
||||
/// This is used for strict AIA binding checks (RFC 6487 §4.8.7) when validating
|
||||
@ -74,6 +151,32 @@ impl CaInstanceHandle {
|
||||
self.depth = depth;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn ca_certificate_der<'a>(
|
||||
&'a self,
|
||||
store: &crate::storage::RocksStore,
|
||||
) -> Result<Cow<'a, [u8]>, String> {
|
||||
self.ca_certificate.der(store)
|
||||
}
|
||||
|
||||
pub fn ca_certificate_sha256_hex(&self) -> Option<&str> {
|
||||
self.ca_certificate.sha256_hex()
|
||||
}
|
||||
|
||||
pub fn ca_certificate_sha256_32(&self) -> Option<[u8; 32]> {
|
||||
match &self.ca_certificate {
|
||||
CaCertificateRef::InlineDer(bytes) => {
|
||||
use sha2::Digest as _;
|
||||
let digest = sha2::Sha256::digest(bytes);
|
||||
Some(digest.into())
|
||||
}
|
||||
CaCertificateRef::RepoBytes { sha256_hex, .. } => {
|
||||
let mut out = [0u8; 32];
|
||||
hex::decode_to_slice(sha256_hex, &mut out).ok()?;
|
||||
Some(out)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
@ -337,7 +440,7 @@ mod tests {
|
||||
use crate::validation::objects::{ObjectsOutput, ObjectsStats};
|
||||
|
||||
use super::{
|
||||
CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult,
|
||||
CaCertificateRef, CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult,
|
||||
PublicationPointRunner, TreeRunConfig, run_tree_serial_audit,
|
||||
run_tree_serial_audit_multi_root,
|
||||
};
|
||||
@ -347,7 +450,7 @@ mod tests {
|
||||
depth: 0,
|
||||
tal_id: "arin".to_string(),
|
||||
parent_manifest_rsync_uri: None,
|
||||
ca_certificate_der: vec![1, 2, 3],
|
||||
ca_certificate: CaCertificateRef::inline_der(vec![1, 2, 3]),
|
||||
ca_certificate_rsync_uri: None,
|
||||
effective_ip_resources: None,
|
||||
effective_as_resources: None,
|
||||
|
||||
@ -994,6 +994,11 @@ fn stage_ready_publication_point(
|
||||
result.discovered_children.clone(),
|
||||
);
|
||||
metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started);
|
||||
runner.record_publication_point_step_ms(
|
||||
metrics.manifest_rsync_uri.as_deref().unwrap_or_default(),
|
||||
"publication_point_cache_child_enqueue",
|
||||
metrics.child_enqueue_ms,
|
||||
);
|
||||
finished.push(FinishedPublicationPoint {
|
||||
node: FinishedPublicationPointNode::from_queued(ready.node),
|
||||
result: compact_phase2_finished_result(result, compact_audit),
|
||||
@ -1182,7 +1187,7 @@ fn stage_ready_publication_point(
|
||||
ready.node.id,
|
||||
&fresh_stage.fresh_point,
|
||||
runner.policy,
|
||||
&ready.node.handle.ca_certificate_der,
|
||||
fresh_stage.issuer_ca_der.as_ref(),
|
||||
ready.node.handle.ca_certificate_rsync_uri.as_deref(),
|
||||
ready.node.handle.effective_ip_resources.as_ref(),
|
||||
ready.node.handle.effective_as_resources.as_ref(),
|
||||
@ -2420,7 +2425,7 @@ mod tests {
|
||||
.push(crate::validation::tree::DiscoveredChildCaInstance {
|
||||
handle: crate::validation::tree::CaInstanceHandle {
|
||||
tal_id: "test".to_string(),
|
||||
ca_certificate_der: vec![1],
|
||||
ca_certificate: crate::validation::tree::CaCertificateRef::inline_der(vec![1]),
|
||||
ca_certificate_rsync_uri: Some(
|
||||
"rsync://example.test/repo/child.cer".to_string(),
|
||||
),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -190,6 +190,7 @@ fn apnic_tree_full_stats_serial() {
|
||||
ccr_accumulator: None,
|
||||
persist_vcir: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
};
|
||||
@ -224,6 +225,7 @@ fn apnic_tree_full_stats_serial() {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
|
||||
@ -40,6 +40,7 @@ fn apnic_tree_depth1_processes_more_than_root() {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
@ -85,6 +86,7 @@ fn apnic_tree_root_only_processes_root_with_long_timeouts() {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
|
||||
@ -5,8 +5,8 @@ use rpki::validation::manifest::PublicationPointSource;
|
||||
use rpki::validation::objects::process_publication_point_snapshot_for_issuer;
|
||||
use rpki::validation::publication_point::PublicationPointSnapshot;
|
||||
use rpki::validation::tree::{
|
||||
CaInstanceHandle, PublicationPointRunResult, PublicationPointRunner, TreeRunConfig,
|
||||
run_tree_serial_audit,
|
||||
CaCertificateRef, CaInstanceHandle, PublicationPointRunResult, PublicationPointRunner,
|
||||
TreeRunConfig, run_tree_serial_audit,
|
||||
};
|
||||
|
||||
fn fixture_bytes(path: &str) -> Vec<u8> {
|
||||
@ -46,7 +46,10 @@ impl PublicationPointRunner for SinglePackRunner {
|
||||
let objects = process_publication_point_snapshot_for_issuer(
|
||||
&self.snapshot,
|
||||
&self.policy,
|
||||
&ca.ca_certificate_der,
|
||||
match &ca.ca_certificate {
|
||||
CaCertificateRef::InlineDer(bytes) => bytes.as_slice(),
|
||||
other => panic!("test runner expects inline CA DER, got {other:?}"),
|
||||
},
|
||||
ca.ca_certificate_rsync_uri.as_deref(),
|
||||
ca.effective_ip_resources.as_ref(),
|
||||
ca.effective_as_resources.as_ref(),
|
||||
@ -92,7 +95,9 @@ fn crl_mismatch_drops_publication_point_and_cites_rfc_sections() {
|
||||
parent_manifest_rsync_uri: None,
|
||||
// Use a real, parseable CA certificate DER so objects processing can reach CRL selection.
|
||||
// The test only asserts CRLDP/locked-pack error handling, not signature chaining.
|
||||
ca_certificate_der: fixture_bytes("tests/fixtures/ta/apnic-ta.cer"),
|
||||
ca_certificate: CaCertificateRef::inline_der(fixture_bytes(
|
||||
"tests/fixtures/ta/apnic-ta.cer",
|
||||
)),
|
||||
ca_certificate_rsync_uri: None,
|
||||
effective_ip_resources: None,
|
||||
effective_as_resources: None,
|
||||
@ -114,6 +119,7 @@ fn crl_mismatch_drops_publication_point_and_cites_rfc_sections() {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
|
||||
@ -6,7 +6,7 @@ use rpki::validation::run_tree_from_tal::{
|
||||
run_tree_from_tal_and_ta_der_serial_audit_with_timing, run_tree_from_tal_url_serial,
|
||||
run_tree_from_tal_url_serial_audit, run_tree_from_tal_url_serial_audit_with_timing,
|
||||
};
|
||||
use rpki::validation::tree::TreeRunConfig;
|
||||
use rpki::validation::tree::{CaCertificateRef, TreeRunConfig};
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
@ -71,10 +71,12 @@ fn root_handle_is_constructible_from_fixture_tal_and_ta() {
|
||||
discovery.ca_instance.manifest_rsync_uri
|
||||
);
|
||||
assert_eq!(root.rsync_base_uri, discovery.ca_instance.rsync_base_uri);
|
||||
assert!(
|
||||
root.ca_certificate_der.len() > 100,
|
||||
"TA der should be non-empty"
|
||||
);
|
||||
match &root.ca_certificate {
|
||||
CaCertificateRef::InlineDer(bytes) => {
|
||||
assert!(bytes.len() > 100, "TA der should be non-empty");
|
||||
}
|
||||
other => panic!("TA root certificate should be inline DER, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@ -118,6 +120,7 @@ fn run_tree_from_tal_url_entry_executes_and_records_failure_when_repo_empty() {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
@ -169,6 +172,7 @@ fn run_tree_from_tal_and_ta_der_entry_executes_and_records_failure_when_repo_emp
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
@ -228,6 +232,7 @@ fn run_tree_from_tal_url_audit_entry_collects_no_publication_points_when_repo_em
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
@ -275,6 +280,7 @@ fn run_tree_from_tal_and_ta_der_audit_entry_collects_no_publication_points_when_
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
@ -330,6 +336,7 @@ fn run_tree_from_tal_url_audit_with_timing_records_phases_when_repo_empty() {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
@ -384,6 +391,7 @@ fn run_tree_from_tal_and_ta_der_audit_with_timing_records_phases_when_repo_empty
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
|
||||
@ -7,8 +7,8 @@ use rpki::validation::manifest::PublicationPointSource;
|
||||
use rpki::validation::objects::{ObjectsOutput, ObjectsStats};
|
||||
use rpki::validation::publication_point::PublicationPointSnapshot;
|
||||
use rpki::validation::tree::{
|
||||
CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult, PublicationPointRunner,
|
||||
TreeRunConfig, run_tree_serial,
|
||||
CaCertificateRef, CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult,
|
||||
PublicationPointRunner, TreeRunConfig, run_tree_serial,
|
||||
};
|
||||
|
||||
fn empty_snapshot(manifest_uri: &str, pp_uri: &str) -> PublicationPointSnapshot {
|
||||
@ -36,7 +36,7 @@ fn ca_handle(manifest_uri: &str) -> CaInstanceHandle {
|
||||
depth: 0,
|
||||
tal_id: "test-tal".to_string(),
|
||||
parent_manifest_rsync_uri: None,
|
||||
ca_certificate_der: Vec::new(),
|
||||
ca_certificate: CaCertificateRef::inline_der(Vec::new()),
|
||||
ca_certificate_rsync_uri: None,
|
||||
effective_ip_resources: None,
|
||||
effective_as_resources: None,
|
||||
|
||||
@ -7,8 +7,8 @@ use rpki::validation::manifest::PublicationPointSource;
|
||||
use rpki::validation::objects::{ObjectsOutput, ObjectsStats, RouterKeyPayload};
|
||||
use rpki::validation::publication_point::PublicationPointSnapshot;
|
||||
use rpki::validation::tree::{
|
||||
CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult, PublicationPointRunner,
|
||||
TreeRunConfig, run_tree_serial, run_tree_serial_audit,
|
||||
CaCertificateRef, CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult,
|
||||
PublicationPointRunner, TreeRunConfig, run_tree_serial, run_tree_serial_audit,
|
||||
};
|
||||
|
||||
#[derive(Default)]
|
||||
@ -69,7 +69,7 @@ fn ca_handle(manifest_uri: &str) -> CaInstanceHandle {
|
||||
depth: 0,
|
||||
tal_id: "test-tal".to_string(),
|
||||
parent_manifest_rsync_uri: None,
|
||||
ca_certificate_der: Vec::new(),
|
||||
ca_certificate: CaCertificateRef::inline_der(Vec::new()),
|
||||
ca_certificate_rsync_uri: None,
|
||||
effective_ip_resources: None,
|
||||
effective_as_resources: None,
|
||||
@ -314,6 +314,7 @@ fn tree_respects_max_depth_and_max_instances() {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
@ -333,6 +334,7 @@ fn tree_respects_max_depth_and_max_instances() {
|
||||
persist_vcir: true,
|
||||
build_ccr_accumulator: true,
|
||||
enable_roa_validation_cache: false,
|
||||
enable_child_certificate_validation_cache: false,
|
||||
publication_point_cache_observe_only: false,
|
||||
enable_publication_point_validation_cache: false,
|
||||
enable_transport_request_prefetch: false,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user