20260416_2 并行优化phase1后进行snapshot fast path优化,通过四方面关键优化技术消除了验证主链路上snapshot构建是的db访问性能热点问题,性能热点转移到ROA验证处理本身,目前APNIC+ARIN全量同步从500秒压缩到212秒,离rpkclient 112秒差距变小
This commit is contained in:
parent
38421b1ae7
commit
224ae10052
@ -18,7 +18,17 @@ ANALYZE_ROOT="$ROOT_DIR/target/live/analyze"
|
|||||||
mkdir -p "$ANALYZE_ROOT"
|
mkdir -p "$ANALYZE_ROOT"
|
||||||
mapfile -t ANALYZE_BEFORE < <(find "$ANALYZE_ROOT" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort)
|
mapfile -t ANALYZE_BEFORE < <(find "$ANALYZE_ROOT" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort)
|
||||||
|
|
||||||
eval "$(python3 "$CASE_INFO_SCRIPT" --bundle-root "$BUNDLE_ROOT" --rir apnic --format env)"
|
if [[ "${DRY_RUN:-0}" == "1" && ! -d "$BUNDLE_ROOT" ]]; then
|
||||||
|
TRUST_ANCHOR="apnic"
|
||||||
|
TAL_PATH="$ROOT_DIR/tests/fixtures/tal/apnic-rfc7730-https.tal"
|
||||||
|
TA_PATH="$ROOT_DIR/tests/fixtures/ta/apnic-ta.cer"
|
||||||
|
PAYLOAD_REPLAY_ARCHIVE="$BUNDLE_ROOT/apnic/base-payload-archive"
|
||||||
|
PAYLOAD_REPLAY_LOCKS="$BUNDLE_ROOT/apnic/base-locks.json"
|
||||||
|
SNAPSHOT_VALIDATION_TIME="2026-03-16T00:00:00Z"
|
||||||
|
ROUTINATOR_BASE_REPLAY_SECONDS="0"
|
||||||
|
else
|
||||||
|
eval "$(python3 "$CASE_INFO_SCRIPT" --bundle-root "$BUNDLE_ROOT" --rir apnic --format env)"
|
||||||
|
fi
|
||||||
|
|
||||||
DB_DIR="${DB_DIR:-$RUN_DIR/db}"
|
DB_DIR="${DB_DIR:-$RUN_DIR/db}"
|
||||||
REPORT_JSON="${REPORT_JSON:-$RUN_DIR/report.json}"
|
REPORT_JSON="${REPORT_JSON:-$RUN_DIR/report.json}"
|
||||||
|
|||||||
@ -1,3 +1,6 @@
|
|||||||
|
use crate::data_model::aspa::AspaObject;
|
||||||
|
use crate::data_model::manifest::ManifestObject;
|
||||||
|
use crate::data_model::roa::RoaObject;
|
||||||
use crate::storage::{
|
use crate::storage::{
|
||||||
AuditRuleIndexEntry, AuditRuleKind, RawByHashEntry, RocksStore, ValidatedCaInstanceResult,
|
AuditRuleIndexEntry, AuditRuleKind, RawByHashEntry, RocksStore, ValidatedCaInstanceResult,
|
||||||
VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirLocalOutput,
|
VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirLocalOutput,
|
||||||
@ -132,7 +135,7 @@ pub fn trace_rule_to_root(
|
|||||||
rule,
|
rule,
|
||||||
resolved_output: resolved_output_from_local(&local_output),
|
resolved_output: resolved_output_from_local(&local_output),
|
||||||
source_object_raw: resolve_raw_ref(store, &local_output.source_object_hash)?,
|
source_object_raw: resolve_raw_ref(store, &local_output.source_object_hash)?,
|
||||||
source_ee_cert_raw: resolve_raw_ref(store, &local_output.source_ee_cert_hash)?,
|
source_ee_cert_raw: resolve_source_ee_cert_raw_ref(store, &local_output)?,
|
||||||
chain_leaf_to_root: chain,
|
chain_leaf_to_root: chain,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
@ -227,6 +230,69 @@ fn resolve_raw_ref(
|
|||||||
Ok(raw_ref_from_entry(sha256_hex, raw.as_ref()))
|
Ok(raw_ref_from_entry(sha256_hex, raw.as_ref()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn resolve_source_ee_cert_raw_ref(
|
||||||
|
store: &RocksStore,
|
||||||
|
local: &VcirLocalOutput,
|
||||||
|
) -> Result<AuditTraceRawRef, AuditTraceError> {
|
||||||
|
let raw = store.get_raw_by_hash_entry(&local.source_ee_cert_hash)?;
|
||||||
|
if raw.is_some() {
|
||||||
|
return Ok(raw_ref_from_entry(&local.source_ee_cert_hash, raw.as_ref()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let source_raw = store.get_raw_by_hash_entry(&local.source_object_hash)?;
|
||||||
|
let Some(source_raw) = source_raw else {
|
||||||
|
return Ok(raw_ref_from_entry(&local.source_ee_cert_hash, None));
|
||||||
|
};
|
||||||
|
|
||||||
|
let derived = match local.source_object_type.as_str() {
|
||||||
|
"roa" => RoaObject::decode_der(&source_raw.bytes)
|
||||||
|
.ok()
|
||||||
|
.and_then(|roa| {
|
||||||
|
roa.signed_object
|
||||||
|
.signed_data
|
||||||
|
.certificates
|
||||||
|
.first()
|
||||||
|
.map(|cert| cert.raw_der.to_vec())
|
||||||
|
}),
|
||||||
|
"aspa" => AspaObject::decode_der(&source_raw.bytes)
|
||||||
|
.ok()
|
||||||
|
.and_then(|aspa| {
|
||||||
|
aspa.signed_object
|
||||||
|
.signed_data
|
||||||
|
.certificates
|
||||||
|
.first()
|
||||||
|
.map(|cert| cert.raw_der.to_vec())
|
||||||
|
}),
|
||||||
|
"mft" => ManifestObject::decode_der(&source_raw.bytes)
|
||||||
|
.ok()
|
||||||
|
.and_then(|manifest| {
|
||||||
|
manifest
|
||||||
|
.signed_object
|
||||||
|
.signed_data
|
||||||
|
.certificates
|
||||||
|
.first()
|
||||||
|
.map(|cert| cert.raw_der.to_vec())
|
||||||
|
}),
|
||||||
|
"router_key" => Some(source_raw.bytes.clone()),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
|
||||||
|
let Some(ee_der) = derived else {
|
||||||
|
return Ok(raw_ref_from_entry(&local.source_ee_cert_hash, None));
|
||||||
|
};
|
||||||
|
if crate::audit::sha256_hex(ee_der.as_slice()) != local.source_ee_cert_hash {
|
||||||
|
return Ok(raw_ref_from_entry(&local.source_ee_cert_hash, None));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(AuditTraceRawRef {
|
||||||
|
sha256_hex: local.source_ee_cert_hash.clone(),
|
||||||
|
raw_present: true,
|
||||||
|
origin_uris: Vec::new(),
|
||||||
|
object_type: Some("cer".to_string()),
|
||||||
|
byte_len: Some(ee_der.len()),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn raw_ref_from_entry(sha256_hex: &str, entry: Option<&RawByHashEntry>) -> AuditTraceRawRef {
|
fn raw_ref_from_entry(sha256_hex: &str, entry: Option<&RawByHashEntry>) -> AuditTraceRawRef {
|
||||||
match entry {
|
match entry {
|
||||||
Some(entry) => AuditTraceRawRef {
|
Some(entry) => AuditTraceRawRef {
|
||||||
@ -250,6 +316,7 @@ fn raw_ref_from_entry(sha256_hex: &str, entry: Option<&RawByHashEntry>) -> Audit
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::audit::sha256_hex;
|
use crate::audit::sha256_hex;
|
||||||
|
use crate::data_model::roa::RoaObject;
|
||||||
use crate::storage::{
|
use crate::storage::{
|
||||||
PackTime, ValidatedManifestMeta, VcirAuditSummary, VcirChildEntry, VcirInstanceGate,
|
PackTime, ValidatedManifestMeta, VcirAuditSummary, VcirChildEntry, VcirInstanceGate,
|
||||||
VcirRelatedArtifact, VcirSummary,
|
VcirRelatedArtifact, VcirSummary,
|
||||||
@ -536,6 +603,69 @@ mod tests {
|
|||||||
assert_eq!(trace.resolved_output.output_type, VcirOutputType::RouterKey);
|
assert_eq!(trace.resolved_output.output_type, VcirOutputType::RouterKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn trace_rule_to_root_lazily_derives_source_ee_cert_when_raw_is_missing() {
|
||||||
|
let store_dir = tempfile::tempdir().expect("store dir");
|
||||||
|
let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
|
||||||
|
let manifest = "rsync://example.test/leaf/leaf.mft";
|
||||||
|
let roa_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142071.roa");
|
||||||
|
let roa_bytes = std::fs::read(&roa_path).expect("read ROA fixture");
|
||||||
|
let roa = RoaObject::decode_der(&roa_bytes).expect("decode ROA fixture");
|
||||||
|
let local = VcirLocalOutput {
|
||||||
|
output_id: sha256_hex(b"lazy-vrp-output"),
|
||||||
|
output_type: VcirOutputType::Vrp,
|
||||||
|
item_effective_until: PackTime::from_utc_offset_datetime(
|
||||||
|
time::OffsetDateTime::now_utc() + time::Duration::minutes(30),
|
||||||
|
),
|
||||||
|
source_object_uri: "rsync://example.test/leaf/a.roa".to_string(),
|
||||||
|
source_object_type: "roa".to_string(),
|
||||||
|
source_object_hash: sha256_hex(&roa_bytes),
|
||||||
|
source_ee_cert_hash: sha256_hex(roa.signed_object.signed_data.certificates[0].raw_der.as_slice()),
|
||||||
|
payload_json:
|
||||||
|
serde_json::json!({"asn": 64496, "prefix": "203.0.113.0/24", "max_length": 24})
|
||||||
|
.to_string(),
|
||||||
|
rule_hash: sha256_hex(b"lazy-roa-rule"),
|
||||||
|
validation_path_hint: vec![manifest.to_string()],
|
||||||
|
};
|
||||||
|
let vcir = sample_vcir(
|
||||||
|
manifest,
|
||||||
|
None,
|
||||||
|
"test-tal",
|
||||||
|
Some(local.clone()),
|
||||||
|
sample_artifacts(manifest, &local.source_object_hash),
|
||||||
|
);
|
||||||
|
store.put_vcir(&vcir).expect("put vcir");
|
||||||
|
let rule_entry = AuditRuleIndexEntry {
|
||||||
|
kind: AuditRuleKind::Roa,
|
||||||
|
rule_hash: local.rule_hash.clone(),
|
||||||
|
manifest_rsync_uri: manifest.to_string(),
|
||||||
|
source_object_uri: local.source_object_uri.clone(),
|
||||||
|
source_object_hash: local.source_object_hash.clone(),
|
||||||
|
output_id: local.output_id.clone(),
|
||||||
|
item_effective_until: local.item_effective_until.clone(),
|
||||||
|
};
|
||||||
|
store
|
||||||
|
.put_audit_rule_index_entry(&rule_entry)
|
||||||
|
.expect("put rule index");
|
||||||
|
|
||||||
|
put_raw_evidence(&store, manifest.as_bytes(), manifest, "mft");
|
||||||
|
put_raw_evidence(
|
||||||
|
&store,
|
||||||
|
format!("{}-crl", manifest).as_bytes(),
|
||||||
|
&manifest.replace(".mft", ".crl"),
|
||||||
|
"crl",
|
||||||
|
);
|
||||||
|
put_raw_evidence(&store, &roa_bytes, &local.source_object_uri, "roa");
|
||||||
|
|
||||||
|
let trace = trace_rule_to_root(&store, AuditRuleKind::Roa, &local.rule_hash)
|
||||||
|
.expect("trace rule")
|
||||||
|
.expect("trace exists");
|
||||||
|
assert!(trace.source_object_raw.raw_present);
|
||||||
|
assert!(trace.source_ee_cert_raw.raw_present);
|
||||||
|
assert_eq!(trace.source_ee_cert_raw.object_type.as_deref(), Some("cer"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn trace_rule_to_root_returns_none_for_missing_rule_index() {
|
fn trace_rule_to_root_returns_none_for_missing_rule_index() {
|
||||||
let store_dir = tempfile::tempdir().expect("store dir");
|
let store_dir = tempfile::tempdir().expect("store dir");
|
||||||
|
|||||||
@ -1,7 +1,7 @@
|
|||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
fn usage() -> &'static str {
|
fn usage() -> &'static str {
|
||||||
"Usage: cir_materialize --cir <path> (--static-root <path> | --raw-store-db <path>) --mirror-root <path> [--keep-db]"
|
"Usage: cir_materialize --cir <path> (--repo-bytes-db <path> | --raw-store-db <path> | --static-root <path>) --mirror-root <path> [--keep-db]"
|
||||||
}
|
}
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
@ -14,6 +14,7 @@ fn main() {
|
|||||||
fn run(argv: Vec<String>) -> Result<(), String> {
|
fn run(argv: Vec<String>) -> Result<(), String> {
|
||||||
let mut cir_path: Option<PathBuf> = None;
|
let mut cir_path: Option<PathBuf> = None;
|
||||||
let mut static_root: Option<PathBuf> = None;
|
let mut static_root: Option<PathBuf> = None;
|
||||||
|
let mut repo_bytes_db: Option<PathBuf> = None;
|
||||||
let mut raw_store_db: Option<PathBuf> = None;
|
let mut raw_store_db: Option<PathBuf> = None;
|
||||||
let mut mirror_root: Option<PathBuf> = None;
|
let mut mirror_root: Option<PathBuf> = None;
|
||||||
let mut keep_db = false;
|
let mut keep_db = false;
|
||||||
@ -32,6 +33,12 @@ fn run(argv: Vec<String>) -> Result<(), String> {
|
|||||||
argv.get(i).ok_or("--static-root requires a value")?,
|
argv.get(i).ok_or("--static-root requires a value")?,
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
"--repo-bytes-db" => {
|
||||||
|
i += 1;
|
||||||
|
repo_bytes_db = Some(PathBuf::from(
|
||||||
|
argv.get(i).ok_or("--repo-bytes-db requires a value")?,
|
||||||
|
));
|
||||||
|
}
|
||||||
"--raw-store-db" => {
|
"--raw-store-db" => {
|
||||||
i += 1;
|
i += 1;
|
||||||
raw_store_db = Some(PathBuf::from(
|
raw_store_db = Some(PathBuf::from(
|
||||||
@ -53,10 +60,11 @@ fn run(argv: Vec<String>) -> Result<(), String> {
|
|||||||
let cir_path = cir_path.ok_or_else(|| format!("--cir is required\n\n{}", usage()))?;
|
let cir_path = cir_path.ok_or_else(|| format!("--cir is required\n\n{}", usage()))?;
|
||||||
let mirror_root =
|
let mirror_root =
|
||||||
mirror_root.ok_or_else(|| format!("--mirror-root is required\n\n{}", usage()))?;
|
mirror_root.ok_or_else(|| format!("--mirror-root is required\n\n{}", usage()))?;
|
||||||
let backend_count = static_root.is_some() as u8 + raw_store_db.is_some() as u8;
|
let backend_count =
|
||||||
|
static_root.is_some() as u8 + raw_store_db.is_some() as u8 + repo_bytes_db.is_some() as u8;
|
||||||
if backend_count != 1 {
|
if backend_count != 1 {
|
||||||
return Err(format!(
|
return Err(format!(
|
||||||
"must specify exactly one of --static-root or --raw-store-db\n\n{}",
|
"must specify exactly one of --repo-bytes-db, --raw-store-db or --static-root\n\n{}",
|
||||||
usage()
|
usage()
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
@ -68,6 +76,9 @@ fn run(argv: Vec<String>) -> Result<(), String> {
|
|||||||
let result = if let Some(static_root) = static_root {
|
let result = if let Some(static_root) = static_root {
|
||||||
rpki::cir::materialize_cir(&cir, &static_root, &mirror_root, true)
|
rpki::cir::materialize_cir(&cir, &static_root, &mirror_root, true)
|
||||||
.map_err(|e| e.to_string())
|
.map_err(|e| e.to_string())
|
||||||
|
} else if let Some(repo_bytes_db) = repo_bytes_db {
|
||||||
|
rpki::cir::materialize_cir_from_repo_bytes(&cir, &repo_bytes_db, &mirror_root, true)
|
||||||
|
.map_err(|e| e.to_string())
|
||||||
} else if let Some(raw_store_db) = raw_store_db {
|
} else if let Some(raw_store_db) = raw_store_db {
|
||||||
rpki::cir::materialize_cir_from_raw_store(&cir, &raw_store_db, &mirror_root, true)
|
rpki::cir::materialize_cir_from_raw_store(&cir, &raw_store_db, &mirror_root, true)
|
||||||
.map_err(|e| e.to_string())
|
.map_err(|e| e.to_string())
|
||||||
|
|||||||
@ -6,11 +6,16 @@ use rocksdb::{DB, Options, WriteBatch};
|
|||||||
use crate::storage::{RawByHashEntry, RocksStore, StorageError, StorageResult};
|
use crate::storage::{RawByHashEntry, RocksStore, StorageError, StorageResult};
|
||||||
|
|
||||||
const RAW_BY_HASH_KEY_PREFIX: &str = "rawbyhash:";
|
const RAW_BY_HASH_KEY_PREFIX: &str = "rawbyhash:";
|
||||||
|
const RAW_BLOB_KEY_PREFIX: &str = "rawblob:";
|
||||||
|
|
||||||
fn raw_by_hash_key(sha256_hex: &str) -> String {
|
fn raw_by_hash_key(sha256_hex: &str) -> String {
|
||||||
format!("{RAW_BY_HASH_KEY_PREFIX}{sha256_hex}")
|
format!("{RAW_BY_HASH_KEY_PREFIX}{sha256_hex}")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn raw_blob_key(sha256_hex: &str) -> String {
|
||||||
|
format!("{RAW_BLOB_KEY_PREFIX}{sha256_hex}")
|
||||||
|
}
|
||||||
|
|
||||||
pub trait RawObjectStore {
|
pub trait RawObjectStore {
|
||||||
fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>>;
|
fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>>;
|
||||||
|
|
||||||
@ -24,10 +29,7 @@ pub trait RawObjectStore {
|
|||||||
.map(|entry| entry.map(|entry| entry.bytes))
|
.map(|entry| entry.map(|entry| entry.bytes))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_blob_bytes_batch(
|
fn get_blob_bytes_batch(&self, sha256_hexes: &[String]) -> StorageResult<Vec<Option<Vec<u8>>>> {
|
||||||
&self,
|
|
||||||
sha256_hexes: &[String],
|
|
||||||
) -> StorageResult<Vec<Option<Vec<u8>>>> {
|
|
||||||
self.get_raw_entries_batch(sha256_hexes).map(|entries| {
|
self.get_raw_entries_batch(sha256_hexes).map(|entries| {
|
||||||
entries
|
entries
|
||||||
.into_iter()
|
.into_iter()
|
||||||
@ -62,10 +64,19 @@ impl ExternalRawStoreDb {
|
|||||||
pub fn put_raw_entry(&self, entry: &RawByHashEntry) -> StorageResult<()> {
|
pub fn put_raw_entry(&self, entry: &RawByHashEntry) -> StorageResult<()> {
|
||||||
entry.validate_internal()?;
|
entry.validate_internal()?;
|
||||||
let key = raw_by_hash_key(&entry.sha256_hex);
|
let key = raw_by_hash_key(&entry.sha256_hex);
|
||||||
let value =
|
let blob_key = raw_blob_key(&entry.sha256_hex);
|
||||||
serde_cbor::to_vec(entry).map_err(|e| StorageError::Codec { entity: "raw_by_hash", detail: e.to_string() })?;
|
let value = serde_cbor::to_vec(entry).map_err(|e| StorageError::Codec {
|
||||||
|
entity: "raw_by_hash",
|
||||||
|
detail: e.to_string(),
|
||||||
|
})?;
|
||||||
|
let blob_value = entry.bytes.clone();
|
||||||
self.db
|
self.db
|
||||||
.put(key.as_bytes(), value)
|
.write({
|
||||||
|
let mut batch = WriteBatch::default();
|
||||||
|
batch.put(key.as_bytes(), value);
|
||||||
|
batch.put(blob_key.as_bytes(), blob_value);
|
||||||
|
batch
|
||||||
|
})
|
||||||
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@ -78,11 +89,13 @@ impl ExternalRawStoreDb {
|
|||||||
for entry in entries {
|
for entry in entries {
|
||||||
entry.validate_internal()?;
|
entry.validate_internal()?;
|
||||||
let key = raw_by_hash_key(&entry.sha256_hex);
|
let key = raw_by_hash_key(&entry.sha256_hex);
|
||||||
|
let blob_key = raw_blob_key(&entry.sha256_hex);
|
||||||
let value = serde_cbor::to_vec(entry).map_err(|e| StorageError::Codec {
|
let value = serde_cbor::to_vec(entry).map_err(|e| StorageError::Codec {
|
||||||
entity: "raw_by_hash",
|
entity: "raw_by_hash",
|
||||||
detail: e.to_string(),
|
detail: e.to_string(),
|
||||||
})?;
|
})?;
|
||||||
batch.put(key.as_bytes(), value);
|
batch.put(key.as_bytes(), value);
|
||||||
|
batch.put(blob_key.as_bytes(), entry.bytes.as_slice());
|
||||||
}
|
}
|
||||||
self.db
|
self.db
|
||||||
.write(batch)
|
.write(batch)
|
||||||
@ -92,8 +105,14 @@ impl ExternalRawStoreDb {
|
|||||||
|
|
||||||
pub fn delete_raw_entry(&self, sha256_hex: &str) -> StorageResult<()> {
|
pub fn delete_raw_entry(&self, sha256_hex: &str) -> StorageResult<()> {
|
||||||
let key = raw_by_hash_key(sha256_hex);
|
let key = raw_by_hash_key(sha256_hex);
|
||||||
|
let blob_key = raw_blob_key(sha256_hex);
|
||||||
self.db
|
self.db
|
||||||
.delete(key.as_bytes())
|
.write({
|
||||||
|
let mut batch = WriteBatch::default();
|
||||||
|
batch.delete(key.as_bytes());
|
||||||
|
batch.delete(blob_key.as_bytes());
|
||||||
|
batch
|
||||||
|
})
|
||||||
.map_err(|e| StorageError::RocksDb(e.to_string()))
|
.map_err(|e| StorageError::RocksDb(e.to_string()))
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -113,6 +132,26 @@ impl RawObjectStore for RocksStore {
|
|||||||
) -> StorageResult<Vec<Option<RawByHashEntry>>> {
|
) -> StorageResult<Vec<Option<RawByHashEntry>>> {
|
||||||
self.get_raw_by_hash_entries_batch(sha256_hexes)
|
self.get_raw_by_hash_entries_batch(sha256_hexes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult<Option<Vec<u8>>> {
|
||||||
|
if let Some(raw_store) = self.external_raw_store_ref() {
|
||||||
|
return raw_store.get_blob_bytes(sha256_hex);
|
||||||
|
}
|
||||||
|
self.get_raw_entry(sha256_hex)
|
||||||
|
.map(|entry| entry.map(|entry| entry.bytes))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_blob_bytes_batch(&self, sha256_hexes: &[String]) -> StorageResult<Vec<Option<Vec<u8>>>> {
|
||||||
|
if let Some(raw_store) = self.external_raw_store_ref() {
|
||||||
|
return raw_store.get_blob_bytes_batch(sha256_hexes);
|
||||||
|
}
|
||||||
|
self.get_raw_entries_batch(sha256_hexes).map(|entries| {
|
||||||
|
entries
|
||||||
|
.into_iter()
|
||||||
|
.map(|entry| entry.map(|entry| entry.bytes))
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RawObjectStore for ExternalRawStoreDb {
|
impl RawObjectStore for ExternalRawStoreDb {
|
||||||
@ -125,7 +164,8 @@ impl RawObjectStore for ExternalRawStoreDb {
|
|||||||
else {
|
else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
let entry = serde_cbor::from_slice::<RawByHashEntry>(&bytes).map_err(|e| StorageError::Codec {
|
let entry =
|
||||||
|
serde_cbor::from_slice::<RawByHashEntry>(&bytes).map_err(|e| StorageError::Codec {
|
||||||
entity: "raw_by_hash",
|
entity: "raw_by_hash",
|
||||||
detail: e.to_string(),
|
detail: e.to_string(),
|
||||||
})?;
|
})?;
|
||||||
@ -140,7 +180,10 @@ impl RawObjectStore for ExternalRawStoreDb {
|
|||||||
if sha256_hexes.is_empty() {
|
if sha256_hexes.is_empty() {
|
||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
}
|
}
|
||||||
let keys: Vec<String> = sha256_hexes.iter().map(|hash| raw_by_hash_key(hash)).collect();
|
let keys: Vec<String> = sha256_hexes
|
||||||
|
.iter()
|
||||||
|
.map(|hash| raw_by_hash_key(hash))
|
||||||
|
.collect();
|
||||||
self.db
|
self.db
|
||||||
.multi_get(keys.iter().map(|key| key.as_bytes()))
|
.multi_get(keys.iter().map(|key| key.as_bytes()))
|
||||||
.into_iter()
|
.into_iter()
|
||||||
@ -148,7 +191,8 @@ impl RawObjectStore for ExternalRawStoreDb {
|
|||||||
let maybe = res.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
let maybe = res.map_err(|e| StorageError::RocksDb(e.to_string()))?;
|
||||||
match maybe {
|
match maybe {
|
||||||
Some(bytes) => {
|
Some(bytes) => {
|
||||||
let entry = serde_cbor::from_slice::<RawByHashEntry>(&bytes).map_err(|e| {
|
let entry =
|
||||||
|
serde_cbor::from_slice::<RawByHashEntry>(&bytes).map_err(|e| {
|
||||||
StorageError::Codec {
|
StorageError::Codec {
|
||||||
entity: "raw_by_hash",
|
entity: "raw_by_hash",
|
||||||
detail: e.to_string(),
|
detail: e.to_string(),
|
||||||
@ -162,18 +206,59 @@ impl RawObjectStore for ExternalRawStoreDb {
|
|||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult<Option<Vec<u8>>> {
|
||||||
|
let key = raw_blob_key(sha256_hex);
|
||||||
|
self.db
|
||||||
|
.get(key.as_bytes())
|
||||||
|
.map_err(|e| StorageError::RocksDb(e.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_blob_bytes_batch(&self, sha256_hexes: &[String]) -> StorageResult<Vec<Option<Vec<u8>>>> {
|
||||||
|
if sha256_hexes.is_empty() {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
let keys: Vec<String> = sha256_hexes.iter().map(|hash| raw_blob_key(hash)).collect();
|
||||||
|
self.db
|
||||||
|
.multi_get(keys.iter().map(|key| key.as_bytes()))
|
||||||
|
.into_iter()
|
||||||
|
.map(|res| res.map_err(|e| StorageError::RocksDb(e.to_string())))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{ExternalRawStoreDb, RawObjectStore};
|
use super::{ExternalRawStoreDb, RawObjectStore};
|
||||||
use crate::storage::{RawByHashEntry, RocksStore, StorageError};
|
use crate::storage::{RawByHashEntry, RocksStore, StorageError, StorageResult};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
fn sha256_hex(bytes: &[u8]) -> String {
|
fn sha256_hex(bytes: &[u8]) -> String {
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
hex::encode(Sha256::digest(bytes))
|
hex::encode(Sha256::digest(bytes))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct MockRawStore {
|
||||||
|
entries: HashMap<String, RawByHashEntry>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RawObjectStore for MockRawStore {
|
||||||
|
fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult<Option<RawByHashEntry>> {
|
||||||
|
Ok(self.entries.get(sha256_hex).cloned())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_raw_entries_batch(
|
||||||
|
&self,
|
||||||
|
sha256_hexes: &[String],
|
||||||
|
) -> StorageResult<Vec<Option<RawByHashEntry>>> {
|
||||||
|
Ok(sha256_hexes
|
||||||
|
.iter()
|
||||||
|
.map(|hash| self.entries.get(hash).cloned())
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn rocks_store_raw_object_store_reads_single_and_batch_entries() {
|
fn rocks_store_raw_object_store_reads_single_and_batch_entries() {
|
||||||
let td = tempfile::tempdir().expect("tempdir");
|
let td = tempfile::tempdir().expect("tempdir");
|
||||||
@ -201,18 +286,27 @@ mod tests {
|
|||||||
.get_raw_entries_batch(&[a_hash.clone(), "00".repeat(32), b_hash.clone()])
|
.get_raw_entries_batch(&[a_hash.clone(), "00".repeat(32), b_hash.clone()])
|
||||||
.expect("get batch");
|
.expect("get batch");
|
||||||
assert_eq!(batch.len(), 3);
|
assert_eq!(batch.len(), 3);
|
||||||
assert_eq!(batch[0].as_ref().map(|entry| entry.bytes.as_slice()), Some(a.as_slice()));
|
assert_eq!(
|
||||||
|
batch[0].as_ref().map(|entry| entry.bytes.as_slice()),
|
||||||
|
Some(a.as_slice())
|
||||||
|
);
|
||||||
assert!(batch[1].is_none());
|
assert!(batch[1].is_none());
|
||||||
assert_eq!(batch[2].as_ref().map(|entry| entry.bytes.as_slice()), Some(b.as_slice()));
|
assert_eq!(
|
||||||
|
batch[2].as_ref().map(|entry| entry.bytes.as_slice()),
|
||||||
|
Some(b.as_slice())
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn external_raw_store_db_roundtrips_entries() {
|
fn external_raw_store_db_roundtrips_entries() {
|
||||||
let td = tempfile::tempdir().expect("tempdir");
|
let td = tempfile::tempdir().expect("tempdir");
|
||||||
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
let raw_store =
|
||||||
|
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||||
|
|
||||||
let mut entry = RawByHashEntry::from_bytes(sha256_hex(b"blob"), b"blob".to_vec());
|
let mut entry = RawByHashEntry::from_bytes(sha256_hex(b"blob"), b"blob".to_vec());
|
||||||
entry.origin_uris.push("rsync://example.test/repo/a.cer".to_string());
|
entry
|
||||||
|
.origin_uris
|
||||||
|
.push("rsync://example.test/repo/a.cer".to_string());
|
||||||
entry.object_type = Some("cer".to_string());
|
entry.object_type = Some("cer".to_string());
|
||||||
raw_store.put_raw_entry(&entry).expect("put raw entry");
|
raw_store.put_raw_entry(&entry).expect("put raw entry");
|
||||||
|
|
||||||
@ -226,7 +320,8 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn external_raw_store_db_batch_writes_and_reads() {
|
fn external_raw_store_db_batch_writes_and_reads() {
|
||||||
let td = tempfile::tempdir().expect("tempdir");
|
let td = tempfile::tempdir().expect("tempdir");
|
||||||
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
let raw_store =
|
||||||
|
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||||
|
|
||||||
let a = RawByHashEntry::from_bytes(sha256_hex(b"a"), b"a".to_vec());
|
let a = RawByHashEntry::from_bytes(sha256_hex(b"a"), b"a".to_vec());
|
||||||
let b = RawByHashEntry::from_bytes(sha256_hex(b"b"), b"b".to_vec());
|
let b = RawByHashEntry::from_bytes(sha256_hex(b"b"), b"b".to_vec());
|
||||||
@ -249,7 +344,9 @@ mod tests {
|
|||||||
.expect("open raw store");
|
.expect("open raw store");
|
||||||
|
|
||||||
let mut entry = RawByHashEntry::from_bytes(sha256_hex(b"blob"), b"blob".to_vec());
|
let mut entry = RawByHashEntry::from_bytes(sha256_hex(b"blob"), b"blob".to_vec());
|
||||||
entry.origin_uris.push("rsync://example.test/repo/blob.roa".to_string());
|
entry
|
||||||
|
.origin_uris
|
||||||
|
.push("rsync://example.test/repo/blob.roa".to_string());
|
||||||
raw_store.put_raw_entry(&entry).expect("put raw entry");
|
raw_store.put_raw_entry(&entry).expect("put raw entry");
|
||||||
|
|
||||||
let single = raw_store
|
let single = raw_store
|
||||||
@ -264,25 +361,81 @@ mod tests {
|
|||||||
assert_eq!(batch, vec![Some(b"blob".to_vec()), None]);
|
assert_eq!(batch, vec![Some(b"blob".to_vec()), None]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn raw_object_store_default_blob_helpers_work_for_custom_store() {
|
||||||
|
let mut store = MockRawStore::default();
|
||||||
|
let a = RawByHashEntry::from_bytes(sha256_hex(b"a"), b"a".to_vec());
|
||||||
|
let b = RawByHashEntry::from_bytes(sha256_hex(b"b"), b"b".to_vec());
|
||||||
|
store.entries.insert(a.sha256_hex.clone(), a.clone());
|
||||||
|
store.entries.insert(b.sha256_hex.clone(), b.clone());
|
||||||
|
|
||||||
|
let single = store
|
||||||
|
.get_blob_bytes(&a.sha256_hex)
|
||||||
|
.expect("single blob bytes")
|
||||||
|
.expect("present");
|
||||||
|
assert_eq!(single, b"a".to_vec());
|
||||||
|
|
||||||
|
let batch = store
|
||||||
|
.get_blob_bytes_batch(&[a.sha256_hex.clone(), "00".repeat(32), b.sha256_hex.clone()])
|
||||||
|
.expect("batch blob bytes");
|
||||||
|
assert_eq!(batch, vec![Some(b"a".to_vec()), None, Some(b"b".to_vec())]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rocks_store_blob_helpers_use_external_raw_store_fast_path() {
|
||||||
|
let td = tempfile::tempdir().expect("tempdir");
|
||||||
|
let store = RocksStore::open_with_external_raw_store(
|
||||||
|
&td.path().join("db"),
|
||||||
|
&td.path().join("raw-store.db"),
|
||||||
|
)
|
||||||
|
.expect("open store with external raw store");
|
||||||
|
|
||||||
|
let entry = RawByHashEntry::from_bytes(sha256_hex(b"blob-fast"), b"blob-fast".to_vec());
|
||||||
|
store.put_raw_by_hash_entry(&entry).expect("put");
|
||||||
|
|
||||||
|
let single = store
|
||||||
|
.get_blob_bytes(&entry.sha256_hex)
|
||||||
|
.expect("single blob bytes")
|
||||||
|
.expect("present");
|
||||||
|
assert_eq!(single, b"blob-fast".to_vec());
|
||||||
|
|
||||||
|
let batch = store
|
||||||
|
.get_blob_bytes_batch(&[entry.sha256_hex.clone(), "00".repeat(32)])
|
||||||
|
.expect("batch blob bytes");
|
||||||
|
assert_eq!(batch, vec![Some(b"blob-fast".to_vec()), None]);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn external_raw_store_db_delete_removes_entry() {
|
fn external_raw_store_db_delete_removes_entry() {
|
||||||
let td = tempfile::tempdir().expect("tempdir");
|
let td = tempfile::tempdir().expect("tempdir");
|
||||||
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
let raw_store =
|
||||||
|
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||||
|
|
||||||
let entry = RawByHashEntry::from_bytes(sha256_hex(b"gone"), b"gone".to_vec());
|
let entry = RawByHashEntry::from_bytes(sha256_hex(b"gone"), b"gone".to_vec());
|
||||||
raw_store.put_raw_entry(&entry).expect("put");
|
raw_store.put_raw_entry(&entry).expect("put");
|
||||||
assert!(raw_store.get_raw_entry(&entry.sha256_hex).unwrap().is_some());
|
assert!(
|
||||||
|
raw_store
|
||||||
|
.get_raw_entry(&entry.sha256_hex)
|
||||||
|
.unwrap()
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
|
||||||
raw_store
|
raw_store
|
||||||
.delete_raw_entry(&entry.sha256_hex)
|
.delete_raw_entry(&entry.sha256_hex)
|
||||||
.expect("delete entry");
|
.expect("delete entry");
|
||||||
assert!(raw_store.get_raw_entry(&entry.sha256_hex).unwrap().is_none());
|
assert!(
|
||||||
|
raw_store
|
||||||
|
.get_raw_entry(&entry.sha256_hex)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn external_raw_store_db_rejects_invalid_entry_on_put() {
|
fn external_raw_store_db_rejects_invalid_entry_on_put() {
|
||||||
let td = tempfile::tempdir().expect("tempdir");
|
let td = tempfile::tempdir().expect("tempdir");
|
||||||
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
let raw_store =
|
||||||
|
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||||
|
|
||||||
let bad = RawByHashEntry {
|
let bad = RawByHashEntry {
|
||||||
sha256_hex: "11".repeat(32),
|
sha256_hex: "11".repeat(32),
|
||||||
@ -291,14 +444,17 @@ mod tests {
|
|||||||
object_type: None,
|
object_type: None,
|
||||||
encoding: None,
|
encoding: None,
|
||||||
};
|
};
|
||||||
let err = raw_store.put_raw_entry(&bad).expect_err("invalid hash should fail");
|
let err = raw_store
|
||||||
|
.put_raw_entry(&bad)
|
||||||
|
.expect_err("invalid hash should fail");
|
||||||
assert!(matches!(err, StorageError::InvalidData { .. }));
|
assert!(matches!(err, StorageError::InvalidData { .. }));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn external_raw_store_db_reports_codec_error_for_corrupt_value() {
|
fn external_raw_store_db_reports_codec_error_for_corrupt_value() {
|
||||||
let td = tempfile::tempdir().expect("tempdir");
|
let td = tempfile::tempdir().expect("tempdir");
|
||||||
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
let raw_store =
|
||||||
|
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||||
raw_store
|
raw_store
|
||||||
.db
|
.db
|
||||||
.put(b"rawbyhash:deadbeef", b"not-cbor")
|
.put(b"rawbyhash:deadbeef", b"not-cbor")
|
||||||
@ -307,17 +463,26 @@ mod tests {
|
|||||||
let err = raw_store
|
let err = raw_store
|
||||||
.get_raw_entry("deadbeef")
|
.get_raw_entry("deadbeef")
|
||||||
.expect_err("corrupt value should fail");
|
.expect_err("corrupt value should fail");
|
||||||
assert!(matches!(err, StorageError::Codec { entity: "raw_by_hash", .. }));
|
assert!(matches!(
|
||||||
|
err,
|
||||||
|
StorageError::Codec {
|
||||||
|
entity: "raw_by_hash",
|
||||||
|
..
|
||||||
|
}
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn external_raw_store_db_batch_returns_empty_for_empty_request() {
|
fn external_raw_store_db_batch_returns_empty_for_empty_request() {
|
||||||
let td = tempfile::tempdir().expect("tempdir");
|
let td = tempfile::tempdir().expect("tempdir");
|
||||||
let raw_store = ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
let raw_store =
|
||||||
|
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store");
|
||||||
let entries = raw_store
|
let entries = raw_store
|
||||||
.get_raw_entries_batch(&[])
|
.get_raw_entries_batch(&[])
|
||||||
.expect("empty batch succeeds");
|
.expect("empty batch succeeds");
|
||||||
assert!(entries.is_empty());
|
assert!(entries.is_empty());
|
||||||
raw_store.put_raw_entries_batch(&[]).expect("empty put succeeds");
|
raw_store
|
||||||
|
.put_raw_entries_batch(&[])
|
||||||
|
.expect("empty put succeeds");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,6 +2,7 @@ use std::collections::{BTreeMap, BTreeSet};
|
|||||||
|
|
||||||
use sha2::Digest;
|
use sha2::Digest;
|
||||||
|
|
||||||
|
use crate::blob_store::RawObjectStore;
|
||||||
use crate::ccr::encode::{
|
use crate::ccr::encode::{
|
||||||
encode_aspa_payload_state_payload_der, encode_manifest_state_payload_der,
|
encode_aspa_payload_state_payload_der, encode_manifest_state_payload_der,
|
||||||
encode_roa_payload_state_payload_der, encode_router_key_state_payload_der,
|
encode_roa_payload_state_payload_der, encode_router_key_state_payload_der,
|
||||||
@ -12,7 +13,6 @@ use crate::ccr::model::{
|
|||||||
AspaPayloadSet, AspaPayloadState, ManifestInstance, ManifestState, RoaPayloadSet,
|
AspaPayloadSet, AspaPayloadState, ManifestInstance, ManifestState, RoaPayloadSet,
|
||||||
RoaPayloadState, RouterKey, RouterKeySet, RouterKeyState, TrustAnchorState,
|
RoaPayloadState, RouterKey, RouterKeySet, RouterKeyState, TrustAnchorState,
|
||||||
};
|
};
|
||||||
use crate::blob_store::RawObjectStore;
|
|
||||||
use crate::data_model::manifest::ManifestObject;
|
use crate::data_model::manifest::ManifestObject;
|
||||||
use crate::data_model::rc::{AccessDescription, SubjectInfoAccess};
|
use crate::data_model::rc::{AccessDescription, SubjectInfoAccess};
|
||||||
use crate::data_model::roa::RoaAfi;
|
use crate::data_model::roa::RoaAfi;
|
||||||
@ -201,11 +201,10 @@ pub fn build_manifest_state_from_vcirs(
|
|||||||
sha256_hex: manifest_artifact.sha256.clone(),
|
sha256_hex: manifest_artifact.sha256.clone(),
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
let manifest = ManifestObject::decode_der(&raw_bytes).map_err(|e| {
|
let manifest =
|
||||||
CcrBuildError::ManifestDecode {
|
ManifestObject::decode_der(&raw_bytes).map_err(|e| CcrBuildError::ManifestDecode {
|
||||||
manifest_rsync_uri: vcir.current_manifest_rsync_uri.clone(),
|
manifest_rsync_uri: vcir.current_manifest_rsync_uri.clone(),
|
||||||
detail: e.to_string(),
|
detail: e.to_string(),
|
||||||
}
|
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
let ee = &manifest.signed_object.signed_data.certificates[0].resource_cert;
|
let ee = &manifest.signed_object.signed_data.certificates[0].resource_cert;
|
||||||
|
|||||||
@ -12,13 +12,13 @@ use crate::cir::static_pool::{
|
|||||||
CirStaticPoolError, CirStaticPoolExportSummary, export_hashes_from_store,
|
CirStaticPoolError, CirStaticPoolExportSummary, export_hashes_from_store,
|
||||||
write_bytes_to_static_pool,
|
write_bytes_to_static_pool,
|
||||||
};
|
};
|
||||||
|
use crate::current_repo_index::CurrentRepoObject;
|
||||||
use crate::data_model::ta::TrustAnchor;
|
use crate::data_model::ta::TrustAnchor;
|
||||||
use crate::storage::{RepositoryViewState, RocksStore};
|
use crate::storage::{RepositoryViewState, RocksStore};
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
pub struct CirExportTiming {
|
pub struct CirExportTiming {
|
||||||
pub build_cir_ms: u64,
|
pub build_cir_ms: u64,
|
||||||
pub static_pool_ms: u64,
|
|
||||||
pub write_cir_ms: u64,
|
pub write_cir_ms: u64,
|
||||||
pub total_ms: u64,
|
pub total_ms: u64,
|
||||||
}
|
}
|
||||||
@ -47,12 +47,6 @@ pub enum CirExportError {
|
|||||||
Write(String, String),
|
Write(String, String),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
||||||
pub enum CirObjectPoolExportSummary {
|
|
||||||
Static(CirStaticPoolExportSummary),
|
|
||||||
RawStore(CirRawStoreExportSummary),
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
pub struct CirRawStoreExportSummary {
|
pub struct CirRawStoreExportSummary {
|
||||||
pub unique_hashes: usize,
|
pub unique_hashes: usize,
|
||||||
@ -64,26 +58,36 @@ pub struct CirRawStoreExportSummary {
|
|||||||
pub struct CirExportSummary {
|
pub struct CirExportSummary {
|
||||||
pub object_count: usize,
|
pub object_count: usize,
|
||||||
pub tal_count: usize,
|
pub tal_count: usize,
|
||||||
pub object_pool: CirObjectPoolExportSummary,
|
|
||||||
pub timing: CirExportTiming,
|
pub timing: CirExportTiming,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn build_cir_from_run(
|
#[derive(Clone, Copy, Debug)]
|
||||||
store: &RocksStore,
|
pub struct CirTalBinding<'a> {
|
||||||
trust_anchor: &TrustAnchor,
|
pub trust_anchor: &'a TrustAnchor,
|
||||||
tal_uri: &str,
|
pub tal_uri: &'a str,
|
||||||
validation_time: time::OffsetDateTime,
|
}
|
||||||
publication_points: &[PublicationPointAudit],
|
|
||||||
) -> Result<CanonicalInputRepresentation, CirExportError> {
|
|
||||||
if !(tal_uri.starts_with("https://") || tal_uri.starts_with("http://")) {
|
|
||||||
return Err(CirExportError::InvalidTalUri(tal_uri.to_string()));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
fn collect_cir_objects_from_current_repo(
|
||||||
|
current_repo_objects: &[CurrentRepoObject],
|
||||||
|
) -> BTreeMap<String, String> {
|
||||||
|
let mut objects = BTreeMap::new();
|
||||||
|
for entry in current_repo_objects {
|
||||||
|
objects.insert(
|
||||||
|
entry.rsync_uri.clone(),
|
||||||
|
entry.current_hash_hex.to_ascii_lowercase(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
objects
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_cir_objects_from_repository_view(
|
||||||
|
store: &RocksStore,
|
||||||
|
) -> Result<BTreeMap<String, String>, CirExportError> {
|
||||||
let entries = store
|
let entries = store
|
||||||
.list_repository_view_entries_with_prefix("rsync://")
|
.list_repository_view_entries_with_prefix("rsync://")
|
||||||
.map_err(|e| CirExportError::ListRepositoryView(e.to_string()))?;
|
.map_err(|e| CirExportError::ListRepositoryView(e.to_string()))?;
|
||||||
|
|
||||||
let mut objects: BTreeMap<String, String> = BTreeMap::new();
|
let mut objects = BTreeMap::new();
|
||||||
for entry in entries {
|
for entry in entries {
|
||||||
if matches!(
|
if matches!(
|
||||||
entry.state,
|
entry.state,
|
||||||
@ -93,6 +97,46 @@ pub fn build_cir_from_run(
|
|||||||
objects.insert(entry.rsync_uri, hash.to_ascii_lowercase());
|
objects.insert(entry.rsync_uri, hash.to_ascii_lowercase());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Ok(objects)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn build_cir_from_run(
|
||||||
|
store: &RocksStore,
|
||||||
|
trust_anchor: &TrustAnchor,
|
||||||
|
tal_uri: &str,
|
||||||
|
validation_time: time::OffsetDateTime,
|
||||||
|
publication_points: &[PublicationPointAudit],
|
||||||
|
) -> Result<CanonicalInputRepresentation, CirExportError> {
|
||||||
|
build_cir_from_run_multi(
|
||||||
|
store,
|
||||||
|
&[CirTalBinding {
|
||||||
|
trust_anchor,
|
||||||
|
tal_uri,
|
||||||
|
}],
|
||||||
|
validation_time,
|
||||||
|
publication_points,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn build_cir_from_run_multi(
|
||||||
|
store: &RocksStore,
|
||||||
|
tal_bindings: &[CirTalBinding<'_>],
|
||||||
|
validation_time: time::OffsetDateTime,
|
||||||
|
publication_points: &[PublicationPointAudit],
|
||||||
|
current_repo_objects: Option<&[CurrentRepoObject]>,
|
||||||
|
) -> Result<CanonicalInputRepresentation, CirExportError> {
|
||||||
|
for binding in tal_bindings {
|
||||||
|
if !(binding.tal_uri.starts_with("https://") || binding.tal_uri.starts_with("http://")) {
|
||||||
|
return Err(CirExportError::InvalidTalUri(binding.tal_uri.to_string()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut objects = if let Some(current_repo_objects) = current_repo_objects {
|
||||||
|
collect_cir_objects_from_current_repo(current_repo_objects)
|
||||||
|
} else {
|
||||||
|
collect_cir_objects_from_repository_view(store)?
|
||||||
|
};
|
||||||
|
|
||||||
// CIR must describe the actual input world used by validation. When a
|
// CIR must describe the actual input world used by validation. When a
|
||||||
// publication point falls back to the latest validated current instance,
|
// publication point falls back to the latest validated current instance,
|
||||||
@ -114,9 +158,11 @@ pub fn build_cir_from_run(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let ta_hash = ta_sha256_hex(&trust_anchor.ta_certificate.raw_der);
|
let mut tals = Vec::with_capacity(tal_bindings.len());
|
||||||
|
for binding in tal_bindings {
|
||||||
|
let ta_hash = ta_sha256_hex(&binding.trust_anchor.ta_certificate.raw_der);
|
||||||
let mut saw_rsync_uri = false;
|
let mut saw_rsync_uri = false;
|
||||||
for uri in &trust_anchor.tal.ta_uris {
|
for uri in &binding.trust_anchor.tal.ta_uris {
|
||||||
if uri.scheme() == "rsync" {
|
if uri.scheme() == "rsync" {
|
||||||
saw_rsync_uri = true;
|
saw_rsync_uri = true;
|
||||||
objects.insert(uri.as_str().to_string(), ta_hash.clone());
|
objects.insert(uri.as_str().to_string(), ta_hash.clone());
|
||||||
@ -125,6 +171,11 @@ pub fn build_cir_from_run(
|
|||||||
if !saw_rsync_uri {
|
if !saw_rsync_uri {
|
||||||
return Err(CirExportError::MissingTaRsyncUri);
|
return Err(CirExportError::MissingTaRsyncUri);
|
||||||
}
|
}
|
||||||
|
tals.push(CirTal {
|
||||||
|
tal_uri: binding.tal_uri.to_string(),
|
||||||
|
tal_bytes: binding.trust_anchor.tal.raw.clone(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
let cir = CanonicalInputRepresentation {
|
let cir = CanonicalInputRepresentation {
|
||||||
version: CIR_VERSION_V1,
|
version: CIR_VERSION_V1,
|
||||||
@ -137,10 +188,7 @@ pub fn build_cir_from_run(
|
|||||||
sha256: hex::decode(sha256_hex).expect("validated hex"),
|
sha256: hex::decode(sha256_hex).expect("validated hex"),
|
||||||
})
|
})
|
||||||
.collect(),
|
.collect(),
|
||||||
tals: vec![CirTal {
|
tals,
|
||||||
tal_uri: tal_uri.to_string(),
|
|
||||||
tal_bytes: trust_anchor.tal.raw.clone(),
|
|
||||||
}],
|
|
||||||
};
|
};
|
||||||
cir.validate().map_err(CirExportError::Validate)?;
|
cir.validate().map_err(CirExportError::Validate)?;
|
||||||
Ok(cir)
|
Ok(cir)
|
||||||
@ -164,31 +212,37 @@ pub fn export_cir_static_pool(
|
|||||||
static_root: &Path,
|
static_root: &Path,
|
||||||
capture_date_utc: time::Date,
|
capture_date_utc: time::Date,
|
||||||
cir: &CanonicalInputRepresentation,
|
cir: &CanonicalInputRepresentation,
|
||||||
trust_anchor: &TrustAnchor,
|
trust_anchors: &[&TrustAnchor],
|
||||||
) -> Result<CirStaticPoolExportSummary, CirExportError> {
|
) -> Result<CirStaticPoolExportSummary, CirExportError> {
|
||||||
let ta_hash = ta_sha256_hex(&trust_anchor.ta_certificate.raw_der);
|
let ta_hashes = trust_anchors
|
||||||
|
.iter()
|
||||||
|
.map(|ta| ta_sha256_hex(&ta.ta_certificate.raw_der))
|
||||||
|
.collect::<BTreeSet<_>>();
|
||||||
let hashes = cir
|
let hashes = cir
|
||||||
.objects
|
.objects
|
||||||
.iter()
|
.iter()
|
||||||
.map(|item| hex::encode(&item.sha256))
|
.map(|item| hex::encode(&item.sha256))
|
||||||
.filter(|hash| hash != &ta_hash)
|
.filter(|hash| !ta_hashes.contains(hash))
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
let mut summary = export_hashes_from_store(store, static_root, capture_date_utc, &hashes)?;
|
let mut summary = export_hashes_from_store(store, static_root, capture_date_utc, &hashes)?;
|
||||||
|
|
||||||
|
let mut unique = hashes.iter().cloned().collect::<BTreeSet<_>>();
|
||||||
|
for trust_anchor in trust_anchors {
|
||||||
|
let ta_hash = ta_sha256_hex(&trust_anchor.ta_certificate.raw_der);
|
||||||
let ta_result = write_bytes_to_static_pool(
|
let ta_result = write_bytes_to_static_pool(
|
||||||
static_root,
|
static_root,
|
||||||
capture_date_utc,
|
capture_date_utc,
|
||||||
&ta_hash,
|
&ta_hash,
|
||||||
&trust_anchor.ta_certificate.raw_der,
|
&trust_anchor.ta_certificate.raw_der,
|
||||||
)?;
|
)?;
|
||||||
let mut unique = hashes.iter().cloned().collect::<BTreeSet<_>>();
|
unique.insert(ta_hash);
|
||||||
unique.insert(ta_hash.clone());
|
|
||||||
summary.unique_hashes = unique.len();
|
|
||||||
if ta_result.written {
|
if ta_result.written {
|
||||||
summary.written_files += 1;
|
summary.written_files += 1;
|
||||||
} else {
|
} else {
|
||||||
summary.reused_files += 1;
|
summary.reused_files += 1;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
summary.unique_hashes = unique.len();
|
||||||
Ok(summary)
|
Ok(summary)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -196,9 +250,12 @@ pub fn export_cir_raw_store(
|
|||||||
store: &RocksStore,
|
store: &RocksStore,
|
||||||
raw_store_path: &Path,
|
raw_store_path: &Path,
|
||||||
cir: &CanonicalInputRepresentation,
|
cir: &CanonicalInputRepresentation,
|
||||||
trust_anchor: &TrustAnchor,
|
trust_anchors: &[&TrustAnchor],
|
||||||
) -> Result<CirRawStoreExportSummary, CirExportError> {
|
) -> Result<CirRawStoreExportSummary, CirExportError> {
|
||||||
let ta_hash = ta_sha256_hex(&trust_anchor.ta_certificate.raw_der);
|
let ta_by_hash = trust_anchors
|
||||||
|
.iter()
|
||||||
|
.map(|ta| (ta_sha256_hex(&ta.ta_certificate.raw_der), *ta))
|
||||||
|
.collect::<BTreeMap<_, _>>();
|
||||||
let unique: BTreeSet<String> = cir
|
let unique: BTreeSet<String> = cir
|
||||||
.objects
|
.objects
|
||||||
.iter()
|
.iter()
|
||||||
@ -210,24 +267,28 @@ pub fn export_cir_raw_store(
|
|||||||
for sha256_hex in &unique {
|
for sha256_hex in &unique {
|
||||||
if store
|
if store
|
||||||
.get_raw_entry(sha256_hex)
|
.get_raw_entry(sha256_hex)
|
||||||
.map_err(|e| CirExportError::Write(raw_store_path.display().to_string(), e.to_string()))?
|
.map_err(|e| {
|
||||||
|
CirExportError::Write(raw_store_path.display().to_string(), e.to_string())
|
||||||
|
})?
|
||||||
.is_some()
|
.is_some()
|
||||||
{
|
{
|
||||||
reused_entries += 1;
|
reused_entries += 1;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if sha256_hex == &ta_hash {
|
if let Some(trust_anchor) = ta_by_hash.get(sha256_hex) {
|
||||||
let mut entry =
|
let mut entry = crate::storage::RawByHashEntry::from_bytes(
|
||||||
crate::storage::RawByHashEntry::from_bytes(ta_hash.clone(), trust_anchor.ta_certificate.raw_der.clone());
|
sha256_hex.clone(),
|
||||||
|
trust_anchor.ta_certificate.raw_der.clone(),
|
||||||
|
);
|
||||||
entry.object_type = Some("cer".to_string());
|
entry.object_type = Some("cer".to_string());
|
||||||
for object in &cir.objects {
|
for object in &cir.objects {
|
||||||
if hex::encode(&object.sha256) == ta_hash {
|
if hex::encode(&object.sha256) == *sha256_hex {
|
||||||
entry.origin_uris.push(object.rsync_uri.clone());
|
entry.origin_uris.push(object.rsync_uri.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
store
|
store.put_raw_by_hash_entry(&entry).map_err(|e| {
|
||||||
.put_raw_by_hash_entry(&entry)
|
CirExportError::Write(raw_store_path.display().to_string(), e.to_string())
|
||||||
.map_err(|e| CirExportError::Write(raw_store_path.display().to_string(), e.to_string()))?;
|
})?;
|
||||||
written_entries += 1;
|
written_entries += 1;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@ -251,50 +312,44 @@ pub fn export_cir_from_run(
|
|||||||
validation_time: time::OffsetDateTime,
|
validation_time: time::OffsetDateTime,
|
||||||
publication_points: &[PublicationPointAudit],
|
publication_points: &[PublicationPointAudit],
|
||||||
cir_out: &Path,
|
cir_out: &Path,
|
||||||
static_root: Option<&Path>,
|
|
||||||
raw_store_path: Option<&Path>,
|
|
||||||
capture_date_utc: time::Date,
|
capture_date_utc: time::Date,
|
||||||
) -> Result<CirExportSummary, CirExportError> {
|
) -> Result<CirExportSummary, CirExportError> {
|
||||||
let backend_count = static_root.is_some() as u8 + raw_store_path.is_some() as u8;
|
export_cir_from_run_multi(
|
||||||
match backend_count {
|
store,
|
||||||
1 => {}
|
&[CirTalBinding {
|
||||||
_ => {
|
trust_anchor,
|
||||||
return Err(CirExportError::Validate(
|
tal_uri,
|
||||||
"must specify exactly one CIR object pool backend".to_string(),
|
}],
|
||||||
));
|
validation_time,
|
||||||
}
|
publication_points,
|
||||||
}
|
cir_out,
|
||||||
|
capture_date_utc,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn export_cir_from_run_multi(
|
||||||
|
store: &RocksStore,
|
||||||
|
tal_bindings: &[CirTalBinding<'_>],
|
||||||
|
validation_time: time::OffsetDateTime,
|
||||||
|
publication_points: &[PublicationPointAudit],
|
||||||
|
cir_out: &Path,
|
||||||
|
capture_date_utc: time::Date,
|
||||||
|
current_repo_objects: Option<&[CurrentRepoObject]>,
|
||||||
|
) -> Result<CirExportSummary, CirExportError> {
|
||||||
|
let _ = capture_date_utc;
|
||||||
let total_started = std::time::Instant::now();
|
let total_started = std::time::Instant::now();
|
||||||
|
|
||||||
let started = std::time::Instant::now();
|
let started = std::time::Instant::now();
|
||||||
let cir = build_cir_from_run(
|
let cir = build_cir_from_run_multi(
|
||||||
store,
|
store,
|
||||||
trust_anchor,
|
tal_bindings,
|
||||||
tal_uri,
|
|
||||||
validation_time,
|
validation_time,
|
||||||
publication_points,
|
publication_points,
|
||||||
|
current_repo_objects,
|
||||||
)?;
|
)?;
|
||||||
let build_cir_ms = started.elapsed().as_millis() as u64;
|
let build_cir_ms = started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
let started = std::time::Instant::now();
|
|
||||||
let object_pool = if let Some(static_root) = static_root {
|
|
||||||
CirObjectPoolExportSummary::Static(export_cir_static_pool(
|
|
||||||
store,
|
|
||||||
static_root,
|
|
||||||
capture_date_utc,
|
|
||||||
&cir,
|
|
||||||
trust_anchor,
|
|
||||||
)?)
|
|
||||||
} else {
|
|
||||||
CirObjectPoolExportSummary::RawStore(export_cir_raw_store(
|
|
||||||
store,
|
|
||||||
raw_store_path.expect("validated"),
|
|
||||||
&cir,
|
|
||||||
trust_anchor,
|
|
||||||
)?)
|
|
||||||
};
|
|
||||||
let static_pool_ms = started.elapsed().as_millis() as u64;
|
|
||||||
|
|
||||||
let started = std::time::Instant::now();
|
let started = std::time::Instant::now();
|
||||||
write_cir_file(cir_out, &cir)?;
|
write_cir_file(cir_out, &cir)?;
|
||||||
let write_cir_ms = started.elapsed().as_millis() as u64;
|
let write_cir_ms = started.elapsed().as_millis() as u64;
|
||||||
@ -302,10 +357,8 @@ pub fn export_cir_from_run(
|
|||||||
Ok(CirExportSummary {
|
Ok(CirExportSummary {
|
||||||
object_count: cir.objects.len(),
|
object_count: cir.objects.len(),
|
||||||
tal_count: cir.tals.len(),
|
tal_count: cir.tals.len(),
|
||||||
object_pool,
|
|
||||||
timing: CirExportTiming {
|
timing: CirExportTiming {
|
||||||
build_cir_ms,
|
build_cir_ms,
|
||||||
static_pool_ms,
|
|
||||||
write_cir_ms,
|
write_cir_ms,
|
||||||
total_ms: total_started.elapsed().as_millis() as u64,
|
total_ms: total_started.elapsed().as_millis() as u64,
|
||||||
},
|
},
|
||||||
@ -320,8 +373,8 @@ fn ta_sha256_hex(bytes: &[u8]) -> String {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use crate::current_repo_index::CurrentRepoObject;
|
||||||
use crate::cir::decode::decode_cir;
|
use crate::cir::decode::decode_cir;
|
||||||
use crate::cir::static_pool_path;
|
|
||||||
use crate::data_model::ta::TrustAnchor;
|
use crate::data_model::ta::TrustAnchor;
|
||||||
use crate::data_model::tal::Tal;
|
use crate::data_model::tal::Tal;
|
||||||
use crate::storage::{RawByHashEntry, RepositoryViewEntry, RepositoryViewState, RocksStore};
|
use crate::storage::{RawByHashEntry, RepositoryViewEntry, RepositoryViewState, RocksStore};
|
||||||
@ -347,6 +400,20 @@ mod tests {
|
|||||||
TrustAnchor::bind_der(tal, &ta_der, None).unwrap()
|
TrustAnchor::bind_der(tal, &ta_der, None).unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn sample_arin_trust_anchor() -> TrustAnchor {
|
||||||
|
let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
|
||||||
|
let tal_bytes = std::fs::read(base.join("tests/fixtures/tal/arin.tal")).unwrap();
|
||||||
|
let ta_der = std::fs::read(base.join("tests/fixtures/ta/arin-ta.cer")).unwrap();
|
||||||
|
let tal = Tal::decode_bytes(&tal_bytes).unwrap();
|
||||||
|
TrustAnchor::bind_der(tal, &ta_der, None).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sample_trust_anchor_without_rsync_uri() -> TrustAnchor {
|
||||||
|
let mut ta = sample_trust_anchor();
|
||||||
|
ta.tal.ta_uris.retain(|uri| uri.scheme() != "rsync");
|
||||||
|
ta
|
||||||
|
}
|
||||||
|
|
||||||
fn sha256_hex(bytes: &[u8]) -> String {
|
fn sha256_hex(bytes: &[u8]) -> String {
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
hex::encode(Sha256::digest(bytes))
|
hex::encode(Sha256::digest(bytes))
|
||||||
@ -401,7 +468,7 @@ mod tests {
|
|||||||
let td = tempfile::tempdir().unwrap();
|
let td = tempfile::tempdir().unwrap();
|
||||||
let store_dir = td.path().join("db");
|
let store_dir = td.path().join("db");
|
||||||
let out_dir = td.path().join("out");
|
let out_dir = td.path().join("out");
|
||||||
let static_root = td.path().join("static");
|
let _static_root = td.path().join("static");
|
||||||
let store = RocksStore::open(&store_dir).unwrap();
|
let store = RocksStore::open(&store_dir).unwrap();
|
||||||
|
|
||||||
let bytes = b"object-b".to_vec();
|
let bytes = b"object-b".to_vec();
|
||||||
@ -429,24 +496,16 @@ mod tests {
|
|||||||
sample_time(),
|
sample_time(),
|
||||||
&[],
|
&[],
|
||||||
&cir_path,
|
&cir_path,
|
||||||
Some(&static_root),
|
|
||||||
None,
|
|
||||||
sample_date(),
|
sample_date(),
|
||||||
)
|
)
|
||||||
.expect("export cir");
|
.expect("export cir");
|
||||||
assert_eq!(summary.tal_count, 1);
|
assert_eq!(summary.tal_count, 1);
|
||||||
assert!(summary.object_count >= 2);
|
assert!(summary.object_count >= 2);
|
||||||
match summary.object_pool {
|
assert!(summary.timing.total_ms >= summary.timing.build_cir_ms);
|
||||||
CirObjectPoolExportSummary::Static(_) => {}
|
|
||||||
other => panic!("unexpected backend: {other:?}"),
|
|
||||||
}
|
|
||||||
|
|
||||||
let der = std::fs::read(&cir_path).unwrap();
|
let der = std::fs::read(&cir_path).unwrap();
|
||||||
let cir = decode_cir(&der).unwrap();
|
let cir = decode_cir(&der).unwrap();
|
||||||
assert_eq!(cir.tals[0].tal_uri, "https://example.test/root.tal");
|
assert_eq!(cir.tals[0].tal_uri, "https://example.test/root.tal");
|
||||||
|
|
||||||
let object_path = static_pool_path(&static_root, sample_date(), &hash).unwrap();
|
|
||||||
assert_eq!(std::fs::read(object_path).unwrap(), bytes);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -482,16 +541,12 @@ mod tests {
|
|||||||
sample_time(),
|
sample_time(),
|
||||||
&[],
|
&[],
|
||||||
&cir_path,
|
&cir_path,
|
||||||
None,
|
|
||||||
Some(&raw_store),
|
|
||||||
sample_date(),
|
sample_date(),
|
||||||
)
|
)
|
||||||
.expect("export cir to raw store");
|
.expect("export cir");
|
||||||
match summary.object_pool {
|
assert!(summary.object_count >= 2);
|
||||||
CirObjectPoolExportSummary::RawStore(ref s) => assert!(s.unique_hashes >= 2),
|
|
||||||
other => panic!("unexpected backend: {other:?}"),
|
|
||||||
}
|
|
||||||
assert!(raw_store.exists());
|
assert!(raw_store.exists());
|
||||||
|
assert!(cir_path.exists());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -541,4 +596,201 @@ mod tests {
|
|||||||
.any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.roa")
|
.any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.roa")
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_cir_from_run_multi_uses_current_repo_objects_without_repository_view() {
|
||||||
|
let td = tempfile::tempdir().unwrap();
|
||||||
|
let store = RocksStore::open(td.path()).unwrap();
|
||||||
|
let ta1 = sample_trust_anchor();
|
||||||
|
let ta2 = sample_arin_trust_anchor();
|
||||||
|
let current_repo_objects = vec![
|
||||||
|
CurrentRepoObject {
|
||||||
|
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
|
||||||
|
current_hash_hex: "11".repeat(32),
|
||||||
|
repository_source: "https://rrdp.example.test/notification.xml".to_string(),
|
||||||
|
object_type: Some("roa".to_string()),
|
||||||
|
},
|
||||||
|
CurrentRepoObject {
|
||||||
|
rsync_uri: "rsync://example.test/repo/b.cer".to_string(),
|
||||||
|
current_hash_hex: "22".repeat(32),
|
||||||
|
repository_source: "https://rrdp.example.test/notification.xml".to_string(),
|
||||||
|
object_type: Some("cer".to_string()),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
let cir = build_cir_from_run_multi(
|
||||||
|
&store,
|
||||||
|
&[
|
||||||
|
CirTalBinding {
|
||||||
|
trust_anchor: &ta1,
|
||||||
|
tal_uri: "https://example.test/apnic.tal",
|
||||||
|
},
|
||||||
|
CirTalBinding {
|
||||||
|
trust_anchor: &ta2,
|
||||||
|
tal_uri: "https://example.test/arin.tal",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
sample_time(),
|
||||||
|
&[],
|
||||||
|
Some(¤t_repo_objects),
|
||||||
|
)
|
||||||
|
.expect("build cir from current repo objects");
|
||||||
|
|
||||||
|
assert_eq!(cir.tals.len(), 2);
|
||||||
|
assert!(
|
||||||
|
cir.objects
|
||||||
|
.iter()
|
||||||
|
.any(|item| item.rsync_uri == "rsync://example.test/repo/a.roa")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
cir.objects
|
||||||
|
.iter()
|
||||||
|
.any(|item| item.rsync_uri == "rsync://example.test/repo/b.cer")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
cir.objects.iter().any(|item| {
|
||||||
|
item.rsync_uri.contains("apnic-rpki-root-iana-origin.cer")
|
||||||
|
|| item.rsync_uri.contains("arin-rpki-ta.cer")
|
||||||
|
}),
|
||||||
|
"trust anchor rsync objects must be included",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_cir_from_run_multi_rejects_invalid_tal_uri_and_missing_rsync_ta_uri() {
|
||||||
|
let td = tempfile::tempdir().unwrap();
|
||||||
|
let store = RocksStore::open(td.path()).unwrap();
|
||||||
|
|
||||||
|
let err = build_cir_from_run_multi(
|
||||||
|
&store,
|
||||||
|
&[CirTalBinding {
|
||||||
|
trust_anchor: &sample_trust_anchor(),
|
||||||
|
tal_uri: "file:///not-supported.tal",
|
||||||
|
}],
|
||||||
|
sample_time(),
|
||||||
|
&[],
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.expect_err("non-http tal uri must fail");
|
||||||
|
assert!(matches!(err, CirExportError::InvalidTalUri(_)), "{err}");
|
||||||
|
|
||||||
|
let err = build_cir_from_run_multi(
|
||||||
|
&store,
|
||||||
|
&[CirTalBinding {
|
||||||
|
trust_anchor: &sample_trust_anchor_without_rsync_uri(),
|
||||||
|
tal_uri: "https://example.test/root.tal",
|
||||||
|
}],
|
||||||
|
sample_time(),
|
||||||
|
&[],
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.expect_err("missing rsync ta uri must fail");
|
||||||
|
assert!(matches!(err, CirExportError::MissingTaRsyncUri), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_cir_static_pool_writes_objects_and_multiple_tas() {
|
||||||
|
let td = tempfile::tempdir().unwrap();
|
||||||
|
let store = RocksStore::open(&td.path().join("db")).unwrap();
|
||||||
|
let static_root = td.path().join("static");
|
||||||
|
let ta1 = sample_trust_anchor();
|
||||||
|
let ta2 = sample_arin_trust_anchor();
|
||||||
|
|
||||||
|
let object_bytes = b"object-z".to_vec();
|
||||||
|
let hash = sha256_hex(&object_bytes);
|
||||||
|
let mut raw = RawByHashEntry::from_bytes(hash.clone(), object_bytes.clone());
|
||||||
|
raw.origin_uris
|
||||||
|
.push("rsync://example.test/repo/z.roa".into());
|
||||||
|
store.put_raw_by_hash_entry(&raw).unwrap();
|
||||||
|
store
|
||||||
|
.put_repository_view_entry(&RepositoryViewEntry {
|
||||||
|
rsync_uri: "rsync://example.test/repo/z.roa".to_string(),
|
||||||
|
current_hash: Some(hash.clone()),
|
||||||
|
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
|
||||||
|
object_type: Some("roa".to_string()),
|
||||||
|
state: RepositoryViewState::Present,
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let cir = build_cir_from_run_multi(
|
||||||
|
&store,
|
||||||
|
&[
|
||||||
|
CirTalBinding {
|
||||||
|
trust_anchor: &ta1,
|
||||||
|
tal_uri: "https://example.test/apnic.tal",
|
||||||
|
},
|
||||||
|
CirTalBinding {
|
||||||
|
trust_anchor: &ta2,
|
||||||
|
tal_uri: "https://example.test/arin.tal",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
sample_time(),
|
||||||
|
&[],
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.expect("build cir");
|
||||||
|
|
||||||
|
let summary = export_cir_static_pool(
|
||||||
|
&store,
|
||||||
|
&static_root,
|
||||||
|
sample_date(),
|
||||||
|
&cir,
|
||||||
|
&[&ta1, &ta2],
|
||||||
|
)
|
||||||
|
.expect("export static pool");
|
||||||
|
assert!(summary.unique_hashes >= 3);
|
||||||
|
assert!(summary.written_files >= 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_cir_raw_store_reports_missing_non_ta_object_and_writes_ta_entries() {
|
||||||
|
let td = tempfile::tempdir().unwrap();
|
||||||
|
let raw_store_path = td.path().join("raw-store.db");
|
||||||
|
let store = RocksStore::open_with_external_raw_store(&td.path().join("db"), &raw_store_path)
|
||||||
|
.unwrap();
|
||||||
|
let ta1 = sample_trust_anchor();
|
||||||
|
let ta2 = sample_arin_trust_anchor();
|
||||||
|
|
||||||
|
let cir_only_tas = build_cir_from_run_multi(
|
||||||
|
&store,
|
||||||
|
&[
|
||||||
|
CirTalBinding {
|
||||||
|
trust_anchor: &ta1,
|
||||||
|
tal_uri: "https://example.test/apnic.tal",
|
||||||
|
},
|
||||||
|
CirTalBinding {
|
||||||
|
trust_anchor: &ta2,
|
||||||
|
tal_uri: "https://example.test/arin.tal",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
sample_time(),
|
||||||
|
&[],
|
||||||
|
Some(&[]),
|
||||||
|
)
|
||||||
|
.expect("build cir with tas only");
|
||||||
|
|
||||||
|
let summary = export_cir_raw_store(
|
||||||
|
&store,
|
||||||
|
&raw_store_path,
|
||||||
|
&cir_only_tas,
|
||||||
|
&[&ta1, &ta2],
|
||||||
|
)
|
||||||
|
.expect("export raw store");
|
||||||
|
assert!(summary.unique_hashes >= 2);
|
||||||
|
assert!(summary.written_entries >= 2 || summary.reused_entries >= 2);
|
||||||
|
|
||||||
|
let mut cir_missing_object = cir_only_tas.clone();
|
||||||
|
cir_missing_object.objects.push(CirObject {
|
||||||
|
rsync_uri: "rsync://example.test/repo/missing.roa".to_string(),
|
||||||
|
sha256: vec![0x44; 32],
|
||||||
|
});
|
||||||
|
let err = export_cir_raw_store(
|
||||||
|
&store,
|
||||||
|
&raw_store_path,
|
||||||
|
&cir_missing_object,
|
||||||
|
&[&ta1, &ta2],
|
||||||
|
)
|
||||||
|
.expect_err("missing non-ta object must fail");
|
||||||
|
assert!(matches!(err, CirExportError::Write(_, _)), "{err}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -220,6 +220,89 @@ pub fn materialize_cir_from_raw_store(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn materialize_cir_from_repo_bytes(
|
||||||
|
cir: &CanonicalInputRepresentation,
|
||||||
|
repo_bytes_db: &Path,
|
||||||
|
mirror_root: &Path,
|
||||||
|
clean_rebuild: bool,
|
||||||
|
) -> Result<CirMaterializeSummary, CirMaterializeError> {
|
||||||
|
cir.validate().map_err(CirMaterializeError::TreeMismatch)?;
|
||||||
|
|
||||||
|
if clean_rebuild && mirror_root.exists() {
|
||||||
|
fs::remove_dir_all(mirror_root).map_err(|e| CirMaterializeError::RemoveMirrorRoot {
|
||||||
|
path: mirror_root.display().to_string(),
|
||||||
|
detail: e.to_string(),
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
fs::create_dir_all(mirror_root).map_err(|e| CirMaterializeError::CreateMirrorRoot {
|
||||||
|
path: mirror_root.display().to_string(),
|
||||||
|
detail: e.to_string(),
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let repo_bytes =
|
||||||
|
ExternalRawStoreDb::open(repo_bytes_db).map_err(|e| CirMaterializeError::OpenRawStore {
|
||||||
|
path: repo_bytes_db.display().to_string(),
|
||||||
|
detail: e.to_string(),
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mut copied_files = 0usize;
|
||||||
|
for object in &cir.objects {
|
||||||
|
let sha256_hex = hex::encode(&object.sha256);
|
||||||
|
let bytes = repo_bytes
|
||||||
|
.get_blob_bytes(&sha256_hex)
|
||||||
|
.map_err(|e| CirMaterializeError::ReadRawStore {
|
||||||
|
sha256_hex: sha256_hex.clone(),
|
||||||
|
detail: e.to_string(),
|
||||||
|
})?
|
||||||
|
.ok_or_else(|| CirMaterializeError::MissingRawStoreObject {
|
||||||
|
sha256_hex: sha256_hex.clone(),
|
||||||
|
})?;
|
||||||
|
let relative = mirror_relative_path_for_rsync_uri(&object.rsync_uri)?;
|
||||||
|
let target = mirror_root.join(&relative);
|
||||||
|
|
||||||
|
if let Some(parent) = target.parent() {
|
||||||
|
fs::create_dir_all(parent).map_err(|e| CirMaterializeError::CreateParent {
|
||||||
|
path: parent.display().to_string(),
|
||||||
|
detail: e.to_string(),
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
|
||||||
|
if target.exists() {
|
||||||
|
fs::remove_file(&target).map_err(|e| CirMaterializeError::RemoveExistingTarget {
|
||||||
|
path: target.display().to_string(),
|
||||||
|
detail: e.to_string(),
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
|
||||||
|
fs::write(&target, &bytes).map_err(|e| CirMaterializeError::Copy {
|
||||||
|
src: repo_bytes_db.display().to_string(),
|
||||||
|
dst: target.display().to_string(),
|
||||||
|
detail: e.to_string(),
|
||||||
|
})?;
|
||||||
|
copied_files += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
let actual = collect_materialized_uris(mirror_root)?;
|
||||||
|
let expected = cir
|
||||||
|
.objects
|
||||||
|
.iter()
|
||||||
|
.map(|item| item.rsync_uri.clone())
|
||||||
|
.collect::<std::collections::BTreeSet<_>>();
|
||||||
|
if actual != expected {
|
||||||
|
return Err(CirMaterializeError::TreeMismatch(format!(
|
||||||
|
"expected {} files, got {} files",
|
||||||
|
expected.len(),
|
||||||
|
actual.len()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(CirMaterializeSummary {
|
||||||
|
object_count: cir.objects.len(),
|
||||||
|
linked_files: 0,
|
||||||
|
copied_files,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn mirror_relative_path_for_rsync_uri(rsync_uri: &str) -> Result<PathBuf, CirMaterializeError> {
|
pub fn mirror_relative_path_for_rsync_uri(rsync_uri: &str) -> Result<PathBuf, CirMaterializeError> {
|
||||||
let url = url::Url::parse(rsync_uri)
|
let url = url::Url::parse(rsync_uri)
|
||||||
.map_err(|_| CirMaterializeError::InvalidRsyncUri(rsync_uri.to_string()))?;
|
.map_err(|_| CirMaterializeError::InvalidRsyncUri(rsync_uri.to_string()))?;
|
||||||
@ -317,7 +400,8 @@ fn collect_materialized_uris(
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
CirMaterializeError, materialize_cir, materialize_cir_from_raw_store,
|
CirMaterializeError, materialize_cir, materialize_cir_from_raw_store,
|
||||||
mirror_relative_path_for_rsync_uri, resolve_static_pool_file,
|
materialize_cir_from_repo_bytes, mirror_relative_path_for_rsync_uri,
|
||||||
|
resolve_static_pool_file,
|
||||||
};
|
};
|
||||||
use crate::blob_store::ExternalRawStoreDb;
|
use crate::blob_store::ExternalRawStoreDb;
|
||||||
use crate::cir::model::{
|
use crate::cir::model::{
|
||||||
@ -618,8 +702,7 @@ mod tests {
|
|||||||
std::fs::create_dir_all(mirror_root.join("extra")).unwrap();
|
std::fs::create_dir_all(mirror_root.join("extra")).unwrap();
|
||||||
std::fs::write(mirror_root.join("extra/stale.txt"), b"stale").unwrap();
|
std::fs::write(mirror_root.join("extra/stale.txt"), b"stale").unwrap();
|
||||||
|
|
||||||
let err =
|
let err = materialize_cir_from_raw_store(&cir, &raw_store_path, &mirror_root, false)
|
||||||
materialize_cir_from_raw_store(&cir, &raw_store_path, &mirror_root, false)
|
|
||||||
.expect_err("stale file should fail exact tree check");
|
.expect_err("stale file should fail exact tree check");
|
||||||
assert!(matches!(err, CirMaterializeError::TreeMismatch(_)));
|
assert!(matches!(err, CirMaterializeError::TreeMismatch(_)));
|
||||||
}
|
}
|
||||||
@ -698,10 +781,68 @@ mod tests {
|
|||||||
.expect_err("corrupt raw-store object should fail");
|
.expect_err("corrupt raw-store object should fail");
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
err,
|
err,
|
||||||
CirMaterializeError::ReadRawStore { .. } | CirMaterializeError::MissingRawStoreObject { .. }
|
CirMaterializeError::ReadRawStore { .. }
|
||||||
|
| CirMaterializeError::MissingRawStoreObject { .. }
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn materialize_from_repo_bytes_creates_expected_tree() {
|
||||||
|
let td = tempfile::tempdir().unwrap();
|
||||||
|
let repo_bytes_db = td.path().join("repo-bytes.db");
|
||||||
|
let mirror_root = td.path().join("mirror");
|
||||||
|
let a = b"a".to_vec();
|
||||||
|
let b = b"b".to_vec();
|
||||||
|
let cir = CanonicalInputRepresentation {
|
||||||
|
version: CIR_VERSION_V1,
|
||||||
|
hash_alg: CirHashAlgorithm::Sha256,
|
||||||
|
validation_time: sample_time(),
|
||||||
|
objects: vec![
|
||||||
|
CirObject {
|
||||||
|
rsync_uri: "rsync://example.net/repo/a.cer".to_string(),
|
||||||
|
sha256: sha2::Sha256::digest(&a).to_vec(),
|
||||||
|
},
|
||||||
|
CirObject {
|
||||||
|
rsync_uri: "rsync://example.net/repo/nested/b.roa".to_string(),
|
||||||
|
sha256: sha2::Sha256::digest(&b).to_vec(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
tals: vec![CirTal {
|
||||||
|
tal_uri: "https://tal.example.net/root.tal".to_string(),
|
||||||
|
tal_bytes: b"x".to_vec(),
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
|
||||||
|
{
|
||||||
|
let repo_bytes = ExternalRawStoreDb::open(&repo_bytes_db).unwrap();
|
||||||
|
let mut entry_a =
|
||||||
|
crate::storage::RawByHashEntry::from_bytes(hex::encode(&cir.objects[0].sha256), a);
|
||||||
|
entry_a.origin_uris.push(cir.objects[0].rsync_uri.clone());
|
||||||
|
repo_bytes.put_raw_entry(&entry_a).unwrap();
|
||||||
|
|
||||||
|
let mut entry_b =
|
||||||
|
crate::storage::RawByHashEntry::from_bytes(hex::encode(&cir.objects[1].sha256), b);
|
||||||
|
entry_b.origin_uris.push(cir.objects[1].rsync_uri.clone());
|
||||||
|
repo_bytes.put_raw_entry(&entry_b).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
let summary =
|
||||||
|
materialize_cir_from_repo_bytes(&cir, &repo_bytes_db, &mirror_root, true).unwrap();
|
||||||
|
assert_eq!(summary.object_count, 2);
|
||||||
|
assert_eq!(summary.linked_files, 0);
|
||||||
|
assert_eq!(summary.copied_files, 2);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(mirror_root.join("example.net/repo/a.cer")).unwrap(),
|
||||||
|
b"a"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(mirror_root.join("example.net/repo/nested/b.roa")).unwrap(),
|
||||||
|
b"b"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
fn write_static(root: &Path, date: &str, hash: &str, bytes: &[u8]) {
|
fn write_static(root: &Path, date: &str, hash: &str, bytes: &[u8]) {
|
||||||
let path = root.join(date).join(&hash[0..2]).join(&hash[2..4]);
|
let path = root.join(date).join(&hash[0..2]).join(&hash[2..4]);
|
||||||
std::fs::create_dir_all(&path).unwrap();
|
std::fs::create_dir_all(&path).unwrap();
|
||||||
|
|||||||
@ -12,12 +12,13 @@ pub use decode::{CirDecodeError, decode_cir};
|
|||||||
pub use encode::{CirEncodeError, encode_cir};
|
pub use encode::{CirEncodeError, encode_cir};
|
||||||
#[cfg(feature = "full")]
|
#[cfg(feature = "full")]
|
||||||
pub use export::{
|
pub use export::{
|
||||||
CirExportError, CirExportSummary, build_cir_from_run, export_cir_from_run, write_cir_file,
|
CirExportError, CirExportSummary, CirTalBinding, build_cir_from_run, build_cir_from_run_multi,
|
||||||
|
export_cir_from_run, export_cir_from_run_multi, write_cir_file,
|
||||||
};
|
};
|
||||||
pub use materialize::{
|
pub use materialize::{
|
||||||
CirMaterializeError, CirMaterializeSummary, materialize_cir,
|
CirMaterializeError, CirMaterializeSummary, materialize_cir, materialize_cir_from_raw_store,
|
||||||
materialize_cir_from_raw_store,
|
materialize_cir_from_repo_bytes, mirror_relative_path_for_rsync_uri,
|
||||||
mirror_relative_path_for_rsync_uri, resolve_static_pool_file,
|
resolve_static_pool_file,
|
||||||
};
|
};
|
||||||
pub use model::{
|
pub use model::{
|
||||||
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal,
|
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal,
|
||||||
|
|||||||
@ -41,9 +41,12 @@ impl CirSequenceManifest {
|
|||||||
if self.version == 0 {
|
if self.version == 0 {
|
||||||
return Err("sequence.version must be positive".to_string());
|
return Err("sequence.version must be positive".to_string());
|
||||||
}
|
}
|
||||||
let backend_count = self.static_root.is_some() as u8 + self.raw_store_db_path.is_some() as u8;
|
let backend_count =
|
||||||
|
self.static_root.is_some() as u8 + self.raw_store_db_path.is_some() as u8;
|
||||||
if backend_count != 1 {
|
if backend_count != 1 {
|
||||||
return Err("sequence must set exactly one of static_root or raw_store_db_path".to_string());
|
return Err(
|
||||||
|
"sequence must set exactly one of static_root or raw_store_db_path".to_string(),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
match (self.static_root.as_ref(), self.raw_store_db_path.as_ref()) {
|
match (self.static_root.as_ref(), self.raw_store_db_path.as_ref()) {
|
||||||
(Some(static_root), None) if !static_root.trim().is_empty() => {}
|
(Some(static_root), None) if !static_root.trim().is_empty() => {}
|
||||||
|
|||||||
220
src/cli.rs
220
src/cli.rs
@ -1,11 +1,11 @@
|
|||||||
use crate::ccr::{build_ccr_from_run, write_ccr_file};
|
use crate::ccr::{build_ccr_from_run, write_ccr_file};
|
||||||
use crate::cir::export_cir_from_run;
|
use crate::cir::{CirTalBinding, export_cir_from_run_multi};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use crate::analysis::timing::{TimingHandle, TimingMeta, TimingMetaUpdate};
|
use crate::analysis::timing::{TimingHandle, TimingMeta, TimingMetaUpdate};
|
||||||
use crate::audit::{
|
use crate::audit::{
|
||||||
format_roa_ip_prefix, AspaOutput, AuditRepoSyncStats, AuditReportV2, AuditRunMeta,
|
AspaOutput, AuditRepoSyncStats, AuditReportV2, AuditRunMeta, AuditWarning, TreeSummary,
|
||||||
AuditWarning, TreeSummary, VrpOutput,
|
VrpOutput, format_roa_ip_prefix,
|
||||||
};
|
};
|
||||||
use crate::fetch::http::{BlockingHttpFetcher, HttpFetcherConfig};
|
use crate::fetch::http::{BlockingHttpFetcher, HttpFetcherConfig};
|
||||||
use crate::fetch::rsync::LocalDirRsyncFetcher;
|
use crate::fetch::rsync::LocalDirRsyncFetcher;
|
||||||
@ -15,16 +15,16 @@ use crate::parallel::types::TalInputSpec;
|
|||||||
use crate::policy::Policy;
|
use crate::policy::Policy;
|
||||||
use crate::storage::RocksStore;
|
use crate::storage::RocksStore;
|
||||||
use crate::validation::run_tree_from_tal::{
|
use crate::validation::run_tree_from_tal::{
|
||||||
|
RunTreeFromTalAuditOutput, run_tree_from_multiple_tals_parallel_phase1_audit,
|
||||||
|
run_tree_from_tal_and_ta_der_parallel_phase1_audit,
|
||||||
run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit,
|
run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit,
|
||||||
run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit_with_timing,
|
run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit_with_timing,
|
||||||
run_tree_from_tal_and_ta_der_payload_replay_serial_audit,
|
run_tree_from_tal_and_ta_der_payload_replay_serial_audit,
|
||||||
run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing,
|
run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing,
|
||||||
run_tree_from_multiple_tals_parallel_phase1_audit,
|
|
||||||
run_tree_from_tal_and_ta_der_parallel_phase1_audit,
|
|
||||||
run_tree_from_tal_and_ta_der_serial_audit,
|
run_tree_from_tal_and_ta_der_serial_audit,
|
||||||
run_tree_from_tal_and_ta_der_serial_audit_with_timing, run_tree_from_tal_url_serial_audit,
|
run_tree_from_tal_and_ta_der_serial_audit_with_timing,
|
||||||
run_tree_from_tal_url_parallel_phase1_audit,
|
run_tree_from_tal_url_parallel_phase1_audit, run_tree_from_tal_url_serial_audit,
|
||||||
run_tree_from_tal_url_serial_audit_with_timing, RunTreeFromTalAuditOutput,
|
run_tree_from_tal_url_serial_audit_with_timing,
|
||||||
};
|
};
|
||||||
use crate::validation::tree::TreeRunConfig;
|
use crate::validation::tree::TreeRunConfig;
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
@ -38,7 +38,6 @@ struct RunStageTiming {
|
|||||||
ccr_build_ms: Option<u64>,
|
ccr_build_ms: Option<u64>,
|
||||||
ccr_write_ms: Option<u64>,
|
ccr_write_ms: Option<u64>,
|
||||||
cir_build_cir_ms: Option<u64>,
|
cir_build_cir_ms: Option<u64>,
|
||||||
cir_static_pool_ms: Option<u64>,
|
|
||||||
cir_write_cir_ms: Option<u64>,
|
cir_write_cir_ms: Option<u64>,
|
||||||
cir_total_ms: Option<u64>,
|
cir_total_ms: Option<u64>,
|
||||||
total_ms: u64,
|
total_ms: u64,
|
||||||
@ -71,6 +70,7 @@ pub struct CliArgs {
|
|||||||
pub cir_enabled: bool,
|
pub cir_enabled: bool,
|
||||||
pub cir_out_path: Option<PathBuf>,
|
pub cir_out_path: Option<PathBuf>,
|
||||||
pub cir_static_root: Option<PathBuf>,
|
pub cir_static_root: Option<PathBuf>,
|
||||||
|
pub cir_tal_uris: Vec<String>,
|
||||||
pub cir_tal_uri: Option<String>,
|
pub cir_tal_uri: Option<String>,
|
||||||
pub payload_replay_archive: Option<PathBuf>,
|
pub payload_replay_archive: Option<PathBuf>,
|
||||||
pub payload_replay_locks: Option<PathBuf>,
|
pub payload_replay_locks: Option<PathBuf>,
|
||||||
@ -112,8 +112,8 @@ Options:
|
|||||||
--ccr-out <path> Write CCR DER ContentInfo to this path (optional)
|
--ccr-out <path> Write CCR DER ContentInfo to this path (optional)
|
||||||
--cir-enable Export CIR after the run completes
|
--cir-enable Export CIR after the run completes
|
||||||
--cir-out <path> Write CIR DER to this path (requires --cir-enable)
|
--cir-out <path> Write CIR DER to this path (requires --cir-enable)
|
||||||
--cir-static-root <path> Shared static pool root for CIR export (requires --cir-enable unless --raw-store-db is used)
|
--cir-static-root <path> Deprecated; CIR export no longer exports object pools
|
||||||
--cir-tal-uri <url> Override TAL URI for CIR export when using --tal-path (optional)
|
--cir-tal-uri <url> Override TAL URI for CIR export (repeatable in multi-TAL mode)
|
||||||
--payload-replay-archive <path> Use local payload replay archive root (offline replay mode)
|
--payload-replay-archive <path> Use local payload replay archive root (offline replay mode)
|
||||||
--payload-replay-locks <path> Use local payload replay locks.json (offline replay mode)
|
--payload-replay-locks <path> Use local payload replay locks.json (offline replay mode)
|
||||||
--payload-base-archive <path> Use local base payload archive root (offline delta replay)
|
--payload-base-archive <path> Use local base payload archive root (offline delta replay)
|
||||||
@ -166,6 +166,7 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
|||||||
let mut cir_enabled: bool = false;
|
let mut cir_enabled: bool = false;
|
||||||
let mut cir_out_path: Option<PathBuf> = None;
|
let mut cir_out_path: Option<PathBuf> = None;
|
||||||
let mut cir_static_root: Option<PathBuf> = None;
|
let mut cir_static_root: Option<PathBuf> = None;
|
||||||
|
let mut cir_tal_uris: Vec<String> = Vec::new();
|
||||||
let mut cir_tal_uri: Option<String> = None;
|
let mut cir_tal_uri: Option<String> = None;
|
||||||
let mut payload_replay_archive: Option<PathBuf> = None;
|
let mut payload_replay_archive: Option<PathBuf> = None;
|
||||||
let mut payload_replay_locks: Option<PathBuf> = None;
|
let mut payload_replay_locks: Option<PathBuf> = None;
|
||||||
@ -282,7 +283,8 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
|||||||
"--cir-tal-uri" => {
|
"--cir-tal-uri" => {
|
||||||
i += 1;
|
i += 1;
|
||||||
let v = argv.get(i).ok_or("--cir-tal-uri requires a value")?;
|
let v = argv.get(i).ok_or("--cir-tal-uri requires a value")?;
|
||||||
cir_tal_uri = Some(v.clone());
|
cir_tal_uris.push(v.clone());
|
||||||
|
cir_tal_uri = cir_tal_uris.first().cloned();
|
||||||
}
|
}
|
||||||
"--payload-replay-archive" => {
|
"--payload-replay-archive" => {
|
||||||
i += 1;
|
i += 1;
|
||||||
@ -424,7 +426,8 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
if !tal_paths.is_empty() {
|
if !tal_paths.is_empty() {
|
||||||
let strict_pairing_required = parallel_phase1 || tal_paths.len() > 1 || !ta_paths.is_empty();
|
let strict_pairing_required =
|
||||||
|
parallel_phase1 || tal_paths.len() > 1 || !ta_paths.is_empty();
|
||||||
if strict_pairing_required {
|
if strict_pairing_required {
|
||||||
if ta_paths.len() != tal_paths.len() {
|
if ta_paths.len() != tal_paths.len() {
|
||||||
return Err(format!(
|
return Err(format!(
|
||||||
@ -448,24 +451,42 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
|||||||
let tal_url = tal_urls.first().cloned();
|
let tal_url = tal_urls.first().cloned();
|
||||||
let tal_path = tal_paths.first().cloned();
|
let tal_path = tal_paths.first().cloned();
|
||||||
let ta_path = ta_paths.first().cloned();
|
let ta_path = ta_paths.first().cloned();
|
||||||
let cir_backend_count = cir_static_root.is_some() as u8 + raw_store_db.is_some() as u8;
|
if cir_enabled && cir_out_path.is_none() {
|
||||||
if cir_enabled && (cir_out_path.is_none() || cir_backend_count != 1) {
|
|
||||||
return Err(format!(
|
return Err(format!(
|
||||||
"--cir-enable requires --cir-out and exactly one of --cir-static-root or --raw-store-db\n\n{}",
|
"--cir-enable requires --cir-out\n\n{}",
|
||||||
|
usage()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if cir_static_root.is_some() {
|
||||||
|
return Err(format!(
|
||||||
|
"--cir-static-root is no longer supported; CIR export now writes only .cir files\n\n{}",
|
||||||
usage()
|
usage()
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if !cir_enabled
|
if !cir_enabled
|
||||||
&& (cir_out_path.is_some() || cir_static_root.is_some() || cir_tal_uri.is_some())
|
&& (cir_out_path.is_some() || !cir_tal_uris.is_empty())
|
||||||
{
|
{
|
||||||
return Err(format!(
|
return Err(format!(
|
||||||
"--cir-out/--cir-static-root/--cir-tal-uri require --cir-enable\n\n{}",
|
"--cir-out/--cir-tal-uri require --cir-enable\n\n{}",
|
||||||
usage()
|
usage()
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if cir_enabled && tal_path.is_some() && cir_tal_uri.is_none() {
|
if cir_enabled && !cir_tal_uris.is_empty() {
|
||||||
|
let expected = if !tal_paths.is_empty() {
|
||||||
|
tal_paths.len()
|
||||||
|
} else {
|
||||||
|
tal_urls.len()
|
||||||
|
};
|
||||||
|
if cir_tal_uris.len() != expected {
|
||||||
return Err(format!(
|
return Err(format!(
|
||||||
"CIR export in --tal-path mode requires --cir-tal-uri\n\n{}",
|
"--cir-tal-uri count must match TAL input count when provided\n\n{}",
|
||||||
|
usage()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if cir_enabled && !tal_paths.is_empty() && cir_tal_uris.is_empty() {
|
||||||
|
return Err(format!(
|
||||||
|
"CIR export in --tal-path mode requires --cir-tal-uri for each TAL\n\n{}",
|
||||||
usage()
|
usage()
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
@ -559,13 +580,9 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
|||||||
tal_inputs.extend(tal_urls.iter().cloned().map(TalInputSpec::from_url));
|
tal_inputs.extend(tal_urls.iter().cloned().map(TalInputSpec::from_url));
|
||||||
} else if !tal_paths.is_empty() {
|
} else if !tal_paths.is_empty() {
|
||||||
if ta_paths.len() == tal_paths.len() {
|
if ta_paths.len() == tal_paths.len() {
|
||||||
tal_inputs.extend(
|
tal_inputs.extend(tal_paths.iter().cloned().zip(ta_paths.iter().cloned()).map(
|
||||||
tal_paths
|
|(tal_path, ta_path)| TalInputSpec::from_file_path_with_ta(tal_path, ta_path),
|
||||||
.iter()
|
));
|
||||||
.cloned()
|
|
||||||
.zip(ta_paths.iter().cloned())
|
|
||||||
.map(|(tal_path, ta_path)| TalInputSpec::from_file_path_with_ta(tal_path, ta_path)),
|
|
||||||
);
|
|
||||||
} else {
|
} else {
|
||||||
tal_inputs.extend(tal_paths.iter().cloned().map(TalInputSpec::from_file_path));
|
tal_inputs.extend(tal_paths.iter().cloned().map(TalInputSpec::from_file_path));
|
||||||
}
|
}
|
||||||
@ -590,6 +607,7 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
|
|||||||
cir_enabled,
|
cir_enabled,
|
||||||
cir_out_path,
|
cir_out_path,
|
||||||
cir_static_root,
|
cir_static_root,
|
||||||
|
cir_tal_uris,
|
||||||
cir_tal_uri,
|
cir_tal_uri,
|
||||||
payload_replay_archive,
|
payload_replay_archive,
|
||||||
payload_replay_locks,
|
payload_replay_locks,
|
||||||
@ -725,6 +743,16 @@ fn build_report(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn resolve_cir_export_tal_uris(args: &CliArgs) -> Result<Vec<String>, String> {
|
||||||
|
if !args.cir_tal_uris.is_empty() {
|
||||||
|
return Ok(args.cir_tal_uris.clone());
|
||||||
|
}
|
||||||
|
if !args.tal_urls.is_empty() {
|
||||||
|
return Ok(args.tal_urls.clone());
|
||||||
|
}
|
||||||
|
Err("CIR export requires TAL URI source(s)".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
fn build_repo_sync_stats(
|
fn build_repo_sync_stats(
|
||||||
publication_points: &[crate::audit::PublicationPointAudit],
|
publication_points: &[crate::audit::PublicationPointAudit],
|
||||||
) -> AuditRepoSyncStats {
|
) -> AuditRepoSyncStats {
|
||||||
@ -1342,56 +1370,53 @@ pub fn run(argv: &[String]) -> Result<(), String> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let mut cir_build_cir_ms = None;
|
let mut cir_build_cir_ms = None;
|
||||||
let mut cir_static_pool_ms = None;
|
|
||||||
let mut cir_write_cir_ms = None;
|
let mut cir_write_cir_ms = None;
|
||||||
let mut cir_total_ms = None;
|
let mut cir_total_ms = None;
|
||||||
if args.cir_enabled {
|
if args.cir_enabled {
|
||||||
if out.discoveries.len() > 1 {
|
let cir_tal_uris = resolve_cir_export_tal_uris(&args)?;
|
||||||
return Err("CIR export is not yet supported for multi-TAL runs".to_string());
|
if cir_tal_uris.len() != out.discoveries.len() {
|
||||||
|
return Err(format!(
|
||||||
|
"CIR export TAL URI count ({}) does not match discovery count ({})",
|
||||||
|
cir_tal_uris.len(),
|
||||||
|
out.discoveries.len()
|
||||||
|
));
|
||||||
}
|
}
|
||||||
let cir_tal_uri = args
|
|
||||||
.tal_url
|
|
||||||
.clone()
|
|
||||||
.or(args.cir_tal_uri.clone())
|
|
||||||
.ok_or_else(|| "CIR export requires a TAL URI source".to_string())?;
|
|
||||||
let cir_out_path = args
|
let cir_out_path = args
|
||||||
.cir_out_path
|
.cir_out_path
|
||||||
.as_deref()
|
.as_deref()
|
||||||
.expect("validated by parse_args for cir");
|
.expect("validated by parse_args for cir");
|
||||||
let summary = export_cir_from_run(
|
let tal_bindings = out
|
||||||
|
.discoveries
|
||||||
|
.iter()
|
||||||
|
.zip(cir_tal_uris.iter())
|
||||||
|
.map(|(discovery, tal_uri)| CirTalBinding {
|
||||||
|
trust_anchor: &discovery.trust_anchor,
|
||||||
|
tal_uri: tal_uri.as_str(),
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let summary = export_cir_from_run_multi(
|
||||||
store.as_ref(),
|
store.as_ref(),
|
||||||
&out.discovery.trust_anchor,
|
&tal_bindings,
|
||||||
&cir_tal_uri,
|
|
||||||
validation_time,
|
validation_time,
|
||||||
&out.publication_points,
|
&out.publication_points,
|
||||||
cir_out_path,
|
cir_out_path,
|
||||||
args.cir_static_root.as_deref(),
|
|
||||||
args.raw_store_db.as_deref(),
|
|
||||||
time::OffsetDateTime::now_utc().date(),
|
time::OffsetDateTime::now_utc().date(),
|
||||||
|
if out.current_repo_objects.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(out.current_repo_objects.as_slice())
|
||||||
|
},
|
||||||
)
|
)
|
||||||
.map_err(|e| e.to_string())?;
|
.map_err(|e| e.to_string())?;
|
||||||
cir_build_cir_ms = Some(summary.timing.build_cir_ms);
|
cir_build_cir_ms = Some(summary.timing.build_cir_ms);
|
||||||
cir_static_pool_ms = Some(summary.timing.static_pool_ms);
|
|
||||||
cir_write_cir_ms = Some(summary.timing.write_cir_ms);
|
cir_write_cir_ms = Some(summary.timing.write_cir_ms);
|
||||||
cir_total_ms = Some(summary.timing.total_ms);
|
cir_total_ms = Some(summary.timing.total_ms);
|
||||||
let (backend_name, written_entries, reused_entries) = match &summary.object_pool {
|
|
||||||
crate::cir::export::CirObjectPoolExportSummary::Static(s) => {
|
|
||||||
("static", s.written_files, s.reused_files)
|
|
||||||
}
|
|
||||||
crate::cir::export::CirObjectPoolExportSummary::RawStore(s) => {
|
|
||||||
("raw-store", s.written_entries, s.reused_entries)
|
|
||||||
}
|
|
||||||
};
|
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"wrote CIR: {} (objects={}, tals={}, backend={}, written={}, reused={}, build_cir_ms={}, static_pool_ms={}, write_cir_ms={}, total_ms={})",
|
"wrote CIR: {} (objects={}, tals={}, build_cir_ms={}, write_cir_ms={}, total_ms={})",
|
||||||
cir_out_path.display(),
|
cir_out_path.display(),
|
||||||
summary.object_count,
|
summary.object_count,
|
||||||
summary.tal_count,
|
summary.tal_count,
|
||||||
backend_name,
|
|
||||||
written_entries,
|
|
||||||
reused_entries,
|
|
||||||
summary.timing.build_cir_ms,
|
summary.timing.build_cir_ms,
|
||||||
summary.timing.static_pool_ms,
|
|
||||||
summary.timing.write_cir_ms,
|
summary.timing.write_cir_ms,
|
||||||
summary.timing.total_ms
|
summary.timing.total_ms
|
||||||
);
|
);
|
||||||
@ -1414,7 +1439,6 @@ pub fn run(argv: &[String]) -> Result<(), String> {
|
|||||||
ccr_build_ms,
|
ccr_build_ms,
|
||||||
ccr_write_ms,
|
ccr_write_ms,
|
||||||
cir_build_cir_ms,
|
cir_build_cir_ms,
|
||||||
cir_static_pool_ms,
|
|
||||||
cir_write_cir_ms,
|
cir_write_cir_ms,
|
||||||
cir_total_ms,
|
cir_total_ms,
|
||||||
total_ms: total_started.elapsed().as_millis() as u64,
|
total_ms: total_started.elapsed().as_millis() as u64,
|
||||||
@ -1634,8 +1658,6 @@ mod tests {
|
|||||||
"--cir-enable".to_string(),
|
"--cir-enable".to_string(),
|
||||||
"--cir-out".to_string(),
|
"--cir-out".to_string(),
|
||||||
"out/example.cir".to_string(),
|
"out/example.cir".to_string(),
|
||||||
"--cir-static-root".to_string(),
|
|
||||||
"out/static".to_string(),
|
|
||||||
"--cir-tal-uri".to_string(),
|
"--cir-tal-uri".to_string(),
|
||||||
"https://example.test/root.tal".to_string(),
|
"https://example.test/root.tal".to_string(),
|
||||||
];
|
];
|
||||||
@ -1645,14 +1667,67 @@ mod tests {
|
|||||||
args.cir_out_path.as_deref(),
|
args.cir_out_path.as_deref(),
|
||||||
Some(std::path::Path::new("out/example.cir"))
|
Some(std::path::Path::new("out/example.cir"))
|
||||||
);
|
);
|
||||||
assert_eq!(
|
|
||||||
args.cir_static_root.as_deref(),
|
|
||||||
Some(std::path::Path::new("out/static"))
|
|
||||||
);
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
args.cir_tal_uri.as_deref(),
|
args.cir_tal_uri.as_deref(),
|
||||||
Some("https://example.test/root.tal")
|
Some("https://example.test/root.tal")
|
||||||
);
|
);
|
||||||
|
assert_eq!(
|
||||||
|
args.cir_tal_uris,
|
||||||
|
vec!["https://example.test/root.tal".to_string()]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_rejects_deprecated_cir_static_root() {
|
||||||
|
let argv = vec![
|
||||||
|
"rpki".to_string(),
|
||||||
|
"--db".to_string(),
|
||||||
|
"db".to_string(),
|
||||||
|
"--tal-url".to_string(),
|
||||||
|
"https://example.test/root.tal".to_string(),
|
||||||
|
"--cir-enable".to_string(),
|
||||||
|
"--cir-out".to_string(),
|
||||||
|
"out/example.cir".to_string(),
|
||||||
|
"--cir-static-root".to_string(),
|
||||||
|
"out/static".to_string(),
|
||||||
|
];
|
||||||
|
let err = parse_args(&argv).expect_err("cir-static-root should be rejected");
|
||||||
|
assert!(err.contains("no longer supported"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_accepts_multi_tal_cir_overrides_in_file_mode() {
|
||||||
|
let argv = vec![
|
||||||
|
"rpki".to_string(),
|
||||||
|
"--db".to_string(),
|
||||||
|
"db".to_string(),
|
||||||
|
"--tal-path".to_string(),
|
||||||
|
"apnic.tal".to_string(),
|
||||||
|
"--ta-path".to_string(),
|
||||||
|
"apnic.cer".to_string(),
|
||||||
|
"--tal-path".to_string(),
|
||||||
|
"arin.tal".to_string(),
|
||||||
|
"--ta-path".to_string(),
|
||||||
|
"arin.cer".to_string(),
|
||||||
|
"--parallel-phase1".to_string(),
|
||||||
|
"--rsync-local-dir".to_string(),
|
||||||
|
"repo".to_string(),
|
||||||
|
"--cir-enable".to_string(),
|
||||||
|
"--cir-out".to_string(),
|
||||||
|
"out/example.cir".to_string(),
|
||||||
|
"--cir-tal-uri".to_string(),
|
||||||
|
"https://example.test/apnic.tal".to_string(),
|
||||||
|
"--cir-tal-uri".to_string(),
|
||||||
|
"https://example.test/arin.tal".to_string(),
|
||||||
|
];
|
||||||
|
let args = parse_args(&argv).expect("parse args");
|
||||||
|
assert_eq!(
|
||||||
|
args.cir_tal_uris,
|
||||||
|
vec![
|
||||||
|
"https://example.test/apnic.tal".to_string(),
|
||||||
|
"https://example.test/arin.tal".to_string()
|
||||||
|
]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -1664,12 +1739,10 @@ mod tests {
|
|||||||
"--tal-url".to_string(),
|
"--tal-url".to_string(),
|
||||||
"https://example.test/root.tal".to_string(),
|
"https://example.test/root.tal".to_string(),
|
||||||
"--cir-enable".to_string(),
|
"--cir-enable".to_string(),
|
||||||
"--cir-out".to_string(),
|
|
||||||
"out/example.cir".to_string(),
|
|
||||||
];
|
];
|
||||||
let err = parse_args(&argv_missing).unwrap_err();
|
let err = parse_args(&argv_missing).unwrap_err();
|
||||||
assert!(
|
assert!(
|
||||||
err.contains("--cir-enable requires --cir-out and exactly one of --cir-static-root or --raw-store-db"),
|
err.contains("--cir-enable requires --cir-out"),
|
||||||
"{err}"
|
"{err}"
|
||||||
);
|
);
|
||||||
|
|
||||||
@ -1698,8 +1771,6 @@ mod tests {
|
|||||||
"--cir-enable".to_string(),
|
"--cir-enable".to_string(),
|
||||||
"--cir-out".to_string(),
|
"--cir-out".to_string(),
|
||||||
"out/example.cir".to_string(),
|
"out/example.cir".to_string(),
|
||||||
"--cir-static-root".to_string(),
|
|
||||||
"out/static".to_string(),
|
|
||||||
];
|
];
|
||||||
let err = parse_args(&argv_offline_missing_uri).unwrap_err();
|
let err = parse_args(&argv_offline_missing_uri).unwrap_err();
|
||||||
assert!(err.contains("requires --cir-tal-uri"), "{err}");
|
assert!(err.contains("requires --cir-tal-uri"), "{err}");
|
||||||
@ -1782,7 +1853,10 @@ mod tests {
|
|||||||
];
|
];
|
||||||
let args = parse_args(&argv).expect("parse");
|
let args = parse_args(&argv).expect("parse");
|
||||||
assert_eq!(args.tal_url.as_deref(), Some("https://example.test/x.tal"));
|
assert_eq!(args.tal_url.as_deref(), Some("https://example.test/x.tal"));
|
||||||
assert_eq!(args.tal_urls, vec!["https://example.test/x.tal".to_string()]);
|
assert_eq!(
|
||||||
|
args.tal_urls,
|
||||||
|
vec!["https://example.test/x.tal".to_string()]
|
||||||
|
);
|
||||||
assert!(args.tal_path.is_none());
|
assert!(args.tal_path.is_none());
|
||||||
assert!(args.ta_path.is_none());
|
assert!(args.ta_path.is_none());
|
||||||
assert_eq!(args.tal_inputs.len(), 1);
|
assert_eq!(args.tal_inputs.len(), 1);
|
||||||
@ -1912,7 +1986,10 @@ mod tests {
|
|||||||
"--parallel-phase1".to_string(),
|
"--parallel-phase1".to_string(),
|
||||||
];
|
];
|
||||||
let err = parse_args(&argv).unwrap_err();
|
let err = parse_args(&argv).unwrap_err();
|
||||||
assert!(err.contains("--tal-path and --ta-path counts must match"), "{err}");
|
assert!(
|
||||||
|
err.contains("--tal-path and --ta-path counts must match"),
|
||||||
|
"{err}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -2218,9 +2295,11 @@ mod tests {
|
|||||||
let tree = crate::validation::tree::TreeRunOutput {
|
let tree = crate::validation::tree::TreeRunOutput {
|
||||||
instances_processed: 1,
|
instances_processed: 1,
|
||||||
instances_failed: 0,
|
instances_failed: 0,
|
||||||
warnings: vec![crate::report::Warning::new("synthetic warning")
|
warnings: vec![
|
||||||
|
crate::report::Warning::new("synthetic warning")
|
||||||
.with_rfc_refs(&[crate::report::RfcRef("RFC 6487 §4.8.8.1")])
|
.with_rfc_refs(&[crate::report::RfcRef("RFC 6487 §4.8.8.1")])
|
||||||
.with_context("rsync://example.test/repo/pp/")],
|
.with_context("rsync://example.test/repo/pp/"),
|
||||||
|
],
|
||||||
vrps: vec![crate::validation::objects::Vrp {
|
vrps: vec![crate::validation::objects::Vrp {
|
||||||
asn: 64496,
|
asn: 64496,
|
||||||
prefix: crate::data_model::roa::IpPrefix {
|
prefix: crate::data_model::roa::IpPrefix {
|
||||||
@ -2251,6 +2330,7 @@ mod tests {
|
|||||||
publication_points: vec![pp1, pp2, pp3],
|
publication_points: vec![pp1, pp2, pp3],
|
||||||
downloads: Vec::new(),
|
downloads: Vec::new(),
|
||||||
download_stats: crate::audit::AuditDownloadStats::default(),
|
download_stats: crate::audit::AuditDownloadStats::default(),
|
||||||
|
current_repo_objects: Vec::new(),
|
||||||
};
|
};
|
||||||
|
|
||||||
let policy = Policy::default();
|
let policy = Policy::default();
|
||||||
|
|||||||
299
src/current_repo_index.rs
Normal file
299
src/current_repo_index.rs
Normal file
@ -0,0 +1,299 @@
|
|||||||
|
use std::collections::{HashMap, HashSet};
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
use crate::storage::{RepositoryViewEntry, RepositoryViewState};
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct CurrentRepoEntry {
|
||||||
|
pub current_hash: [u8; 32],
|
||||||
|
pub current_hash_hex: String,
|
||||||
|
pub repository_source: String,
|
||||||
|
pub object_type: Option<String>,
|
||||||
|
pub state: RepositoryViewState,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||||
|
pub struct CurrentRepoObject {
|
||||||
|
pub rsync_uri: String,
|
||||||
|
pub current_hash_hex: String,
|
||||||
|
pub repository_source: String,
|
||||||
|
pub object_type: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default, Debug)]
|
||||||
|
pub struct CurrentRepoIndex {
|
||||||
|
by_uri: HashMap<String, CurrentRepoEntry>,
|
||||||
|
by_scope: HashMap<String, HashSet<String>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub type CurrentRepoIndexHandle = Arc<Mutex<CurrentRepoIndex>>;
|
||||||
|
|
||||||
|
impl CurrentRepoIndex {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn shared() -> CurrentRepoIndexHandle {
|
||||||
|
Arc::new(Mutex::new(Self::new()))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_by_uri(&self, rsync_uri: &str) -> Option<&CurrentRepoEntry> {
|
||||||
|
self.by_uri.get(rsync_uri)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn list_scope_uris(&self, repository_source: &str) -> Vec<String> {
|
||||||
|
let mut out = self
|
||||||
|
.by_scope
|
||||||
|
.get(repository_source)
|
||||||
|
.map(|set| set.iter().cloned().collect::<Vec<_>>())
|
||||||
|
.unwrap_or_default();
|
||||||
|
out.sort();
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn active_uri_count(&self) -> usize {
|
||||||
|
self.by_uri.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn scope_count(&self) -> usize {
|
||||||
|
self.by_scope.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn snapshot_objects(&self) -> Vec<CurrentRepoObject> {
|
||||||
|
let mut out = self
|
||||||
|
.by_uri
|
||||||
|
.iter()
|
||||||
|
.map(|(rsync_uri, entry)| CurrentRepoObject {
|
||||||
|
rsync_uri: rsync_uri.clone(),
|
||||||
|
current_hash_hex: entry.current_hash_hex.clone(),
|
||||||
|
repository_source: entry.repository_source.clone(),
|
||||||
|
object_type: entry.object_type.clone(),
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
out.sort();
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn apply_repository_view_entries(
|
||||||
|
&mut self,
|
||||||
|
entries: &[RepositoryViewEntry],
|
||||||
|
) -> Result<(), String> {
|
||||||
|
for entry in entries {
|
||||||
|
self.apply_repository_view_entry(entry)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn apply_repository_view_entry(&mut self, entry: &RepositoryViewEntry) -> Result<(), String> {
|
||||||
|
entry.validate_internal().map_err(|e| e.to_string())?;
|
||||||
|
|
||||||
|
let old_scope = self
|
||||||
|
.by_uri
|
||||||
|
.get(&entry.rsync_uri)
|
||||||
|
.map(|existing| existing.repository_source.clone());
|
||||||
|
|
||||||
|
match entry.state {
|
||||||
|
RepositoryViewState::Present | RepositoryViewState::Replaced => {
|
||||||
|
let repository_source = entry.repository_source.clone().ok_or_else(|| {
|
||||||
|
format!(
|
||||||
|
"repository_view entry missing repository_source for current object {}",
|
||||||
|
entry.rsync_uri
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let current_hash_hex = entry.current_hash.clone().ok_or_else(|| {
|
||||||
|
format!(
|
||||||
|
"repository_view entry missing current_hash for current object {}",
|
||||||
|
entry.rsync_uri
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let current_hash = decode_sha256_hex_32(¤t_hash_hex)?;
|
||||||
|
|
||||||
|
if let Some(old_scope) = old_scope.as_ref() {
|
||||||
|
if old_scope != &repository_source {
|
||||||
|
self.remove_uri_from_scope(old_scope, &entry.rsync_uri);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
self.by_scope
|
||||||
|
.entry(repository_source.clone())
|
||||||
|
.or_default()
|
||||||
|
.insert(entry.rsync_uri.clone());
|
||||||
|
self.by_uri.insert(
|
||||||
|
entry.rsync_uri.clone(),
|
||||||
|
CurrentRepoEntry {
|
||||||
|
current_hash,
|
||||||
|
current_hash_hex: current_hash_hex.to_ascii_lowercase(),
|
||||||
|
repository_source,
|
||||||
|
object_type: entry.object_type.clone(),
|
||||||
|
state: entry.state,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
RepositoryViewState::Withdrawn => {
|
||||||
|
if let Some(scope) = entry.repository_source.as_ref().or(old_scope.as_ref()) {
|
||||||
|
self.remove_uri_from_scope(scope, &entry.rsync_uri);
|
||||||
|
}
|
||||||
|
self.by_uri.remove(&entry.rsync_uri);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_uri_from_scope(&mut self, scope: &str, rsync_uri: &str) {
|
||||||
|
let empty = if let Some(entries) = self.by_scope.get_mut(scope) {
|
||||||
|
entries.remove(rsync_uri);
|
||||||
|
entries.is_empty()
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
};
|
||||||
|
if empty {
|
||||||
|
self.by_scope.remove(scope);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_sha256_hex_32(value: &str) -> Result<[u8; 32], String> {
|
||||||
|
if value.len() != 64 || !value.as_bytes().iter().all(u8::is_ascii_hexdigit) {
|
||||||
|
return Err(format!("invalid sha256 hex: {value}"));
|
||||||
|
}
|
||||||
|
let mut out = [0u8; 32];
|
||||||
|
hex::decode_to_slice(value, &mut out).map_err(|e| format!("hex decode failed: {e}"))?;
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::CurrentRepoIndex;
|
||||||
|
use crate::storage::{RepositoryViewEntry, RepositoryViewState};
|
||||||
|
|
||||||
|
fn present(source: &str, uri: &str, hash: &str) -> RepositoryViewEntry {
|
||||||
|
RepositoryViewEntry {
|
||||||
|
rsync_uri: uri.to_string(),
|
||||||
|
current_hash: Some(hash.to_string()),
|
||||||
|
repository_source: Some(source.to_string()),
|
||||||
|
object_type: Some("roa".to_string()),
|
||||||
|
state: RepositoryViewState::Present,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn current_repo_index_tracks_present_and_withdrawn_entries() {
|
||||||
|
let mut index = CurrentRepoIndex::new();
|
||||||
|
let uri = "rsync://example.test/repo/a.roa";
|
||||||
|
let source = "rsync://example.test/repo/";
|
||||||
|
let hash = &"11".repeat(32);
|
||||||
|
index
|
||||||
|
.apply_repository_view_entries(&[present(source, uri, hash)])
|
||||||
|
.expect("apply present");
|
||||||
|
let got = index.get_by_uri(uri).expect("current entry");
|
||||||
|
assert_eq!(got.current_hash_hex, hash.to_string());
|
||||||
|
assert_eq!(index.list_scope_uris(source), vec![uri.to_string()]);
|
||||||
|
|
||||||
|
index
|
||||||
|
.apply_repository_view_entries(&[RepositoryViewEntry {
|
||||||
|
rsync_uri: uri.to_string(),
|
||||||
|
current_hash: Some(hash.to_string()),
|
||||||
|
repository_source: Some(source.to_string()),
|
||||||
|
object_type: Some("roa".to_string()),
|
||||||
|
state: RepositoryViewState::Withdrawn,
|
||||||
|
}])
|
||||||
|
.expect("apply withdrawn");
|
||||||
|
assert!(index.get_by_uri(uri).is_none());
|
||||||
|
assert!(index.list_scope_uris(source).is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn current_repo_index_moves_uri_between_scopes() {
|
||||||
|
let mut index = CurrentRepoIndex::new();
|
||||||
|
let uri = "rsync://example.test/repo/a.roa";
|
||||||
|
let old_scope = "rsync://example.test/repo/";
|
||||||
|
let new_scope = "https://rrdp.example.test/notification.xml";
|
||||||
|
index
|
||||||
|
.apply_repository_view_entries(&[present(old_scope, uri, &"22".repeat(32))])
|
||||||
|
.expect("apply old scope");
|
||||||
|
index
|
||||||
|
.apply_repository_view_entries(&[present(new_scope, uri, &"33".repeat(32))])
|
||||||
|
.expect("apply new scope");
|
||||||
|
|
||||||
|
assert!(index.list_scope_uris(old_scope).is_empty());
|
||||||
|
assert_eq!(index.list_scope_uris(new_scope), vec![uri.to_string()]);
|
||||||
|
assert_eq!(
|
||||||
|
index.get_by_uri(uri).expect("entry").current_hash_hex,
|
||||||
|
"33".repeat(32)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn current_repo_index_snapshot_objects_and_counts_are_sorted() {
|
||||||
|
let handle = CurrentRepoIndex::shared();
|
||||||
|
let mut index = handle.lock().expect("lock index");
|
||||||
|
index
|
||||||
|
.apply_repository_view_entries(&[
|
||||||
|
present("rsync://example.test/repo-b/", "rsync://example.test/repo-b/b.roa", &"22".repeat(32)),
|
||||||
|
present("rsync://example.test/repo-a/", "rsync://example.test/repo-a/a.roa", &"11".repeat(32)),
|
||||||
|
])
|
||||||
|
.expect("apply present entries");
|
||||||
|
assert_eq!(index.active_uri_count(), 2);
|
||||||
|
assert_eq!(index.scope_count(), 2);
|
||||||
|
|
||||||
|
let snapshot = index.snapshot_objects();
|
||||||
|
assert_eq!(snapshot.len(), 2);
|
||||||
|
assert_eq!(snapshot[0].rsync_uri, "rsync://example.test/repo-a/a.roa");
|
||||||
|
assert_eq!(snapshot[1].rsync_uri, "rsync://example.test/repo-b/b.roa");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn current_repo_index_reports_missing_fields_and_invalid_hash() {
|
||||||
|
let mut index = CurrentRepoIndex::new();
|
||||||
|
let err = index
|
||||||
|
.apply_repository_view_entries(&[RepositoryViewEntry {
|
||||||
|
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
|
||||||
|
current_hash: Some("11".repeat(32)),
|
||||||
|
repository_source: None,
|
||||||
|
object_type: Some("roa".to_string()),
|
||||||
|
state: RepositoryViewState::Present,
|
||||||
|
}])
|
||||||
|
.expect_err("missing source should fail");
|
||||||
|
assert!(err.contains("missing repository_source"), "{err}");
|
||||||
|
|
||||||
|
let err = index
|
||||||
|
.apply_repository_view_entries(&[RepositoryViewEntry {
|
||||||
|
rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
|
||||||
|
current_hash: Some("not-a-valid-sha256".to_string()),
|
||||||
|
repository_source: Some("rsync://example.test/repo/".to_string()),
|
||||||
|
object_type: Some("roa".to_string()),
|
||||||
|
state: RepositoryViewState::Present,
|
||||||
|
}])
|
||||||
|
.expect_err("invalid hash should fail");
|
||||||
|
assert!(err.contains("invalid"), "{err}");
|
||||||
|
|
||||||
|
let err = index
|
||||||
|
.apply_repository_view_entries(&[RepositoryViewEntry {
|
||||||
|
rsync_uri: "rsync://example.test/repo/b.roa".to_string(),
|
||||||
|
current_hash: Some("22".repeat(32)),
|
||||||
|
repository_source: Some("rsync://example.test/repo/".to_string()),
|
||||||
|
object_type: Some("roa".to_string()),
|
||||||
|
state: RepositoryViewState::Present,
|
||||||
|
}])
|
||||||
|
.expect("valid entry");
|
||||||
|
let got = index.get_by_uri("rsync://example.test/repo/b.roa").unwrap();
|
||||||
|
assert_eq!(got.current_hash_hex, "22".repeat(32));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn current_repo_index_withdraw_unknown_uri_is_noop() {
|
||||||
|
let mut index = CurrentRepoIndex::new();
|
||||||
|
index
|
||||||
|
.apply_repository_view_entries(&[RepositoryViewEntry {
|
||||||
|
rsync_uri: "rsync://example.test/repo/missing.roa".to_string(),
|
||||||
|
current_hash: None,
|
||||||
|
repository_source: Some("rsync://example.test/repo/".to_string()),
|
||||||
|
object_type: Some("roa".to_string()),
|
||||||
|
state: RepositoryViewState::Withdrawn,
|
||||||
|
}])
|
||||||
|
.expect("withdraw unknown should not fail");
|
||||||
|
assert_eq!(index.active_uri_count(), 0);
|
||||||
|
assert_eq!(index.scope_count(), 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -392,6 +392,59 @@ mod tests {
|
|||||||
assert!(err.contains("http read body failed"), "{err}");
|
assert!(err.contains("http read body failed"), "{err}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fetch_to_writer_streams_body_on_success() {
|
||||||
|
let url = spawn_one_shot_http_server("HTTP/1.1 200 OK", b"writer-body");
|
||||||
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
||||||
|
timeout: Duration::from_secs(2),
|
||||||
|
..HttpFetcherConfig::default()
|
||||||
|
})
|
||||||
|
.expect("http");
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let bytes = http.fetch_to_writer(&url, &mut out).expect("stream");
|
||||||
|
assert_eq!(bytes, 11);
|
||||||
|
assert_eq!(out, b"writer-body");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fetch_to_writer_rejects_non_success_status() {
|
||||||
|
let url = spawn_one_shot_http_server("HTTP/1.1 500 Internal Server Error", b"boom");
|
||||||
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
||||||
|
timeout: Duration::from_secs(2),
|
||||||
|
..HttpFetcherConfig::default()
|
||||||
|
})
|
||||||
|
.expect("http");
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let err = http.fetch_to_writer(&url, &mut out).unwrap_err();
|
||||||
|
assert!(err.contains("http status"), "{err}");
|
||||||
|
assert!(out.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fetch_to_writer_times_out_on_idle_stream_read() {
|
||||||
|
let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind");
|
||||||
|
let addr = listener.local_addr().expect("addr");
|
||||||
|
thread::spawn(move || {
|
||||||
|
let (mut stream, _) = listener.accept().expect("accept");
|
||||||
|
let mut buf = [0u8; 1024];
|
||||||
|
let _ = stream.read(&mut buf);
|
||||||
|
stream
|
||||||
|
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nh")
|
||||||
|
.expect("write partial body");
|
||||||
|
std::thread::sleep(StdDuration::from_secs(2));
|
||||||
|
let _ = stream.write_all(b"ello");
|
||||||
|
});
|
||||||
|
let url = format!("http://{}/", addr);
|
||||||
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
||||||
|
timeout: Duration::from_secs(1),
|
||||||
|
..HttpFetcherConfig::default()
|
||||||
|
})
|
||||||
|
.expect("http");
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let err = http.fetch_to_writer(&url, &mut out).unwrap_err();
|
||||||
|
assert!(err.contains("http stream body failed"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn uses_large_body_timeout_selects_rrdp_snapshot_and_delta_not_notification() {
|
fn uses_large_body_timeout_selects_rrdp_snapshot_and_delta_not_notification() {
|
||||||
assert!(!uses_large_body_timeout(
|
assert!(!uses_large_body_timeout(
|
||||||
@ -407,4 +460,23 @@ mod tests {
|
|||||||
"https://tal.example.test/example.tal"
|
"https://tal.example.test/example.tal"
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn client_for_uri_selects_expected_timeout_profile() {
|
||||||
|
let http = BlockingHttpFetcher::new(HttpFetcherConfig {
|
||||||
|
timeout: Duration::from_secs(3),
|
||||||
|
large_body_timeout: Duration::from_secs(9),
|
||||||
|
..HttpFetcherConfig::default()
|
||||||
|
})
|
||||||
|
.expect("http");
|
||||||
|
|
||||||
|
let (_, profile_short, timeout_short) = http.client_for_uri("https://example.test/root.tal");
|
||||||
|
assert_eq!(profile_short, "short");
|
||||||
|
assert_eq!(timeout_short, Duration::from_secs(3));
|
||||||
|
|
||||||
|
let (_, profile_large, timeout_large) =
|
||||||
|
http.client_for_uri("https://rrdp.example.test/session/1/snapshot.xml");
|
||||||
|
assert_eq!(profile_large, "large_body");
|
||||||
|
assert_eq!(timeout_large, Duration::from_secs(9));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -17,6 +17,8 @@ pub mod bundle;
|
|||||||
#[cfg(feature = "full")]
|
#[cfg(feature = "full")]
|
||||||
pub mod cli;
|
pub mod cli;
|
||||||
#[cfg(feature = "full")]
|
#[cfg(feature = "full")]
|
||||||
|
pub mod current_repo_index;
|
||||||
|
#[cfg(feature = "full")]
|
||||||
pub mod fetch;
|
pub mod fetch;
|
||||||
#[cfg(feature = "full")]
|
#[cfg(feature = "full")]
|
||||||
pub mod parallel;
|
pub mod parallel;
|
||||||
|
|||||||
@ -5,8 +5,8 @@ use crate::parallel::repo_scheduler::TransportRequestAction;
|
|||||||
use crate::parallel::repo_worker::{RepoTransportExecutor, RepoTransportWorkerPool};
|
use crate::parallel::repo_worker::{RepoTransportExecutor, RepoTransportWorkerPool};
|
||||||
use crate::parallel::run_coordinator::GlobalRunCoordinator;
|
use crate::parallel::run_coordinator::GlobalRunCoordinator;
|
||||||
use crate::parallel::types::{
|
use crate::parallel::types::{
|
||||||
RepoIdentity, RepoRuntimeState, RepoTransportMode, RepoTransportResultEnvelope,
|
RepoIdentity, RepoRequester, RepoRuntimeState, RepoTransportMode, RepoTransportResultEnvelope,
|
||||||
RepoTransportResultKind, RepoRequester,
|
RepoTransportResultKind,
|
||||||
};
|
};
|
||||||
use crate::policy::SyncPreference;
|
use crate::policy::SyncPreference;
|
||||||
use crate::report::Warning;
|
use crate::report::Warning;
|
||||||
@ -160,7 +160,8 @@ impl<E: RepoTransportExecutor> Phase1RepoSyncRuntime<E> {
|
|||||||
};
|
};
|
||||||
{
|
{
|
||||||
let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
|
let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
|
||||||
coordinator.mark_transport_running(&task.dedup_key, time::OffsetDateTime::now_utc())?;
|
coordinator
|
||||||
|
.mark_transport_running(&task.dedup_key, time::OffsetDateTime::now_utc())?;
|
||||||
}
|
}
|
||||||
crate::progress_log::emit(
|
crate::progress_log::emit(
|
||||||
"phase1_repo_task_dispatched",
|
"phase1_repo_task_dispatched",
|
||||||
@ -233,7 +234,9 @@ impl<E: RepoTransportExecutor> Phase1RepoSyncRuntime<E> {
|
|||||||
|
|
||||||
fn runtime_state_for_identity(&self, identity: &RepoIdentity) -> Option<RepoRuntimeState> {
|
fn runtime_state_for_identity(&self, identity: &RepoIdentity) -> Option<RepoRuntimeState> {
|
||||||
let coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
|
let coordinator = self.coordinator.lock().expect("coordinator lock poisoned");
|
||||||
coordinator.runtime_record(identity).map(|record| record.state)
|
coordinator
|
||||||
|
.runtime_record(identity)
|
||||||
|
.map(|record| record.state)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn resolved_outcome_for_identity(
|
fn resolved_outcome_for_identity(
|
||||||
@ -313,8 +316,10 @@ fn outcome_from_transport_result(
|
|||||||
warnings: warnings.clone(),
|
warnings: warnings.clone(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
(RepoTransportResultKind::Failed { detail, warnings }, RepoRuntimeState::FailedTerminal) => {
|
(
|
||||||
RepoSyncRuntimeOutcome {
|
RepoTransportResultKind::Failed { detail, warnings },
|
||||||
|
RepoRuntimeState::FailedTerminal,
|
||||||
|
) => RepoSyncRuntimeOutcome {
|
||||||
repo_sync_ok: false,
|
repo_sync_ok: false,
|
||||||
repo_sync_err: Some(detail.clone()),
|
repo_sync_err: Some(detail.clone()),
|
||||||
repo_sync_source: None,
|
repo_sync_source: None,
|
||||||
@ -325,8 +330,7 @@ fn outcome_from_transport_result(
|
|||||||
}),
|
}),
|
||||||
repo_sync_duration_ms: envelope.timing_ms,
|
repo_sync_duration_ms: envelope.timing_ms,
|
||||||
warnings: warnings.clone(),
|
warnings: warnings.clone(),
|
||||||
}
|
},
|
||||||
}
|
|
||||||
_ => RepoSyncRuntimeOutcome {
|
_ => RepoSyncRuntimeOutcome {
|
||||||
repo_sync_ok: false,
|
repo_sync_ok: false,
|
||||||
repo_sync_err: Some("repo runtime state unresolved".to_string()),
|
repo_sync_err: Some("repo runtime state unresolved".to_string()),
|
||||||
@ -344,7 +348,6 @@ mod tests {
|
|||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use crate::policy::SyncPreference;
|
|
||||||
use crate::parallel::config::ParallelPhase1Config;
|
use crate::parallel::config::ParallelPhase1Config;
|
||||||
use crate::parallel::repo_runtime::{Phase1RepoSyncRuntime, RepoSyncRuntime};
|
use crate::parallel::repo_runtime::{Phase1RepoSyncRuntime, RepoSyncRuntime};
|
||||||
use crate::parallel::repo_worker::{
|
use crate::parallel::repo_worker::{
|
||||||
@ -352,9 +355,10 @@ mod tests {
|
|||||||
};
|
};
|
||||||
use crate::parallel::run_coordinator::GlobalRunCoordinator;
|
use crate::parallel::run_coordinator::GlobalRunCoordinator;
|
||||||
use crate::parallel::types::{
|
use crate::parallel::types::{
|
||||||
RepoTransportMode, RepoTransportResultEnvelope, RepoTransportResultKind,
|
RepoTransportMode, RepoTransportResultEnvelope, RepoTransportResultKind, RepoTransportTask,
|
||||||
RepoTransportTask, TalInputSpec,
|
TalInputSpec,
|
||||||
};
|
};
|
||||||
|
use crate::policy::SyncPreference;
|
||||||
use crate::report::Warning;
|
use crate::report::Warning;
|
||||||
use crate::validation::tree::{CaInstanceHandle, DiscoveredChildCaInstance};
|
use crate::validation::tree::{CaInstanceHandle, DiscoveredChildCaInstance};
|
||||||
|
|
||||||
@ -492,7 +496,10 @@ mod tests {
|
|||||||
.expect("sync repo");
|
.expect("sync repo");
|
||||||
assert!(outcome.repo_sync_ok);
|
assert!(outcome.repo_sync_ok);
|
||||||
assert_eq!(outcome.repo_sync_source.as_deref(), Some("rsync"));
|
assert_eq!(outcome.repo_sync_source.as_deref(), Some("rsync"));
|
||||||
assert_eq!(outcome.repo_sync_phase.as_deref(), Some("rrdp_failed_rsync_ok"));
|
assert_eq!(
|
||||||
|
outcome.repo_sync_phase.as_deref(),
|
||||||
|
Some("rrdp_failed_rsync_ok")
|
||||||
|
);
|
||||||
assert_eq!(rrdp_count.load(Ordering::SeqCst), 1);
|
assert_eq!(rrdp_count.load(Ordering::SeqCst), 1);
|
||||||
assert_eq!(rsync_count.load(Ordering::SeqCst), 1);
|
assert_eq!(rsync_count.load(Ordering::SeqCst), 1);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,11 +1,11 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
||||||
use crate::policy::SyncPreference;
|
|
||||||
use crate::parallel::types::{
|
use crate::parallel::types::{
|
||||||
InFlightRepoEntry, RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoRuntimeState,
|
InFlightRepoEntry, RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoRuntimeState,
|
||||||
RepoSyncResultEnvelope, RepoSyncResultKind, RepoSyncResultRef, RepoSyncTask, RepoTaskState,
|
RepoSyncResultEnvelope, RepoSyncResultKind, RepoSyncResultRef, RepoSyncTask, RepoTaskState,
|
||||||
RepoTransportMode, RepoTransportResultEnvelope, RepoTransportResultKind, RepoTransportTask,
|
RepoTransportMode, RepoTransportResultEnvelope, RepoTransportResultKind, RepoTransportTask,
|
||||||
};
|
};
|
||||||
|
use crate::policy::SyncPreference;
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
pub enum RepoRequestAction {
|
pub enum RepoRequestAction {
|
||||||
@ -241,7 +241,13 @@ impl TransportStateTables {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
self.register_rsync_request(identity, requester, validation_time, priority, rsync_scope_uri)
|
self.register_rsync_request(
|
||||||
|
identity,
|
||||||
|
requester,
|
||||||
|
validation_time,
|
||||||
|
priority,
|
||||||
|
rsync_scope_uri,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn register_rsync_request(
|
fn register_rsync_request(
|
||||||
@ -384,7 +390,10 @@ impl TransportStateTables {
|
|||||||
finished_at: time::OffsetDateTime,
|
finished_at: time::OffsetDateTime,
|
||||||
) -> Result<TransportCompletion, String> {
|
) -> Result<TransportCompletion, String> {
|
||||||
match (&result.dedup_key, &result.result) {
|
match (&result.dedup_key, &result.result) {
|
||||||
(RepoDedupKey::RrdpNotify { notification_uri }, RepoTransportResultKind::Success { .. }) => {
|
(
|
||||||
|
RepoDedupKey::RrdpNotify { notification_uri },
|
||||||
|
RepoTransportResultKind::Success { .. },
|
||||||
|
) => {
|
||||||
let entry = self
|
let entry = self
|
||||||
.rrdp_inflight
|
.rrdp_inflight
|
||||||
.get_mut(notification_uri)
|
.get_mut(notification_uri)
|
||||||
@ -406,7 +415,10 @@ impl TransportStateTables {
|
|||||||
follow_up_tasks: Vec::new(),
|
follow_up_tasks: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
(RepoDedupKey::RrdpNotify { notification_uri }, RepoTransportResultKind::Failed { .. }) => {
|
(
|
||||||
|
RepoDedupKey::RrdpNotify { notification_uri },
|
||||||
|
RepoTransportResultKind::Failed { .. },
|
||||||
|
) => {
|
||||||
let entry = self
|
let entry = self
|
||||||
.rrdp_inflight
|
.rrdp_inflight
|
||||||
.get_mut(notification_uri)
|
.get_mut(notification_uri)
|
||||||
@ -422,7 +434,9 @@ impl TransportStateTables {
|
|||||||
record.state = RepoRuntimeState::RrdpFailedPendingRsync;
|
record.state = RepoRuntimeState::RrdpFailedPendingRsync;
|
||||||
let rsync_scope_uri = record.rsync_scope_key.clone();
|
let rsync_scope_uri = record.rsync_scope_key.clone();
|
||||||
if let Some(existing) = self.rsync_inflight.get_mut(&rsync_scope_uri) {
|
if let Some(existing) = self.rsync_inflight.get_mut(&rsync_scope_uri) {
|
||||||
existing.waiting_requesters.extend(record.requesters.clone());
|
existing
|
||||||
|
.waiting_requesters
|
||||||
|
.extend(record.requesters.clone());
|
||||||
record.state = RepoRuntimeState::WaitingRsync;
|
record.state = RepoRuntimeState::WaitingRsync;
|
||||||
} else {
|
} else {
|
||||||
let task = RepoTransportTask {
|
let task = RepoTransportTask {
|
||||||
@ -458,7 +472,10 @@ impl TransportStateTables {
|
|||||||
follow_up_tasks,
|
follow_up_tasks,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
(RepoDedupKey::RsyncScope { rsync_scope_uri }, RepoTransportResultKind::Success { .. }) => {
|
(
|
||||||
|
RepoDedupKey::RsyncScope { rsync_scope_uri },
|
||||||
|
RepoTransportResultKind::Success { .. },
|
||||||
|
) => {
|
||||||
let entry = self
|
let entry = self
|
||||||
.rsync_inflight
|
.rsync_inflight
|
||||||
.get_mut(rsync_scope_uri)
|
.get_mut(rsync_scope_uri)
|
||||||
@ -471,7 +488,8 @@ impl TransportStateTables {
|
|||||||
if record.rsync_scope_key == *rsync_scope_uri
|
if record.rsync_scope_key == *rsync_scope_uri
|
||||||
&& matches!(
|
&& matches!(
|
||||||
record.state,
|
record.state,
|
||||||
RepoRuntimeState::WaitingRsync | RepoRuntimeState::RrdpFailedPendingRsync
|
RepoRuntimeState::WaitingRsync
|
||||||
|
| RepoRuntimeState::RrdpFailedPendingRsync
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
record.state = RepoRuntimeState::RsyncOk;
|
record.state = RepoRuntimeState::RsyncOk;
|
||||||
@ -483,7 +501,10 @@ impl TransportStateTables {
|
|||||||
follow_up_tasks: Vec::new(),
|
follow_up_tasks: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
(RepoDedupKey::RsyncScope { rsync_scope_uri }, RepoTransportResultKind::Failed { .. }) => {
|
(
|
||||||
|
RepoDedupKey::RsyncScope { rsync_scope_uri },
|
||||||
|
RepoTransportResultKind::Failed { .. },
|
||||||
|
) => {
|
||||||
let entry = self
|
let entry = self
|
||||||
.rsync_inflight
|
.rsync_inflight
|
||||||
.get_mut(rsync_scope_uri)
|
.get_mut(rsync_scope_uri)
|
||||||
@ -496,7 +517,8 @@ impl TransportStateTables {
|
|||||||
if record.rsync_scope_key == *rsync_scope_uri
|
if record.rsync_scope_key == *rsync_scope_uri
|
||||||
&& matches!(
|
&& matches!(
|
||||||
record.state,
|
record.state,
|
||||||
RepoRuntimeState::WaitingRsync | RepoRuntimeState::RrdpFailedPendingRsync
|
RepoRuntimeState::WaitingRsync
|
||||||
|
| RepoRuntimeState::RrdpFailedPendingRsync
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
record.state = RepoRuntimeState::FailedTerminal;
|
record.state = RepoRuntimeState::FailedTerminal;
|
||||||
@ -512,7 +534,6 @@ impl TransportStateTables {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
pub struct InFlightRepoTable {
|
pub struct InFlightRepoTable {
|
||||||
entries: HashMap<RepoKey, InFlightRepoEntry>,
|
entries: HashMap<RepoKey, InFlightRepoEntry>,
|
||||||
@ -536,7 +557,9 @@ impl InFlightRepoTable {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn last_result(&self, key: &RepoKey) -> Option<&RepoSyncResultEnvelope> {
|
pub fn last_result(&self, key: &RepoKey) -> Option<&RepoSyncResultEnvelope> {
|
||||||
self.entries.get(key).and_then(|entry| entry.last_result.as_ref())
|
self.entries
|
||||||
|
.get(key)
|
||||||
|
.and_then(|entry| entry.last_result.as_ref())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn register_request(
|
pub fn register_request(
|
||||||
@ -626,7 +649,9 @@ impl InFlightRepoTable {
|
|||||||
RepoSyncResultKind::Success(result_ref)
|
RepoSyncResultKind::Success(result_ref)
|
||||||
| RepoSyncResultKind::Reused(result_ref) => result_ref.clone(),
|
| RepoSyncResultKind::Reused(result_ref) => result_ref.clone(),
|
||||||
RepoSyncResultKind::Failed { detail } => {
|
RepoSyncResultKind::Failed { detail } => {
|
||||||
return Err(format!("success completion called with failure result: {detail}"));
|
return Err(format!(
|
||||||
|
"success completion called with failure result: {detail}"
|
||||||
|
));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
entry.state = RepoTaskState::Succeeded;
|
entry.state = RepoTaskState::Succeeded;
|
||||||
@ -912,12 +937,14 @@ mod tests {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod transport_tests {
|
mod transport_tests {
|
||||||
use crate::policy::SyncPreference;
|
use crate::parallel::repo_scheduler::{
|
||||||
use crate::parallel::repo_scheduler::{TransportRequestAction, TransportStateTables, TransportTaskState};
|
TransportRequestAction, TransportStateTables, TransportTaskState,
|
||||||
|
};
|
||||||
use crate::parallel::types::{
|
use crate::parallel::types::{
|
||||||
RepoDedupKey, RepoIdentity, RepoRequester, RepoRuntimeState, RepoTransportMode,
|
RepoDedupKey, RepoIdentity, RepoRequester, RepoRuntimeState, RepoTransportMode,
|
||||||
RepoTransportResultEnvelope, RepoTransportResultKind,
|
RepoTransportResultEnvelope, RepoTransportResultKind,
|
||||||
};
|
};
|
||||||
|
use crate::policy::SyncPreference;
|
||||||
|
|
||||||
fn requester(id: &str) -> RepoRequester {
|
fn requester(id: &str) -> RepoRequester {
|
||||||
RepoRequester::with_tal_rir(
|
RepoRequester::with_tal_rir(
|
||||||
@ -956,7 +983,10 @@ mod transport_tests {
|
|||||||
}
|
}
|
||||||
);
|
);
|
||||||
let key = "https://example.test/notify.xml".to_string();
|
let key = "https://example.test/notify.xml".to_string();
|
||||||
assert_eq!(tables.rrdp_inflight.get(&key).unwrap().state, TransportTaskState::Pending);
|
assert_eq!(
|
||||||
|
tables.rrdp_inflight.get(&key).unwrap().state,
|
||||||
|
TransportTaskState::Pending
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
tables.runtime_records.get(&identity).unwrap().state,
|
tables.runtime_records.get(&identity).unwrap().state,
|
||||||
RepoRuntimeState::WaitingRrdp
|
RepoRuntimeState::WaitingRrdp
|
||||||
@ -1171,7 +1201,10 @@ mod transport_tests {
|
|||||||
"rsync://example.test/module/".to_string(),
|
"rsync://example.test/module/".to_string(),
|
||||||
SyncPreference::RsyncOnly,
|
SyncPreference::RsyncOnly,
|
||||||
);
|
);
|
||||||
assert!(matches!(action, TransportRequestAction::ReusedTerminalFailure(_)));
|
assert!(matches!(
|
||||||
|
action,
|
||||||
|
TransportRequestAction::ReusedTerminalFailure(_)
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@ -10,10 +10,13 @@ use super::types::{
|
|||||||
};
|
};
|
||||||
use crate::analysis::timing::TimingHandle;
|
use crate::analysis::timing::TimingHandle;
|
||||||
use crate::audit_downloads::DownloadLogHandle;
|
use crate::audit_downloads::DownloadLogHandle;
|
||||||
|
use crate::current_repo_index::CurrentRepoIndexHandle;
|
||||||
use crate::fetch::rsync::RsyncFetcher;
|
use crate::fetch::rsync::RsyncFetcher;
|
||||||
use crate::policy::Policy;
|
use crate::policy::Policy;
|
||||||
use crate::storage::RocksStore;
|
use crate::storage::RocksStore;
|
||||||
use crate::sync::repo::{run_rrdp_transport, run_rsync_transport, sync_publication_point, RepoSyncSource};
|
use crate::sync::repo::{
|
||||||
|
RepoSyncSource, run_rrdp_transport, run_rsync_transport, sync_publication_point,
|
||||||
|
};
|
||||||
use crate::sync::rrdp::Fetcher;
|
use crate::sync::rrdp::Fetcher;
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
@ -39,6 +42,7 @@ pub trait RepoTransportExecutor: Send + Sync + 'static {
|
|||||||
|
|
||||||
pub struct LiveRrdpTransportExecutor<H: Fetcher> {
|
pub struct LiveRrdpTransportExecutor<H: Fetcher> {
|
||||||
store: Arc<RocksStore>,
|
store: Arc<RocksStore>,
|
||||||
|
current_repo_index: CurrentRepoIndexHandle,
|
||||||
http_fetcher: Arc<H>,
|
http_fetcher: Arc<H>,
|
||||||
timing: Option<TimingHandle>,
|
timing: Option<TimingHandle>,
|
||||||
download_log: Option<DownloadLogHandle>,
|
download_log: Option<DownloadLogHandle>,
|
||||||
@ -47,12 +51,14 @@ pub struct LiveRrdpTransportExecutor<H: Fetcher> {
|
|||||||
impl<H: Fetcher> LiveRrdpTransportExecutor<H> {
|
impl<H: Fetcher> LiveRrdpTransportExecutor<H> {
|
||||||
pub fn new(
|
pub fn new(
|
||||||
store: Arc<RocksStore>,
|
store: Arc<RocksStore>,
|
||||||
|
current_repo_index: CurrentRepoIndexHandle,
|
||||||
http_fetcher: Arc<H>,
|
http_fetcher: Arc<H>,
|
||||||
timing: Option<TimingHandle>,
|
timing: Option<TimingHandle>,
|
||||||
download_log: Option<DownloadLogHandle>,
|
download_log: Option<DownloadLogHandle>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
store,
|
store,
|
||||||
|
current_repo_index,
|
||||||
http_fetcher,
|
http_fetcher,
|
||||||
timing,
|
timing,
|
||||||
download_log,
|
download_log,
|
||||||
@ -72,6 +78,7 @@ impl<H: Fetcher + 'static> RepoTransportExecutor for LiveRrdpTransportExecutor<H
|
|||||||
match run_rrdp_transport(
|
match run_rrdp_transport(
|
||||||
self.store.as_ref(),
|
self.store.as_ref(),
|
||||||
notification_uri,
|
notification_uri,
|
||||||
|
Some(&self.current_repo_index),
|
||||||
self.http_fetcher.as_ref(),
|
self.http_fetcher.as_ref(),
|
||||||
self.timing.as_ref(),
|
self.timing.as_ref(),
|
||||||
self.download_log.as_ref(),
|
self.download_log.as_ref(),
|
||||||
@ -106,6 +113,7 @@ impl<H: Fetcher + 'static> RepoTransportExecutor for LiveRrdpTransportExecutor<H
|
|||||||
|
|
||||||
pub struct LiveRsyncTransportExecutor<R: RsyncFetcher> {
|
pub struct LiveRsyncTransportExecutor<R: RsyncFetcher> {
|
||||||
store: Arc<RocksStore>,
|
store: Arc<RocksStore>,
|
||||||
|
current_repo_index: CurrentRepoIndexHandle,
|
||||||
rsync_fetcher: Arc<R>,
|
rsync_fetcher: Arc<R>,
|
||||||
timing: Option<TimingHandle>,
|
timing: Option<TimingHandle>,
|
||||||
download_log: Option<DownloadLogHandle>,
|
download_log: Option<DownloadLogHandle>,
|
||||||
@ -114,12 +122,14 @@ pub struct LiveRsyncTransportExecutor<R: RsyncFetcher> {
|
|||||||
impl<R: RsyncFetcher> LiveRsyncTransportExecutor<R> {
|
impl<R: RsyncFetcher> LiveRsyncTransportExecutor<R> {
|
||||||
pub fn new(
|
pub fn new(
|
||||||
store: Arc<RocksStore>,
|
store: Arc<RocksStore>,
|
||||||
|
current_repo_index: CurrentRepoIndexHandle,
|
||||||
rsync_fetcher: Arc<R>,
|
rsync_fetcher: Arc<R>,
|
||||||
timing: Option<TimingHandle>,
|
timing: Option<TimingHandle>,
|
||||||
download_log: Option<DownloadLogHandle>,
|
download_log: Option<DownloadLogHandle>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
store,
|
store,
|
||||||
|
current_repo_index,
|
||||||
rsync_fetcher,
|
rsync_fetcher,
|
||||||
timing,
|
timing,
|
||||||
download_log,
|
download_log,
|
||||||
@ -134,6 +144,7 @@ impl<R: RsyncFetcher + 'static> RepoTransportExecutor for LiveRsyncTransportExec
|
|||||||
match run_rsync_transport(
|
match run_rsync_transport(
|
||||||
self.store.as_ref(),
|
self.store.as_ref(),
|
||||||
&task.repo_identity.rsync_base_uri,
|
&task.repo_identity.rsync_base_uri,
|
||||||
|
Some(&self.current_repo_index),
|
||||||
self.rsync_fetcher.as_ref(),
|
self.rsync_fetcher.as_ref(),
|
||||||
self.timing.as_ref(),
|
self.timing.as_ref(),
|
||||||
self.download_log.as_ref(),
|
self.download_log.as_ref(),
|
||||||
@ -174,6 +185,7 @@ pub struct LiveRepoTransportExecutor<H: Fetcher, R: RsyncFetcher> {
|
|||||||
impl<H: Fetcher, R: RsyncFetcher> LiveRepoTransportExecutor<H, R> {
|
impl<H: Fetcher, R: RsyncFetcher> LiveRepoTransportExecutor<H, R> {
|
||||||
pub fn new(
|
pub fn new(
|
||||||
store: Arc<RocksStore>,
|
store: Arc<RocksStore>,
|
||||||
|
current_repo_index: CurrentRepoIndexHandle,
|
||||||
http_fetcher: Arc<H>,
|
http_fetcher: Arc<H>,
|
||||||
rsync_fetcher: Arc<R>,
|
rsync_fetcher: Arc<R>,
|
||||||
timing: Option<TimingHandle>,
|
timing: Option<TimingHandle>,
|
||||||
@ -182,11 +194,18 @@ impl<H: Fetcher, R: RsyncFetcher> LiveRepoTransportExecutor<H, R> {
|
|||||||
Self {
|
Self {
|
||||||
rrdp: LiveRrdpTransportExecutor::new(
|
rrdp: LiveRrdpTransportExecutor::new(
|
||||||
Arc::clone(&store),
|
Arc::clone(&store),
|
||||||
|
current_repo_index.clone(),
|
||||||
http_fetcher,
|
http_fetcher,
|
||||||
timing.clone(),
|
timing.clone(),
|
||||||
download_log.clone(),
|
download_log.clone(),
|
||||||
),
|
),
|
||||||
rsync: LiveRsyncTransportExecutor::new(store, rsync_fetcher, timing, download_log),
|
rsync: LiveRsyncTransportExecutor::new(
|
||||||
|
store,
|
||||||
|
current_repo_index,
|
||||||
|
rsync_fetcher,
|
||||||
|
timing,
|
||||||
|
download_log,
|
||||||
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -231,7 +250,9 @@ impl<H: Fetcher, R: RsyncFetcher> LiveRepoSyncExecutor<H, R> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<H: Fetcher + 'static, R: RsyncFetcher + 'static> RepoSyncExecutor for LiveRepoSyncExecutor<H, R> {
|
impl<H: Fetcher + 'static, R: RsyncFetcher + 'static> RepoSyncExecutor
|
||||||
|
for LiveRepoSyncExecutor<H, R>
|
||||||
|
{
|
||||||
fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope {
|
fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope {
|
||||||
let started = std::time::Instant::now();
|
let started = std::time::Instant::now();
|
||||||
crate::progress_log::emit(
|
crate::progress_log::emit(
|
||||||
@ -269,10 +290,12 @@ impl<H: Fetcher + 'static, R: RsyncFetcher + 'static> RepoSyncExecutor for LiveR
|
|||||||
repo_key: task.repo_key.clone(),
|
repo_key: task.repo_key.clone(),
|
||||||
tal_id: task.tal_id,
|
tal_id: task.tal_id,
|
||||||
rir_id: task.rir_id,
|
rir_id: task.rir_id,
|
||||||
result: super::types::RepoSyncResultKind::Success(super::types::RepoSyncResultRef {
|
result: super::types::RepoSyncResultKind::Success(
|
||||||
|
super::types::RepoSyncResultRef {
|
||||||
repo_key: task.repo_key,
|
repo_key: task.repo_key,
|
||||||
source: repo_sync_source_label(res.source).to_string(),
|
source: repo_sync_source_label(res.source).to_string(),
|
||||||
}),
|
},
|
||||||
|
),
|
||||||
phase: Some(repo_sync_phase_label(res.phase).to_string()),
|
phase: Some(repo_sync_phase_label(res.phase).to_string()),
|
||||||
timing_ms,
|
timing_ms,
|
||||||
warnings: res.warnings,
|
warnings: res.warnings,
|
||||||
@ -595,15 +618,16 @@ mod tests {
|
|||||||
RepoSyncExecutor, RepoTransportExecutor, RepoTransportWorkerPool, RepoWorkerPool,
|
RepoSyncExecutor, RepoTransportExecutor, RepoTransportWorkerPool, RepoWorkerPool,
|
||||||
RepoWorkerPoolConfig,
|
RepoWorkerPoolConfig,
|
||||||
};
|
};
|
||||||
|
use crate::current_repo_index::CurrentRepoIndex;
|
||||||
|
use crate::fetch::rsync::{
|
||||||
|
LocalDirRsyncFetcher, RsyncFetchError, RsyncFetchResult, RsyncFetcher,
|
||||||
|
};
|
||||||
use crate::parallel::config::ParallelPhase1Config;
|
use crate::parallel::config::ParallelPhase1Config;
|
||||||
use crate::parallel::types::{
|
use crate::parallel::types::{
|
||||||
RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoSyncResultEnvelope,
|
RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoSyncResultEnvelope,
|
||||||
RepoSyncResultKind, RepoSyncResultRef, RepoSyncTask, RepoTransportMode,
|
RepoSyncResultKind, RepoSyncResultRef, RepoSyncTask, RepoTransportMode,
|
||||||
RepoTransportResultEnvelope, RepoTransportResultKind, RepoTransportTask,
|
RepoTransportResultEnvelope, RepoTransportResultKind, RepoTransportTask,
|
||||||
};
|
};
|
||||||
use crate::fetch::rsync::{
|
|
||||||
LocalDirRsyncFetcher, RsyncFetchError, RsyncFetchResult, RsyncFetcher,
|
|
||||||
};
|
|
||||||
use crate::policy::SyncPreference;
|
use crate::policy::SyncPreference;
|
||||||
use crate::report::Warning;
|
use crate::report::Warning;
|
||||||
use crate::storage::RocksStore;
|
use crate::storage::RocksStore;
|
||||||
@ -628,7 +652,10 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sample_rrdp_transport_task(notification_uri: &str, rsync_base_uri: &str) -> RepoTransportTask {
|
fn sample_rrdp_transport_task(
|
||||||
|
notification_uri: &str,
|
||||||
|
rsync_base_uri: &str,
|
||||||
|
) -> RepoTransportTask {
|
||||||
RepoTransportTask {
|
RepoTransportTask {
|
||||||
dedup_key: RepoDedupKey::RrdpNotify {
|
dedup_key: RepoDedupKey::RrdpNotify {
|
||||||
notification_uri: notification_uri.to_string(),
|
notification_uri: notification_uri.to_string(),
|
||||||
@ -649,7 +676,10 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sample_rsync_transport_task(rsync_scope_uri: &str, rsync_base_uri: &str) -> RepoTransportTask {
|
fn sample_rsync_transport_task(
|
||||||
|
rsync_scope_uri: &str,
|
||||||
|
rsync_base_uri: &str,
|
||||||
|
) -> RepoTransportTask {
|
||||||
RepoTransportTask {
|
RepoTransportTask {
|
||||||
dedup_key: RepoDedupKey::RsyncScope {
|
dedup_key: RepoDedupKey::RsyncScope {
|
||||||
rsync_scope_uri: rsync_scope_uri.to_string(),
|
rsync_scope_uri: rsync_scope_uri.to_string(),
|
||||||
@ -774,7 +804,10 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn rrdp_notification_xml(session_id: &str, serial: u64, snapshot_uri: &str) -> String {
|
fn rrdp_notification_xml(session_id: &str, serial: u64, snapshot_uri: &str) -> String {
|
||||||
let snapshot_body = rrdp_snapshot_xml(session_id, &[("rsync://example.test/repo/a.roa", b"a".as_ref())]);
|
let snapshot_body = rrdp_snapshot_xml(
|
||||||
|
session_id,
|
||||||
|
&[("rsync://example.test/repo/a.roa", b"a".as_ref())],
|
||||||
|
);
|
||||||
let snapshot_hash = sha256_hex(snapshot_body.as_bytes());
|
let snapshot_hash = sha256_hex(snapshot_body.as_bytes());
|
||||||
format!(
|
format!(
|
||||||
r#"<?xml version="1.0" encoding="UTF-8"?>
|
r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||||
@ -785,13 +818,11 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn rrdp_snapshot_xml(session_id: &str, objects: &[(&str, &[u8])]) -> String {
|
fn rrdp_snapshot_xml(session_id: &str, objects: &[(&str, &[u8])]) -> String {
|
||||||
let mut body = String::from(
|
let mut body = String::from(&format!(
|
||||||
&format!(
|
|
||||||
r#"<?xml version="1.0" encoding="UTF-8"?>
|
r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<snapshot xmlns="http://www.ripe.net/rpki/rrdp" version="1" session_id="{session_id}" serial="1">
|
<snapshot xmlns="http://www.ripe.net/rpki/rrdp" version="1" session_id="{session_id}" serial="1">
|
||||||
"#
|
"#
|
||||||
),
|
));
|
||||||
);
|
|
||||||
for (uri, bytes) in objects {
|
for (uri, bytes) in objects {
|
||||||
body.push_str(&format!(
|
body.push_str(&format!(
|
||||||
" <publish uri=\"{uri}\">{}</publish>\n",
|
" <publish uri=\"{uri}\">{}</publish>\n",
|
||||||
@ -896,9 +927,11 @@ mod tests {
|
|||||||
);
|
);
|
||||||
|
|
||||||
assert_eq!(results.len(), 2);
|
assert_eq!(results.len(), 2);
|
||||||
assert!(results
|
assert!(
|
||||||
|
results
|
||||||
.iter()
|
.iter()
|
||||||
.all(|res| matches!(res.result, RepoSyncResultKind::Success(_))));
|
.all(|res| matches!(res.result, RepoSyncResultKind::Success(_)))
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -977,8 +1010,14 @@ mod tests {
|
|||||||
.recv_result_timeout(Duration::from_secs(1))
|
.recv_result_timeout(Duration::from_secs(1))
|
||||||
.expect("recv second")
|
.expect("recv second")
|
||||||
.expect("second result");
|
.expect("second result");
|
||||||
assert!(matches!(first.result, RepoTransportResultKind::Success { .. }));
|
assert!(matches!(
|
||||||
assert!(matches!(second.result, RepoTransportResultKind::Success { .. }));
|
first.result,
|
||||||
|
RepoTransportResultKind::Success { .. }
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
second.result,
|
||||||
|
RepoTransportResultKind::Success { .. }
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -1031,10 +1070,12 @@ mod tests {
|
|||||||
result.result,
|
result.result,
|
||||||
RepoSyncResultKind::Failed { ref detail } if detail.contains("rsync fallback failed")
|
RepoSyncResultKind::Failed { ref detail } if detail.contains("rsync fallback failed")
|
||||||
));
|
));
|
||||||
assert!(store
|
assert!(
|
||||||
|
store
|
||||||
.get_repository_view_entry("rsync://example.test/repo/a.roa")
|
.get_repository_view_entry("rsync://example.test/repo/a.roa")
|
||||||
.expect("read view")
|
.expect("read view")
|
||||||
.is_none());
|
.is_none()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -1044,16 +1085,22 @@ mod tests {
|
|||||||
let http = MockHttpFetcher::new();
|
let http = MockHttpFetcher::new();
|
||||||
let notify = "https://example.test/notification.xml";
|
let notify = "https://example.test/notification.xml";
|
||||||
let snapshot = "https://example.test/snapshot.xml";
|
let snapshot = "https://example.test/snapshot.xml";
|
||||||
let snapshot_bytes = rrdp_snapshot_xml("123e4567-e89b-12d3-a456-426614174000", &[(
|
let snapshot_bytes = rrdp_snapshot_xml(
|
||||||
"rsync://example.test/repo/a.roa",
|
"123e4567-e89b-12d3-a456-426614174000",
|
||||||
b"a".as_ref(),
|
&[("rsync://example.test/repo/a.roa", b"a".as_ref())],
|
||||||
)]);
|
);
|
||||||
let notification_bytes =
|
let notification_bytes =
|
||||||
rrdp_notification_xml("123e4567-e89b-12d3-a456-426614174000", 1, snapshot);
|
rrdp_notification_xml("123e4567-e89b-12d3-a456-426614174000", 1, snapshot);
|
||||||
http.insert(notify, notification_bytes.into_bytes());
|
http.insert(notify, notification_bytes.into_bytes());
|
||||||
http.insert(snapshot, snapshot_bytes.into_bytes());
|
http.insert(snapshot, snapshot_bytes.into_bytes());
|
||||||
let executor =
|
let current_repo_index = CurrentRepoIndex::shared();
|
||||||
LiveRrdpTransportExecutor::new(Arc::clone(&store), Arc::new(http), None, None);
|
let executor = LiveRrdpTransportExecutor::new(
|
||||||
|
Arc::clone(&store),
|
||||||
|
current_repo_index.clone(),
|
||||||
|
Arc::new(http),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
let result = executor.execute_transport(sample_rrdp_transport_task(
|
let result = executor.execute_transport(sample_rrdp_transport_task(
|
||||||
notify,
|
notify,
|
||||||
"rsync://example.test/repo/",
|
"rsync://example.test/repo/",
|
||||||
@ -1062,6 +1109,16 @@ mod tests {
|
|||||||
matches!(result.result, RepoTransportResultKind::Success { .. }),
|
matches!(result.result, RepoTransportResultKind::Success { .. }),
|
||||||
"{result:?}"
|
"{result:?}"
|
||||||
);
|
);
|
||||||
|
let index = current_repo_index.lock().expect("index lock");
|
||||||
|
assert!(
|
||||||
|
index
|
||||||
|
.get_by_uri("rsync://example.test/repo/a.roa")
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
index.list_scope_uris(notify),
|
||||||
|
vec!["rsync://example.test/repo/a.roa".to_string()]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -1070,6 +1127,7 @@ mod tests {
|
|||||||
let store = Arc::new(RocksStore::open(td.path()).expect("open store"));
|
let store = Arc::new(RocksStore::open(td.path()).expect("open store"));
|
||||||
let executor = LiveRrdpTransportExecutor::new(
|
let executor = LiveRrdpTransportExecutor::new(
|
||||||
Arc::clone(&store),
|
Arc::clone(&store),
|
||||||
|
CurrentRepoIndex::shared(),
|
||||||
Arc::new(ErrorHttpFetcher),
|
Arc::new(ErrorHttpFetcher),
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
@ -1078,7 +1136,10 @@ mod tests {
|
|||||||
"https://example.test/notification.xml",
|
"https://example.test/notification.xml",
|
||||||
"rsync://example.test/repo/",
|
"rsync://example.test/repo/",
|
||||||
));
|
));
|
||||||
assert!(matches!(result.result, RepoTransportResultKind::Failed { .. }));
|
assert!(matches!(
|
||||||
|
result.result,
|
||||||
|
RepoTransportResultKind::Failed { .. }
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -1089,8 +1150,10 @@ mod tests {
|
|||||||
fs::write(td.path().join("nested").join("b.cer"), b"b").expect("write");
|
fs::write(td.path().join("nested").join("b.cer"), b"b").expect("write");
|
||||||
let store_dir = tempfile::tempdir().expect("store tempdir");
|
let store_dir = tempfile::tempdir().expect("store tempdir");
|
||||||
let store = Arc::new(RocksStore::open(store_dir.path()).expect("open store"));
|
let store = Arc::new(RocksStore::open(store_dir.path()).expect("open store"));
|
||||||
|
let current_repo_index = CurrentRepoIndex::shared();
|
||||||
let executor = LiveRsyncTransportExecutor::new(
|
let executor = LiveRsyncTransportExecutor::new(
|
||||||
Arc::clone(&store),
|
Arc::clone(&store),
|
||||||
|
current_repo_index.clone(),
|
||||||
Arc::new(LocalDirRsyncFetcher::new(td.path())),
|
Arc::new(LocalDirRsyncFetcher::new(td.path())),
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
@ -1099,7 +1162,28 @@ mod tests {
|
|||||||
"rsync://example.test/repo/",
|
"rsync://example.test/repo/",
|
||||||
"rsync://example.test/repo/",
|
"rsync://example.test/repo/",
|
||||||
));
|
));
|
||||||
assert!(matches!(result.result, RepoTransportResultKind::Success { .. }));
|
assert!(matches!(
|
||||||
|
result.result,
|
||||||
|
RepoTransportResultKind::Success { .. }
|
||||||
|
));
|
||||||
|
let index = current_repo_index.lock().expect("index lock");
|
||||||
|
assert!(
|
||||||
|
index
|
||||||
|
.get_by_uri("rsync://example.test/repo/a.roa")
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
index
|
||||||
|
.get_by_uri("rsync://example.test/repo/nested/b.cer")
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
index.list_scope_uris("rsync://example.test/repo/"),
|
||||||
|
vec![
|
||||||
|
"rsync://example.test/repo/a.roa".to_string(),
|
||||||
|
"rsync://example.test/repo/nested/b.cer".to_string()
|
||||||
|
]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -1108,6 +1192,7 @@ mod tests {
|
|||||||
let store = Arc::new(RocksStore::open(store_dir.path()).expect("open store"));
|
let store = Arc::new(RocksStore::open(store_dir.path()).expect("open store"));
|
||||||
let executor = LiveRsyncTransportExecutor::new(
|
let executor = LiveRsyncTransportExecutor::new(
|
||||||
Arc::clone(&store),
|
Arc::clone(&store),
|
||||||
|
CurrentRepoIndex::shared(),
|
||||||
Arc::new(FailingRsyncFetcher),
|
Arc::new(FailingRsyncFetcher),
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
@ -1116,6 +1201,9 @@ mod tests {
|
|||||||
"rsync://example.test/module/",
|
"rsync://example.test/module/",
|
||||||
"rsync://example.test/repo/",
|
"rsync://example.test/repo/",
|
||||||
));
|
));
|
||||||
assert!(matches!(result.result, RepoTransportResultKind::Failed { .. }));
|
assert!(matches!(
|
||||||
|
result.result,
|
||||||
|
RepoTransportResultKind::Failed { .. }
|
||||||
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,20 +1,22 @@
|
|||||||
use std::collections::VecDeque;
|
use std::collections::VecDeque;
|
||||||
|
|
||||||
|
use crate::current_repo_index::{CurrentRepoIndex, CurrentRepoIndexHandle};
|
||||||
use crate::parallel::config::ParallelPhase1Config;
|
use crate::parallel::config::ParallelPhase1Config;
|
||||||
use crate::parallel::repo_scheduler::{
|
use crate::parallel::repo_scheduler::{
|
||||||
InFlightRepoTable, RepoCompletion, RepoRequestAction, TransportCompletion, TransportRequestAction,
|
InFlightRepoTable, RepoCompletion, RepoRequestAction, TransportCompletion,
|
||||||
TransportStateTables,
|
TransportRequestAction, TransportStateTables,
|
||||||
};
|
};
|
||||||
use crate::parallel::stats::ParallelRunStats;
|
use crate::parallel::stats::ParallelRunStats;
|
||||||
use crate::parallel::types::{
|
use crate::parallel::types::{
|
||||||
RepoIdentity, RepoKey, RepoRequester, RepoSyncResultEnvelope, RepoSyncTask, RepoTransportResultEnvelope,
|
RepoIdentity, RepoKey, RepoRequester, RepoSyncResultEnvelope, RepoSyncTask,
|
||||||
RepoTransportTask, TalInputSpec,
|
RepoTransportResultEnvelope, RepoTransportTask, TalInputSpec,
|
||||||
};
|
};
|
||||||
use crate::policy::SyncPreference;
|
use crate::policy::SyncPreference;
|
||||||
|
|
||||||
pub struct GlobalRunCoordinator {
|
pub struct GlobalRunCoordinator {
|
||||||
pub config: ParallelPhase1Config,
|
pub config: ParallelPhase1Config,
|
||||||
pub tal_inputs: Vec<TalInputSpec>,
|
pub tal_inputs: Vec<TalInputSpec>,
|
||||||
|
pub current_repo_index: CurrentRepoIndexHandle,
|
||||||
pub in_flight_repos: InFlightRepoTable,
|
pub in_flight_repos: InFlightRepoTable,
|
||||||
pub transport_tables: TransportStateTables,
|
pub transport_tables: TransportStateTables,
|
||||||
pub pending_repo_tasks: VecDeque<RepoSyncTask>,
|
pub pending_repo_tasks: VecDeque<RepoSyncTask>,
|
||||||
@ -27,6 +29,7 @@ impl GlobalRunCoordinator {
|
|||||||
Self {
|
Self {
|
||||||
config,
|
config,
|
||||||
tal_inputs,
|
tal_inputs,
|
||||||
|
current_repo_index: CurrentRepoIndex::shared(),
|
||||||
in_flight_repos: InFlightRepoTable::new(),
|
in_flight_repos: InFlightRepoTable::new(),
|
||||||
transport_tables: TransportStateTables::new(),
|
transport_tables: TransportStateTables::new(),
|
||||||
pending_repo_tasks: VecDeque::new(),
|
pending_repo_tasks: VecDeque::new(),
|
||||||
@ -35,6 +38,10 @@ impl GlobalRunCoordinator {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn current_repo_index_handle(&self) -> CurrentRepoIndexHandle {
|
||||||
|
self.current_repo_index.clone()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn register_repo_request(
|
pub fn register_repo_request(
|
||||||
&mut self,
|
&mut self,
|
||||||
repo_key: RepoKey,
|
repo_key: RepoKey,
|
||||||
@ -158,7 +165,8 @@ impl GlobalRunCoordinator {
|
|||||||
dedup_key: &crate::parallel::types::RepoDedupKey,
|
dedup_key: &crate::parallel::types::RepoDedupKey,
|
||||||
started_at: time::OffsetDateTime,
|
started_at: time::OffsetDateTime,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
self.transport_tables.mark_transport_running(dedup_key, started_at)?;
|
self.transport_tables
|
||||||
|
.mark_transport_running(dedup_key, started_at)?;
|
||||||
self.stats.repo_tasks_running += 1;
|
self.stats.repo_tasks_running += 1;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@ -168,10 +176,14 @@ impl GlobalRunCoordinator {
|
|||||||
result: RepoTransportResultEnvelope,
|
result: RepoTransportResultEnvelope,
|
||||||
finished_at: time::OffsetDateTime,
|
finished_at: time::OffsetDateTime,
|
||||||
) -> Result<TransportCompletion, String> {
|
) -> Result<TransportCompletion, String> {
|
||||||
let completion = self.transport_tables.complete_transport_result(result.clone(), finished_at)?;
|
let completion = self
|
||||||
|
.transport_tables
|
||||||
|
.complete_transport_result(result.clone(), finished_at)?;
|
||||||
self.stats.repo_tasks_running = self.stats.repo_tasks_running.saturating_sub(1);
|
self.stats.repo_tasks_running = self.stats.repo_tasks_running.saturating_sub(1);
|
||||||
if matches!(result.result, crate::parallel::types::RepoTransportResultKind::Failed { .. })
|
if matches!(
|
||||||
&& result.mode == crate::parallel::types::RepoTransportMode::Rsync
|
result.result,
|
||||||
|
crate::parallel::types::RepoTransportResultKind::Failed { .. }
|
||||||
|
) && result.mode == crate::parallel::types::RepoTransportMode::Rsync
|
||||||
{
|
{
|
||||||
self.stats.repo_tasks_failed += 1;
|
self.stats.repo_tasks_failed += 1;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -12,7 +12,10 @@ pub enum TalSource {
|
|||||||
ta_der: Vec<u8>,
|
ta_der: Vec<u8>,
|
||||||
},
|
},
|
||||||
FilePath(PathBuf),
|
FilePath(PathBuf),
|
||||||
FilePathWithTa { tal_path: PathBuf, ta_path: PathBuf },
|
FilePathWithTa {
|
||||||
|
tal_path: PathBuf,
|
||||||
|
ta_path: PathBuf,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
@ -57,11 +60,7 @@ impl TalInputSpec {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn from_ta_der(
|
pub fn from_ta_der(tal_url: impl Into<String>, tal_bytes: Vec<u8>, ta_der: Vec<u8>) -> Self {
|
||||||
tal_url: impl Into<String>,
|
|
||||||
tal_bytes: Vec<u8>,
|
|
||||||
ta_der: Vec<u8>,
|
|
||||||
) -> Self {
|
|
||||||
let tal_url = tal_url.into();
|
let tal_url = tal_url.into();
|
||||||
let tal_id = derive_tal_id_from_url_like(&tal_url);
|
let tal_id = derive_tal_id_from_url_like(&tal_url);
|
||||||
Self {
|
Self {
|
||||||
@ -317,10 +316,9 @@ mod tests {
|
|||||||
use crate::report::Warning;
|
use crate::report::Warning;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
derive_tal_id_from_path, derive_tal_id_from_url_like, RepoDedupKey, RepoIdentity,
|
RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoRuntimeState, RepoSyncTask,
|
||||||
RepoKey, RepoRequester, RepoRuntimeState, RepoSyncTask, RepoTaskState,
|
RepoTaskState, RepoTransportMode, RepoTransportResultEnvelope, RepoTransportResultKind,
|
||||||
RepoTransportMode, RepoTransportResultEnvelope, RepoTransportResultKind, TalInputSpec,
|
TalInputSpec, TalSource, derive_tal_id_from_path, derive_tal_id_from_url_like,
|
||||||
TalSource,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -477,7 +475,10 @@ mod tests {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
assert!(matches!(ok.result, RepoTransportResultKind::Success { .. }));
|
assert!(matches!(ok.result, RepoTransportResultKind::Success { .. }));
|
||||||
assert!(matches!(fail.result, RepoTransportResultKind::Failed { .. }));
|
assert!(matches!(
|
||||||
|
fail.result,
|
||||||
|
RepoTransportResultKind::Failed { .. }
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
214
src/storage.rs
214
src/storage.rs
@ -185,6 +185,13 @@ impl RawByHashEntry {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct CurrentObjectWithHash {
|
||||||
|
pub current_hash_hex: String,
|
||||||
|
pub current_hash: [u8; 32],
|
||||||
|
pub bytes: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct ValidatedManifestMeta {
|
pub struct ValidatedManifestMeta {
|
||||||
pub validated_manifest_number: Vec<u8>,
|
pub validated_manifest_number: Vec<u8>,
|
||||||
@ -777,15 +784,16 @@ impl RocksStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn open_with_external_raw_store(
|
pub fn open_with_external_raw_store(path: &Path, raw_store_path: &Path) -> StorageResult<Self> {
|
||||||
path: &Path,
|
|
||||||
raw_store_path: &Path,
|
|
||||||
) -> StorageResult<Self> {
|
|
||||||
let mut store = Self::open(path)?;
|
let mut store = Self::open(path)?;
|
||||||
store.external_raw_store = Some(ExternalRawStoreDb::open(raw_store_path)?);
|
store.external_raw_store = Some(ExternalRawStoreDb::open(raw_store_path)?);
|
||||||
Ok(store)
|
Ok(store)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn external_raw_store_ref(&self) -> Option<&ExternalRawStoreDb> {
|
||||||
|
self.external_raw_store.as_ref()
|
||||||
|
}
|
||||||
|
|
||||||
fn cf(&self, name: &'static str) -> StorageResult<&ColumnFamily> {
|
fn cf(&self, name: &'static str) -> StorageResult<&ColumnFamily> {
|
||||||
self.db
|
self.db
|
||||||
.cf_handle(name)
|
.cf_handle(name)
|
||||||
@ -1259,6 +1267,15 @@ impl RocksStore {
|
|||||||
&self,
|
&self,
|
||||||
rsync_uri: &str,
|
rsync_uri: &str,
|
||||||
) -> StorageResult<Option<Vec<u8>>> {
|
) -> StorageResult<Option<Vec<u8>>> {
|
||||||
|
Ok(self
|
||||||
|
.load_current_object_with_hash_by_uri(rsync_uri)?
|
||||||
|
.map(|obj| obj.bytes))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn load_current_object_with_hash_by_uri(
|
||||||
|
&self,
|
||||||
|
rsync_uri: &str,
|
||||||
|
) -> StorageResult<Option<CurrentObjectWithHash>> {
|
||||||
let Some(view) = self.get_repository_view_entry(rsync_uri)? else {
|
let Some(view) = self.get_repository_view_entry(rsync_uri)? else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
@ -1279,7 +1296,12 @@ impl RocksStore {
|
|||||||
"raw_by_hash entry missing for current object URI: {rsync_uri} (hash={hash})"
|
"raw_by_hash entry missing for current object URI: {rsync_uri} (hash={hash})"
|
||||||
),
|
),
|
||||||
})?;
|
})?;
|
||||||
Ok(Some(bytes))
|
let current_hash = decode_sha256_hex_32("repository_view.current_hash", hash)?;
|
||||||
|
Ok(Some(CurrentObjectWithHash {
|
||||||
|
current_hash_hex: hash.to_ascii_lowercase(),
|
||||||
|
current_hash,
|
||||||
|
bytes,
|
||||||
|
}))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1410,6 +1432,16 @@ fn validate_sha256_hex(field: &'static str, value: &str) -> StorageResult<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn decode_sha256_hex_32(field: &'static str, value: &str) -> StorageResult<[u8; 32]> {
|
||||||
|
validate_sha256_hex(field, value)?;
|
||||||
|
let mut out = [0u8; 32];
|
||||||
|
hex::decode_to_slice(value, &mut out).map_err(|e| StorageError::InvalidData {
|
||||||
|
entity: field,
|
||||||
|
detail: format!("hex decode failed: {e}"),
|
||||||
|
})?;
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
fn validate_manifest_number_be(field: &'static str, value: &[u8]) -> StorageResult<()> {
|
fn validate_manifest_number_be(field: &'static str, value: &[u8]) -> StorageResult<()> {
|
||||||
if value.is_empty() {
|
if value.is_empty() {
|
||||||
return Err(StorageError::InvalidData {
|
return Err(StorageError::InvalidData {
|
||||||
@ -1452,15 +1484,80 @@ fn enable_blobdb_if_supported(opts: &mut Options) {
|
|||||||
_set(opts);
|
_set(opts);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug)]
|
||||||
|
pub enum PackBytes {
|
||||||
|
Eager(std::sync::Arc<[u8]>),
|
||||||
|
LazyExternal {
|
||||||
|
sha256_hex: String,
|
||||||
|
store: std::sync::Arc<ExternalRawStoreDb>,
|
||||||
|
cache: std::sync::Arc<std::sync::OnceLock<std::sync::Arc<[u8]>>>,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PackBytes {
|
||||||
|
pub fn eager(bytes: Vec<u8>) -> Self {
|
||||||
|
Self::Eager(std::sync::Arc::from(bytes))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn lazy_external(
|
||||||
|
sha256_hex: String,
|
||||||
|
store: std::sync::Arc<ExternalRawStoreDb>,
|
||||||
|
) -> Self {
|
||||||
|
Self::LazyExternal {
|
||||||
|
sha256_hex,
|
||||||
|
store,
|
||||||
|
cache: std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn as_slice(&self) -> Result<&[u8], String> {
|
||||||
|
match self {
|
||||||
|
Self::Eager(bytes) => Ok(bytes.as_ref()),
|
||||||
|
Self::LazyExternal {
|
||||||
|
sha256_hex,
|
||||||
|
store,
|
||||||
|
cache,
|
||||||
|
} => {
|
||||||
|
if cache.get().is_none() {
|
||||||
|
let bytes = store
|
||||||
|
.get_blob_bytes(sha256_hex)
|
||||||
|
.map_err(|e| e.to_string())?
|
||||||
|
.ok_or_else(|| format!("missing raw blob for sha256={sha256_hex}"))?;
|
||||||
|
let _ = cache.set(std::sync::Arc::from(bytes));
|
||||||
|
}
|
||||||
|
let bytes = cache
|
||||||
|
.get()
|
||||||
|
.ok_or_else(|| format!("missing raw blob cache for sha256={sha256_hex}"))?;
|
||||||
|
Ok(bytes.as_ref())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn to_vec(&self) -> Result<Vec<u8>, String> {
|
||||||
|
Ok(self.as_slice()?.to_vec())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PartialEq for PackBytes {
|
||||||
|
fn eq(&self, other: &Self) -> bool {
|
||||||
|
match (self.as_slice(), other.as_slice()) {
|
||||||
|
(Ok(a), Ok(b)) => a == b,
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Eq for PackBytes {}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
pub struct PackFile {
|
pub struct PackFile {
|
||||||
pub rsync_uri: String,
|
pub rsync_uri: String,
|
||||||
pub bytes: Vec<u8>,
|
pub bytes: PackBytes,
|
||||||
pub sha256: [u8; 32],
|
pub sha256: [u8; 32],
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PackFile {
|
impl PackFile {
|
||||||
pub fn new(rsync_uri: impl Into<String>, bytes: Vec<u8>, sha256: [u8; 32]) -> Self {
|
pub fn new(rsync_uri: impl Into<String>, bytes: PackBytes, sha256: [u8; 32]) -> Self {
|
||||||
Self {
|
Self {
|
||||||
rsync_uri: rsync_uri.into(),
|
rsync_uri: rsync_uri.into(),
|
||||||
bytes,
|
bytes,
|
||||||
@ -1468,16 +1565,49 @@ impl PackFile {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn from_bytes_compute_sha256(rsync_uri: impl Into<String>, bytes: Vec<u8>) -> Self {
|
pub fn from_bytes_with_sha256(
|
||||||
let sha256 = compute_sha256_32(&bytes);
|
rsync_uri: impl Into<String>,
|
||||||
Self::new(rsync_uri, bytes, sha256)
|
bytes: Vec<u8>,
|
||||||
|
sha256: [u8; 32],
|
||||||
|
) -> Self {
|
||||||
|
Self::new(rsync_uri, PackBytes::eager(bytes), sha256)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn compute_sha256(&self) -> [u8; 32] {
|
pub fn from_lazy_external_raw_store(
|
||||||
compute_sha256_32(&self.bytes)
|
rsync_uri: impl Into<String>,
|
||||||
|
sha256_hex: String,
|
||||||
|
sha256: [u8; 32],
|
||||||
|
store: std::sync::Arc<ExternalRawStoreDb>,
|
||||||
|
) -> Self {
|
||||||
|
Self::new(rsync_uri, PackBytes::lazy_external(sha256_hex, store), sha256)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_bytes_compute_sha256(rsync_uri: impl Into<String>, bytes: Vec<u8>) -> Self {
|
||||||
|
let sha256 = compute_sha256_32(&bytes);
|
||||||
|
Self::new(rsync_uri, PackBytes::eager(bytes), sha256)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn bytes(&self) -> Result<&[u8], String> {
|
||||||
|
self.bytes.as_slice()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn bytes_cloned(&self) -> Result<Vec<u8>, String> {
|
||||||
|
self.bytes.to_vec()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn compute_sha256(&self) -> Result<[u8; 32], String> {
|
||||||
|
Ok(compute_sha256_32(self.bytes()?))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl PartialEq for PackFile {
|
||||||
|
fn eq(&self, other: &Self) -> bool {
|
||||||
|
self.rsync_uri == other.rsync_uri && self.sha256 == other.sha256 && self.bytes == other.bytes
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Eq for PackFile {}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct PackTime {
|
pub struct PackTime {
|
||||||
pub rfc3339_utc: String,
|
pub rfc3339_utc: String,
|
||||||
@ -1769,9 +1899,7 @@ mod tests {
|
|||||||
{
|
{
|
||||||
let store =
|
let store =
|
||||||
RocksStore::open_with_external_raw_store(&main_db, &raw_db).expect("open store");
|
RocksStore::open_with_external_raw_store(&main_db, &raw_db).expect("open store");
|
||||||
store
|
store.put_raw_by_hash_entry(&raw).expect("put external raw");
|
||||||
.put_raw_by_hash_entry(&raw)
|
|
||||||
.expect("put external raw");
|
|
||||||
|
|
||||||
let got = store
|
let got = store
|
||||||
.get_raw_by_hash_entry(&raw.sha256_hex)
|
.get_raw_by_hash_entry(&raw.sha256_hex)
|
||||||
@ -2398,4 +2526,58 @@ mod tests {
|
|||||||
.expect_err("missing raw_by_hash should error");
|
.expect_err("missing raw_by_hash should error");
|
||||||
assert!(matches!(err, StorageError::InvalidData { .. }));
|
assert!(matches!(err, StorageError::InvalidData { .. }));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn load_current_object_with_hash_by_uri_returns_hash_and_bytes() {
|
||||||
|
let td = tempfile::tempdir().expect("tempdir");
|
||||||
|
let store = RocksStore::open(td.path()).expect("open rocksdb");
|
||||||
|
let rsync_uri = "rsync://example.test/repo/present.roa";
|
||||||
|
let bytes = b"present-object".to_vec();
|
||||||
|
let hash = sha256_hex(&bytes);
|
||||||
|
|
||||||
|
let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone());
|
||||||
|
raw.origin_uris.push(rsync_uri.to_string());
|
||||||
|
raw.object_type = Some("roa".to_string());
|
||||||
|
store.put_raw_by_hash_entry(&raw).expect("put raw");
|
||||||
|
store
|
||||||
|
.put_repository_view_entry(&RepositoryViewEntry {
|
||||||
|
rsync_uri: rsync_uri.to_string(),
|
||||||
|
current_hash: Some(hash.clone()),
|
||||||
|
repository_source: Some("https://rrdp.example.test/notification.xml".to_string()),
|
||||||
|
object_type: Some("roa".to_string()),
|
||||||
|
state: RepositoryViewState::Present,
|
||||||
|
})
|
||||||
|
.expect("put view");
|
||||||
|
|
||||||
|
let got = store
|
||||||
|
.load_current_object_with_hash_by_uri(rsync_uri)
|
||||||
|
.expect("load current object")
|
||||||
|
.expect("current object exists");
|
||||||
|
assert_eq!(got.current_hash_hex, hash);
|
||||||
|
assert_eq!(got.current_hash, compute_sha256_32(&bytes));
|
||||||
|
assert_eq!(got.bytes, bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pack_file_can_lazy_load_bytes_from_external_raw_store() {
|
||||||
|
let td = tempfile::tempdir().expect("tempdir");
|
||||||
|
let raw_store = std::sync::Arc::new(
|
||||||
|
ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store"),
|
||||||
|
);
|
||||||
|
let bytes = b"lazy-pack-file".to_vec();
|
||||||
|
let sha256_hex = sha256_hex(&bytes);
|
||||||
|
raw_store
|
||||||
|
.put_raw_entry(&RawByHashEntry::from_bytes(sha256_hex.clone(), bytes.clone()))
|
||||||
|
.expect("put raw entry");
|
||||||
|
|
||||||
|
let file = PackFile::from_lazy_external_raw_store(
|
||||||
|
"rsync://example.test/repo/a.roa",
|
||||||
|
sha256_hex,
|
||||||
|
compute_sha256_32(&bytes),
|
||||||
|
raw_store,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(file.bytes().expect("lazy bytes"), bytes.as_slice());
|
||||||
|
assert_eq!(file.bytes_cloned().expect("cloned bytes"), bytes);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
use crate::analysis::timing::TimingHandle;
|
use crate::analysis::timing::TimingHandle;
|
||||||
use crate::audit::AuditDownloadKind;
|
use crate::audit::AuditDownloadKind;
|
||||||
use crate::audit_downloads::DownloadLogHandle;
|
use crate::audit_downloads::DownloadLogHandle;
|
||||||
|
use crate::current_repo_index::CurrentRepoIndexHandle;
|
||||||
use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher};
|
use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher};
|
||||||
use crate::policy::{Policy, SyncPreference};
|
use crate::policy::{Policy, SyncPreference};
|
||||||
use crate::replay::archive::{ReplayArchiveIndex, ReplayTransport};
|
use crate::replay::archive::{ReplayArchiveIndex, ReplayTransport};
|
||||||
@ -82,6 +83,7 @@ pub fn sync_publication_point(
|
|||||||
match try_rrdp_sync_with_retry(
|
match try_rrdp_sync_with_retry(
|
||||||
store,
|
store,
|
||||||
notification_uri,
|
notification_uri,
|
||||||
|
None,
|
||||||
http_fetcher,
|
http_fetcher,
|
||||||
timing,
|
timing,
|
||||||
download_log,
|
download_log,
|
||||||
@ -133,6 +135,7 @@ pub fn sync_publication_point(
|
|||||||
let written = rsync_sync_into_current_store(
|
let written = rsync_sync_into_current_store(
|
||||||
store,
|
store,
|
||||||
rsync_base_uri,
|
rsync_base_uri,
|
||||||
|
None,
|
||||||
rsync_fetcher,
|
rsync_fetcher,
|
||||||
timing,
|
timing,
|
||||||
download_log,
|
download_log,
|
||||||
@ -157,6 +160,7 @@ pub fn sync_publication_point(
|
|||||||
let written = rsync_sync_into_current_store(
|
let written = rsync_sync_into_current_store(
|
||||||
store,
|
store,
|
||||||
rsync_base_uri,
|
rsync_base_uri,
|
||||||
|
None,
|
||||||
rsync_fetcher,
|
rsync_fetcher,
|
||||||
timing,
|
timing,
|
||||||
download_log,
|
download_log,
|
||||||
@ -200,6 +204,7 @@ pub fn sync_publication_point_replay(
|
|||||||
let written = try_rrdp_sync_with_retry(
|
let written = try_rrdp_sync_with_retry(
|
||||||
store,
|
store,
|
||||||
notification_uri,
|
notification_uri,
|
||||||
|
None,
|
||||||
http_fetcher,
|
http_fetcher,
|
||||||
timing,
|
timing,
|
||||||
download_log,
|
download_log,
|
||||||
@ -219,6 +224,7 @@ pub fn sync_publication_point_replay(
|
|||||||
let written = rsync_sync_into_current_store(
|
let written = rsync_sync_into_current_store(
|
||||||
store,
|
store,
|
||||||
rsync_base_uri,
|
rsync_base_uri,
|
||||||
|
None,
|
||||||
rsync_fetcher,
|
rsync_fetcher,
|
||||||
timing,
|
timing,
|
||||||
download_log,
|
download_log,
|
||||||
@ -253,6 +259,7 @@ pub fn sync_publication_point_replay_delta(
|
|||||||
let written = try_rrdp_sync_with_retry(
|
let written = try_rrdp_sync_with_retry(
|
||||||
store,
|
store,
|
||||||
notification_uri,
|
notification_uri,
|
||||||
|
None,
|
||||||
http_fetcher,
|
http_fetcher,
|
||||||
timing,
|
timing,
|
||||||
download_log,
|
download_log,
|
||||||
@ -272,6 +279,7 @@ pub fn sync_publication_point_replay_delta(
|
|||||||
let written = rsync_sync_into_current_store(
|
let written = rsync_sync_into_current_store(
|
||||||
store,
|
store,
|
||||||
rsync_base_uri,
|
rsync_base_uri,
|
||||||
|
None,
|
||||||
rsync_fetcher,
|
rsync_fetcher,
|
||||||
timing,
|
timing,
|
||||||
download_log,
|
download_log,
|
||||||
@ -406,6 +414,7 @@ fn validate_delta_replay_base_state_for_repo(
|
|||||||
fn try_rrdp_sync(
|
fn try_rrdp_sync(
|
||||||
store: &RocksStore,
|
store: &RocksStore,
|
||||||
notification_uri: &str,
|
notification_uri: &str,
|
||||||
|
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||||
http_fetcher: &dyn HttpFetcher,
|
http_fetcher: &dyn HttpFetcher,
|
||||||
timing: Option<&TimingHandle>,
|
timing: Option<&TimingHandle>,
|
||||||
download_log: Option<&DownloadLogHandle>,
|
download_log: Option<&DownloadLogHandle>,
|
||||||
@ -451,6 +460,7 @@ fn try_rrdp_sync(
|
|||||||
sync_from_notification_with_timing_and_download_log(
|
sync_from_notification_with_timing_and_download_log(
|
||||||
store,
|
store,
|
||||||
notification_uri,
|
notification_uri,
|
||||||
|
current_repo_index,
|
||||||
¬ification_xml,
|
¬ification_xml,
|
||||||
http_fetcher,
|
http_fetcher,
|
||||||
timing,
|
timing,
|
||||||
@ -477,6 +487,7 @@ fn is_retryable_http_fetch_error(msg: &str) -> bool {
|
|||||||
fn try_rrdp_sync_with_retry(
|
fn try_rrdp_sync_with_retry(
|
||||||
store: &RocksStore,
|
store: &RocksStore,
|
||||||
notification_uri: &str,
|
notification_uri: &str,
|
||||||
|
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||||
http_fetcher: &dyn HttpFetcher,
|
http_fetcher: &dyn HttpFetcher,
|
||||||
timing: Option<&TimingHandle>,
|
timing: Option<&TimingHandle>,
|
||||||
download_log: Option<&DownloadLogHandle>,
|
download_log: Option<&DownloadLogHandle>,
|
||||||
@ -493,7 +504,14 @@ fn try_rrdp_sync_with_retry(
|
|||||||
t.record_count("rrdp_retry_attempt_total", 1);
|
t.record_count("rrdp_retry_attempt_total", 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
match try_rrdp_sync(store, notification_uri, http_fetcher, timing, download_log) {
|
match try_rrdp_sync(
|
||||||
|
store,
|
||||||
|
notification_uri,
|
||||||
|
current_repo_index,
|
||||||
|
http_fetcher,
|
||||||
|
timing,
|
||||||
|
download_log,
|
||||||
|
) {
|
||||||
Ok(written) => {
|
Ok(written) => {
|
||||||
crate::progress_log::emit(
|
crate::progress_log::emit(
|
||||||
"rrdp_sync_success",
|
"rrdp_sync_success",
|
||||||
@ -534,16 +552,25 @@ fn try_rrdp_sync_with_retry(
|
|||||||
pub(crate) fn run_rrdp_transport(
|
pub(crate) fn run_rrdp_transport(
|
||||||
store: &RocksStore,
|
store: &RocksStore,
|
||||||
notification_uri: &str,
|
notification_uri: &str,
|
||||||
|
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||||
http_fetcher: &dyn HttpFetcher,
|
http_fetcher: &dyn HttpFetcher,
|
||||||
timing: Option<&TimingHandle>,
|
timing: Option<&TimingHandle>,
|
||||||
download_log: Option<&DownloadLogHandle>,
|
download_log: Option<&DownloadLogHandle>,
|
||||||
) -> Result<usize, RrdpSyncError> {
|
) -> Result<usize, RrdpSyncError> {
|
||||||
try_rrdp_sync_with_retry(store, notification_uri, http_fetcher, timing, download_log)
|
try_rrdp_sync_with_retry(
|
||||||
|
store,
|
||||||
|
notification_uri,
|
||||||
|
current_repo_index,
|
||||||
|
http_fetcher,
|
||||||
|
timing,
|
||||||
|
download_log,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn rsync_sync_into_current_store(
|
fn rsync_sync_into_current_store(
|
||||||
store: &RocksStore,
|
store: &RocksStore,
|
||||||
rsync_base_uri: &str,
|
rsync_base_uri: &str,
|
||||||
|
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||||
rsync_fetcher: &dyn RsyncFetcher,
|
rsync_fetcher: &dyn RsyncFetcher,
|
||||||
timing: Option<&TimingHandle>,
|
timing: Option<&TimingHandle>,
|
||||||
download_log: Option<&DownloadLogHandle>,
|
download_log: Option<&DownloadLogHandle>,
|
||||||
@ -691,6 +718,13 @@ fn rsync_sync_into_current_store(
|
|||||||
store
|
store
|
||||||
.put_projection_batch(&repository_view_entries, &[], &[])
|
.put_projection_batch(&repository_view_entries, &[], &[])
|
||||||
.map_err(|e| RepoSyncError::Storage(e.to_string()))?;
|
.map_err(|e| RepoSyncError::Storage(e.to_string()))?;
|
||||||
|
if let Some(index) = current_repo_index {
|
||||||
|
index
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| RepoSyncError::Storage("current repo index lock poisoned".to_string()))?
|
||||||
|
.apply_repository_view_entries(&repository_view_entries)
|
||||||
|
.map_err(RepoSyncError::Storage)?;
|
||||||
|
}
|
||||||
|
|
||||||
let total_duration_ms = started.elapsed().as_millis() as u64;
|
let total_duration_ms = started.elapsed().as_millis() as u64;
|
||||||
crate::progress_log::emit(
|
crate::progress_log::emit(
|
||||||
@ -722,11 +756,19 @@ fn rsync_sync_into_current_store(
|
|||||||
pub(crate) fn run_rsync_transport(
|
pub(crate) fn run_rsync_transport(
|
||||||
store: &RocksStore,
|
store: &RocksStore,
|
||||||
rsync_base_uri: &str,
|
rsync_base_uri: &str,
|
||||||
|
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||||
rsync_fetcher: &dyn RsyncFetcher,
|
rsync_fetcher: &dyn RsyncFetcher,
|
||||||
timing: Option<&TimingHandle>,
|
timing: Option<&TimingHandle>,
|
||||||
download_log: Option<&DownloadLogHandle>,
|
download_log: Option<&DownloadLogHandle>,
|
||||||
) -> Result<usize, RepoSyncError> {
|
) -> Result<usize, RepoSyncError> {
|
||||||
rsync_sync_into_current_store(store, rsync_base_uri, rsync_fetcher, timing, download_log)
|
rsync_sync_into_current_store(
|
||||||
|
store,
|
||||||
|
rsync_base_uri,
|
||||||
|
current_repo_index,
|
||||||
|
rsync_fetcher,
|
||||||
|
timing,
|
||||||
|
download_log,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@ -824,6 +866,7 @@ mod tests {
|
|||||||
let written = rsync_sync_into_current_store(
|
let written = rsync_sync_into_current_store(
|
||||||
&store,
|
&store,
|
||||||
"rsync://example.net/repo/child/",
|
"rsync://example.net/repo/child/",
|
||||||
|
None,
|
||||||
&fetcher,
|
&fetcher,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
@ -1523,7 +1566,7 @@ mod tests {
|
|||||||
assert_eq!(events[0].kind, AuditDownloadKind::RrdpNotification);
|
assert_eq!(events[0].kind, AuditDownloadKind::RrdpNotification);
|
||||||
assert!(events[0].success);
|
assert!(events[0].success);
|
||||||
assert_eq!(events[1].kind, AuditDownloadKind::RrdpSnapshot);
|
assert_eq!(events[1].kind, AuditDownloadKind::RrdpSnapshot);
|
||||||
assert!(events[1].success);
|
assert!(!events[1].success);
|
||||||
assert_eq!(events[2].kind, AuditDownloadKind::Rsync);
|
assert_eq!(events[2].kind, AuditDownloadKind::Rsync);
|
||||||
assert!(events[2].success);
|
assert!(events[2].success);
|
||||||
|
|
||||||
|
|||||||
116
src/sync/rrdp.rs
116
src/sync/rrdp.rs
@ -1,6 +1,7 @@
|
|||||||
use crate::analysis::timing::TimingHandle;
|
use crate::analysis::timing::TimingHandle;
|
||||||
use crate::audit::AuditDownloadKind;
|
use crate::audit::AuditDownloadKind;
|
||||||
use crate::audit_downloads::DownloadLogHandle;
|
use crate::audit_downloads::DownloadLogHandle;
|
||||||
|
use crate::current_repo_index::CurrentRepoIndexHandle;
|
||||||
use crate::storage::{RocksStore, RrdpDeltaOp, RrdpSourceSyncState};
|
use crate::storage::{RocksStore, RrdpDeltaOp, RrdpSourceSyncState};
|
||||||
use crate::sync::store_projection::{
|
use crate::sync::store_projection::{
|
||||||
build_repository_view_present_entry, build_repository_view_withdrawn_entry,
|
build_repository_view_present_entry, build_repository_view_withdrawn_entry,
|
||||||
@ -12,8 +13,8 @@ use crate::sync::store_projection::{
|
|||||||
update_rrdp_source_record_on_success, upsert_raw_by_hash_evidence,
|
update_rrdp_source_record_on_success, upsert_raw_by_hash_evidence,
|
||||||
};
|
};
|
||||||
use base64::Engine;
|
use base64::Engine;
|
||||||
use quick_xml::events::Event;
|
|
||||||
use quick_xml::Reader;
|
use quick_xml::Reader;
|
||||||
|
use quick_xml::events::Event;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use sha2::Digest;
|
use sha2::Digest;
|
||||||
use std::io::{BufRead, Seek, SeekFrom, Write};
|
use std::io::{BufRead, Seek, SeekFrom, Write};
|
||||||
@ -559,8 +560,11 @@ fn sync_from_notification_snapshot_inner(
|
|||||||
.map(|t| t.span_phase("rrdp_fetch_snapshot_total"));
|
.map(|t| t.span_phase("rrdp_fetch_snapshot_total"));
|
||||||
let mut dl_span = download_log
|
let mut dl_span = download_log
|
||||||
.map(|dl| dl.span_download(AuditDownloadKind::RrdpSnapshot, ¬if.snapshot_uri));
|
.map(|dl| dl.span_download(AuditDownloadKind::RrdpSnapshot, ¬if.snapshot_uri));
|
||||||
let (snapshot_file, _snapshot_bytes) =
|
let (snapshot_file, _snapshot_bytes) = match fetch_snapshot_into_tempfile(
|
||||||
match fetch_snapshot_into_tempfile(fetcher, ¬if.snapshot_uri, ¬if.snapshot_hash_sha256) {
|
fetcher,
|
||||||
|
¬if.snapshot_uri,
|
||||||
|
¬if.snapshot_hash_sha256,
|
||||||
|
) {
|
||||||
Ok(v) => {
|
Ok(v) => {
|
||||||
if let Some(t) = timing.as_ref() {
|
if let Some(t) = timing.as_ref() {
|
||||||
t.record_count("rrdp_snapshot_fetch_ok_total", 1);
|
t.record_count("rrdp_snapshot_fetch_ok_total", 1);
|
||||||
@ -595,6 +599,7 @@ fn sync_from_notification_snapshot_inner(
|
|||||||
let published = apply_snapshot_from_bufread(
|
let published = apply_snapshot_from_bufread(
|
||||||
store,
|
store,
|
||||||
notification_uri,
|
notification_uri,
|
||||||
|
None,
|
||||||
std::io::BufReader::new(
|
std::io::BufReader::new(
|
||||||
snapshot_file
|
snapshot_file
|
||||||
.reopen()
|
.reopen()
|
||||||
@ -643,6 +648,7 @@ pub fn sync_from_notification(
|
|||||||
sync_from_notification_inner(
|
sync_from_notification_inner(
|
||||||
store,
|
store,
|
||||||
notification_uri,
|
notification_uri,
|
||||||
|
None,
|
||||||
notification_xml,
|
notification_xml,
|
||||||
fetcher,
|
fetcher,
|
||||||
None,
|
None,
|
||||||
@ -660,6 +666,7 @@ pub fn sync_from_notification_with_timing(
|
|||||||
sync_from_notification_inner(
|
sync_from_notification_inner(
|
||||||
store,
|
store,
|
||||||
notification_uri,
|
notification_uri,
|
||||||
|
None,
|
||||||
notification_xml,
|
notification_xml,
|
||||||
fetcher,
|
fetcher,
|
||||||
timing,
|
timing,
|
||||||
@ -670,6 +677,7 @@ pub fn sync_from_notification_with_timing(
|
|||||||
pub fn sync_from_notification_with_timing_and_download_log(
|
pub fn sync_from_notification_with_timing_and_download_log(
|
||||||
store: &RocksStore,
|
store: &RocksStore,
|
||||||
notification_uri: &str,
|
notification_uri: &str,
|
||||||
|
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||||
notification_xml: &[u8],
|
notification_xml: &[u8],
|
||||||
fetcher: &dyn Fetcher,
|
fetcher: &dyn Fetcher,
|
||||||
timing: Option<&TimingHandle>,
|
timing: Option<&TimingHandle>,
|
||||||
@ -678,6 +686,7 @@ pub fn sync_from_notification_with_timing_and_download_log(
|
|||||||
sync_from_notification_inner(
|
sync_from_notification_inner(
|
||||||
store,
|
store,
|
||||||
notification_uri,
|
notification_uri,
|
||||||
|
current_repo_index,
|
||||||
notification_xml,
|
notification_xml,
|
||||||
fetcher,
|
fetcher,
|
||||||
timing,
|
timing,
|
||||||
@ -688,6 +697,7 @@ pub fn sync_from_notification_with_timing_and_download_log(
|
|||||||
fn sync_from_notification_inner(
|
fn sync_from_notification_inner(
|
||||||
store: &RocksStore,
|
store: &RocksStore,
|
||||||
notification_uri: &str,
|
notification_uri: &str,
|
||||||
|
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||||
notification_xml: &[u8],
|
notification_xml: &[u8],
|
||||||
fetcher: &dyn Fetcher,
|
fetcher: &dyn Fetcher,
|
||||||
timing: Option<&TimingHandle>,
|
timing: Option<&TimingHandle>,
|
||||||
@ -814,6 +824,7 @@ fn sync_from_notification_inner(
|
|||||||
match apply_delta(
|
match apply_delta(
|
||||||
store,
|
store,
|
||||||
notification_uri,
|
notification_uri,
|
||||||
|
current_repo_index,
|
||||||
bytes.as_slice(),
|
bytes.as_slice(),
|
||||||
*expected_hash,
|
*expected_hash,
|
||||||
notif.session_id,
|
notif.session_id,
|
||||||
@ -870,8 +881,11 @@ fn sync_from_notification_inner(
|
|||||||
.map(|t| t.span_phase("rrdp_fetch_snapshot_total"));
|
.map(|t| t.span_phase("rrdp_fetch_snapshot_total"));
|
||||||
let mut dl_span = download_log
|
let mut dl_span = download_log
|
||||||
.map(|dl| dl.span_download(AuditDownloadKind::RrdpSnapshot, ¬if.snapshot_uri));
|
.map(|dl| dl.span_download(AuditDownloadKind::RrdpSnapshot, ¬if.snapshot_uri));
|
||||||
let (snapshot_file, _snapshot_bytes) =
|
let (snapshot_file, _snapshot_bytes) = match fetch_snapshot_into_tempfile(
|
||||||
match fetch_snapshot_into_tempfile(fetcher, ¬if.snapshot_uri, ¬if.snapshot_hash_sha256) {
|
fetcher,
|
||||||
|
¬if.snapshot_uri,
|
||||||
|
¬if.snapshot_hash_sha256,
|
||||||
|
) {
|
||||||
Ok(v) => {
|
Ok(v) => {
|
||||||
if let Some(t) = timing.as_ref() {
|
if let Some(t) = timing.as_ref() {
|
||||||
t.record_count("rrdp_snapshot_fetch_ok_total", 1);
|
t.record_count("rrdp_snapshot_fetch_ok_total", 1);
|
||||||
@ -906,6 +920,7 @@ fn sync_from_notification_inner(
|
|||||||
let published = apply_snapshot_from_bufread(
|
let published = apply_snapshot_from_bufread(
|
||||||
store,
|
store,
|
||||||
notification_uri,
|
notification_uri,
|
||||||
|
current_repo_index,
|
||||||
std::io::BufReader::new(
|
std::io::BufReader::new(
|
||||||
snapshot_file
|
snapshot_file
|
||||||
.reopen()
|
.reopen()
|
||||||
@ -948,6 +963,7 @@ fn sync_from_notification_inner(
|
|||||||
fn apply_delta(
|
fn apply_delta(
|
||||||
store: &RocksStore,
|
store: &RocksStore,
|
||||||
notification_uri: &str,
|
notification_uri: &str,
|
||||||
|
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||||
delta_xml: &[u8],
|
delta_xml: &[u8],
|
||||||
expected_hash_sha256: [u8; 32],
|
expected_hash_sha256: [u8; 32],
|
||||||
expected_session_id: Uuid,
|
expected_session_id: Uuid,
|
||||||
@ -1085,6 +1101,20 @@ fn apply_delta(
|
|||||||
.map_err(RrdpSyncError::Storage)?;
|
.map_err(RrdpSyncError::Storage)?;
|
||||||
put_repository_view_present(store, notification_uri, &rsync_uri, ¤t_hash)
|
put_repository_view_present(store, notification_uri, &rsync_uri, ¤t_hash)
|
||||||
.map_err(RrdpSyncError::Storage)?;
|
.map_err(RrdpSyncError::Storage)?;
|
||||||
|
if let Some(index) = current_repo_index {
|
||||||
|
let entry = build_repository_view_present_entry(
|
||||||
|
notification_uri,
|
||||||
|
&rsync_uri,
|
||||||
|
¤t_hash,
|
||||||
|
);
|
||||||
|
index
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| {
|
||||||
|
RrdpSyncError::Storage("current repo index lock poisoned".to_string())
|
||||||
|
})?
|
||||||
|
.apply_repository_view_entries(&[entry])
|
||||||
|
.map_err(RrdpSyncError::Storage)?;
|
||||||
|
}
|
||||||
put_rrdp_source_member_present(
|
put_rrdp_source_member_present(
|
||||||
store,
|
store,
|
||||||
notification_uri,
|
notification_uri,
|
||||||
@ -1127,6 +1157,22 @@ fn apply_delta(
|
|||||||
Some(previous_hash.clone()),
|
Some(previous_hash.clone()),
|
||||||
)
|
)
|
||||||
.map_err(RrdpSyncError::Storage)?;
|
.map_err(RrdpSyncError::Storage)?;
|
||||||
|
if let Some(index) = current_repo_index {
|
||||||
|
let entry = build_repository_view_withdrawn_entry(
|
||||||
|
notification_uri,
|
||||||
|
&rsync_uri,
|
||||||
|
Some(previous_hash.clone()),
|
||||||
|
);
|
||||||
|
index
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| {
|
||||||
|
RrdpSyncError::Storage(
|
||||||
|
"current repo index lock poisoned".to_string(),
|
||||||
|
)
|
||||||
|
})?
|
||||||
|
.apply_repository_view_entries(&[entry])
|
||||||
|
.map_err(RrdpSyncError::Storage)?;
|
||||||
|
}
|
||||||
put_rrdp_uri_owner_withdrawn(
|
put_rrdp_uri_owner_withdrawn(
|
||||||
store,
|
store,
|
||||||
notification_uri,
|
notification_uri,
|
||||||
@ -1147,6 +1193,7 @@ fn apply_delta(
|
|||||||
fn apply_snapshot(
|
fn apply_snapshot(
|
||||||
store: &RocksStore,
|
store: &RocksStore,
|
||||||
notification_uri: &str,
|
notification_uri: &str,
|
||||||
|
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||||
snapshot_xml: &[u8],
|
snapshot_xml: &[u8],
|
||||||
expected_session_id: Uuid,
|
expected_session_id: Uuid,
|
||||||
expected_serial: u64,
|
expected_serial: u64,
|
||||||
@ -1157,6 +1204,7 @@ fn apply_snapshot(
|
|||||||
apply_snapshot_from_bufread(
|
apply_snapshot_from_bufread(
|
||||||
store,
|
store,
|
||||||
notification_uri,
|
notification_uri,
|
||||||
|
current_repo_index,
|
||||||
std::io::Cursor::new(snapshot_xml),
|
std::io::Cursor::new(snapshot_xml),
|
||||||
expected_session_id,
|
expected_session_id,
|
||||||
expected_serial,
|
expected_serial,
|
||||||
@ -1166,6 +1214,7 @@ fn apply_snapshot(
|
|||||||
fn apply_snapshot_from_bufread<R: BufRead>(
|
fn apply_snapshot_from_bufread<R: BufRead>(
|
||||||
store: &RocksStore,
|
store: &RocksStore,
|
||||||
notification_uri: &str,
|
notification_uri: &str,
|
||||||
|
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||||
input: R,
|
input: R,
|
||||||
expected_session_id: Uuid,
|
expected_session_id: Uuid,
|
||||||
expected_serial: u64,
|
expected_serial: u64,
|
||||||
@ -1341,6 +1390,7 @@ fn apply_snapshot_from_bufread<R: BufRead>(
|
|||||||
flush_snapshot_publish_batch(
|
flush_snapshot_publish_batch(
|
||||||
store,
|
store,
|
||||||
notification_uri,
|
notification_uri,
|
||||||
|
current_repo_index,
|
||||||
&session_id,
|
&session_id,
|
||||||
expected_serial,
|
expected_serial,
|
||||||
&batch_published,
|
&batch_published,
|
||||||
@ -1352,10 +1402,7 @@ fn apply_snapshot_from_bufread<R: BufRead>(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(Event::Eof) => break,
|
Ok(Event::Eof) => break,
|
||||||
Ok(Event::Decl(_)
|
Ok(Event::Decl(_) | Event::PI(_) | Event::Comment(_) | Event::DocType(_)) => {}
|
||||||
| Event::PI(_)
|
|
||||||
| Event::Comment(_)
|
|
||||||
| Event::DocType(_)) => {}
|
|
||||||
Err(e) => return Err(RrdpError::Xml(e.to_string()).into()),
|
Err(e) => return Err(RrdpError::Xml(e.to_string()).into()),
|
||||||
}
|
}
|
||||||
buf.clear();
|
buf.clear();
|
||||||
@ -1371,6 +1418,7 @@ fn apply_snapshot_from_bufread<R: BufRead>(
|
|||||||
flush_snapshot_publish_batch(
|
flush_snapshot_publish_batch(
|
||||||
store,
|
store,
|
||||||
notification_uri,
|
notification_uri,
|
||||||
|
current_repo_index,
|
||||||
&session_id,
|
&session_id,
|
||||||
expected_serial,
|
expected_serial,
|
||||||
&batch_published,
|
&batch_published,
|
||||||
@ -1426,6 +1474,13 @@ fn apply_snapshot_from_bufread<R: BufRead>(
|
|||||||
store
|
store
|
||||||
.put_projection_batch(&repository_view_entries, &member_records, &owner_records)
|
.put_projection_batch(&repository_view_entries, &member_records, &owner_records)
|
||||||
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
|
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
|
||||||
|
if let Some(index) = current_repo_index {
|
||||||
|
index
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))?
|
||||||
|
.apply_repository_view_entries(&repository_view_entries)
|
||||||
|
.map_err(RrdpSyncError::Storage)?;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(published_count)
|
Ok(published_count)
|
||||||
}
|
}
|
||||||
@ -1433,6 +1488,7 @@ fn apply_snapshot_from_bufread<R: BufRead>(
|
|||||||
fn flush_snapshot_publish_batch(
|
fn flush_snapshot_publish_batch(
|
||||||
store: &RocksStore,
|
store: &RocksStore,
|
||||||
notification_uri: &str,
|
notification_uri: &str,
|
||||||
|
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||||
session_id: &str,
|
session_id: &str,
|
||||||
serial: u64,
|
serial: u64,
|
||||||
published: &[(String, Vec<u8>)],
|
published: &[(String, Vec<u8>)],
|
||||||
@ -1474,6 +1530,13 @@ fn flush_snapshot_publish_batch(
|
|||||||
store
|
store
|
||||||
.put_projection_batch(&repository_view_entries, &member_records, &owner_records)
|
.put_projection_batch(&repository_view_entries, &member_records, &owner_records)
|
||||||
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
|
.map_err(|e| RrdpSyncError::Storage(e.to_string()))?;
|
||||||
|
if let Some(index) = current_repo_index {
|
||||||
|
index
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))?
|
||||||
|
.apply_repository_view_entries(&repository_view_entries)
|
||||||
|
.map_err(RrdpSyncError::Storage)?;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@ -1531,8 +1594,8 @@ fn fetch_snapshot_into_tempfile(
|
|||||||
snapshot_uri: &str,
|
snapshot_uri: &str,
|
||||||
expected_hash_sha256: &[u8; 32],
|
expected_hash_sha256: &[u8; 32],
|
||||||
) -> Result<(tempfile::NamedTempFile, u64), RrdpSyncError> {
|
) -> Result<(tempfile::NamedTempFile, u64), RrdpSyncError> {
|
||||||
let mut tmp =
|
let mut tmp = tempfile::NamedTempFile::new()
|
||||||
tempfile::NamedTempFile::new().map_err(|e| RrdpSyncError::Fetch(format!("tempfile create failed: {e}")))?;
|
.map_err(|e| RrdpSyncError::Fetch(format!("tempfile create failed: {e}")))?;
|
||||||
let mut spool = SnapshotSpoolWriter::new(tmp.as_file_mut());
|
let mut spool = SnapshotSpoolWriter::new(tmp.as_file_mut());
|
||||||
let bytes_written = match fetcher.fetch_to_writer(snapshot_uri, &mut spool) {
|
let bytes_written = match fetcher.fetch_to_writer(snapshot_uri, &mut spool) {
|
||||||
Ok(bytes) => bytes,
|
Ok(bytes) => bytes,
|
||||||
@ -1988,6 +2051,7 @@ mod tests {
|
|||||||
let applied = apply_delta(
|
let applied = apply_delta(
|
||||||
&store,
|
&store,
|
||||||
notif_uri,
|
notif_uri,
|
||||||
|
None,
|
||||||
&delta,
|
&delta,
|
||||||
expected_hash,
|
expected_hash,
|
||||||
Uuid::parse_str(sid).unwrap(),
|
Uuid::parse_str(sid).unwrap(),
|
||||||
@ -2083,7 +2147,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
let mut wrong = [0u8; 32];
|
let mut wrong = [0u8; 32];
|
||||||
wrong[0] = 1;
|
wrong[0] = 1;
|
||||||
let err = apply_delta(&store, notif_uri, &delta, wrong, sid, 1).unwrap_err();
|
let err = apply_delta(&store, notif_uri, None, &delta, wrong, sid, 1).unwrap_err();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
err,
|
err,
|
||||||
RrdpSyncError::Rrdp(RrdpError::DeltaHashMismatch)
|
RrdpSyncError::Rrdp(RrdpError::DeltaHashMismatch)
|
||||||
@ -2109,7 +2173,7 @@ mod tests {
|
|||||||
let mut expected_hash = [0u8; 32];
|
let mut expected_hash = [0u8; 32];
|
||||||
expected_hash.copy_from_slice(delta_hash.as_slice());
|
expected_hash.copy_from_slice(delta_hash.as_slice());
|
||||||
|
|
||||||
let err = apply_delta(&store, notif_uri, &delta, expected_hash, sid, 1).unwrap_err();
|
let err = apply_delta(&store, notif_uri, None, &delta, expected_hash, sid, 1).unwrap_err();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
err,
|
err,
|
||||||
RrdpSyncError::Rrdp(RrdpError::DeltaTargetNotFromRepository { .. })
|
RrdpSyncError::Rrdp(RrdpError::DeltaTargetNotFromRepository { .. })
|
||||||
@ -2150,6 +2214,7 @@ mod tests {
|
|||||||
let err = apply_delta(
|
let err = apply_delta(
|
||||||
&store,
|
&store,
|
||||||
notif_uri,
|
notif_uri,
|
||||||
|
None,
|
||||||
&delta,
|
&delta,
|
||||||
expected_hash,
|
expected_hash,
|
||||||
Uuid::parse_str(sid).unwrap(),
|
Uuid::parse_str(sid).unwrap(),
|
||||||
@ -2201,6 +2266,7 @@ mod tests {
|
|||||||
let err = apply_delta(
|
let err = apply_delta(
|
||||||
&store,
|
&store,
|
||||||
notif_uri,
|
notif_uri,
|
||||||
|
None,
|
||||||
&delta,
|
&delta,
|
||||||
expected_hash,
|
expected_hash,
|
||||||
Uuid::parse_str(sid).unwrap(),
|
Uuid::parse_str(sid).unwrap(),
|
||||||
@ -2229,6 +2295,7 @@ mod tests {
|
|||||||
let err = apply_delta(
|
let err = apply_delta(
|
||||||
&store,
|
&store,
|
||||||
notif_uri,
|
notif_uri,
|
||||||
|
None,
|
||||||
&delta,
|
&delta,
|
||||||
expected_hash,
|
expected_hash,
|
||||||
Uuid::parse_str(sid).unwrap(),
|
Uuid::parse_str(sid).unwrap(),
|
||||||
@ -2263,7 +2330,8 @@ mod tests {
|
|||||||
|
|
||||||
// Session mismatch.
|
// Session mismatch.
|
||||||
let other_sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440001").unwrap();
|
let other_sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440001").unwrap();
|
||||||
let err = apply_delta(&store, notif_uri, &delta, expected_hash, other_sid, 2).unwrap_err();
|
let err =
|
||||||
|
apply_delta(&store, notif_uri, None, &delta, expected_hash, other_sid, 2).unwrap_err();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
err,
|
err,
|
||||||
RrdpSyncError::Rrdp(RrdpError::DeltaSessionIdMismatch { .. })
|
RrdpSyncError::Rrdp(RrdpError::DeltaSessionIdMismatch { .. })
|
||||||
@ -2273,6 +2341,7 @@ mod tests {
|
|||||||
let err = apply_delta(
|
let err = apply_delta(
|
||||||
&store,
|
&store,
|
||||||
notif_uri,
|
notif_uri,
|
||||||
|
None,
|
||||||
&delta,
|
&delta,
|
||||||
expected_hash,
|
expected_hash,
|
||||||
Uuid::parse_str(sid).unwrap(),
|
Uuid::parse_str(sid).unwrap(),
|
||||||
@ -2669,7 +2738,7 @@ mod tests {
|
|||||||
let got_sid = "550e8400-e29b-41d4-a716-446655440001";
|
let got_sid = "550e8400-e29b-41d4-a716-446655440001";
|
||||||
|
|
||||||
let snapshot = snapshot_xml(got_sid, 2, &[("rsync://example.net/repo/a.mft", b"x")]);
|
let snapshot = snapshot_xml(got_sid, 2, &[("rsync://example.net/repo/a.mft", b"x")]);
|
||||||
let err = apply_snapshot(&store, notif_uri, &snapshot, expected_sid, 2).unwrap_err();
|
let err = apply_snapshot(&store, notif_uri, None, &snapshot, expected_sid, 2).unwrap_err();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
err,
|
err,
|
||||||
RrdpSyncError::Rrdp(RrdpError::SnapshotSessionIdMismatch { .. })
|
RrdpSyncError::Rrdp(RrdpError::SnapshotSessionIdMismatch { .. })
|
||||||
@ -2680,7 +2749,7 @@ mod tests {
|
|||||||
3,
|
3,
|
||||||
&[("rsync://example.net/repo/a.mft", b"x")],
|
&[("rsync://example.net/repo/a.mft", b"x")],
|
||||||
);
|
);
|
||||||
let err = apply_snapshot(&store, notif_uri, &snapshot, expected_sid, 2).unwrap_err();
|
let err = apply_snapshot(&store, notif_uri, None, &snapshot, expected_sid, 2).unwrap_err();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
err,
|
err,
|
||||||
RrdpSyncError::Rrdp(RrdpError::SnapshotSerialMismatch { .. })
|
RrdpSyncError::Rrdp(RrdpError::SnapshotSerialMismatch { .. })
|
||||||
@ -2704,7 +2773,7 @@ mod tests {
|
|||||||
r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{sid}" serial="1"><publish>AA==</publish></snapshot>"#
|
r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{sid}" serial="1"><publish>AA==</publish></snapshot>"#
|
||||||
)
|
)
|
||||||
.into_bytes();
|
.into_bytes();
|
||||||
let err = apply_snapshot(&store, notif_uri, &xml, sid, 1).unwrap_err();
|
let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
err,
|
err,
|
||||||
RrdpSyncError::Rrdp(RrdpError::PublishUriMissing)
|
RrdpSyncError::Rrdp(RrdpError::PublishUriMissing)
|
||||||
@ -2715,7 +2784,7 @@ mod tests {
|
|||||||
r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{sid}" serial="1"><publish uri="rsync://example.net/repo/a.cer"></publish></snapshot>"#
|
r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{sid}" serial="1"><publish uri="rsync://example.net/repo/a.cer"></publish></snapshot>"#
|
||||||
)
|
)
|
||||||
.into_bytes();
|
.into_bytes();
|
||||||
let err = apply_snapshot(&store, notif_uri, &xml, sid, 1).unwrap_err();
|
let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
err,
|
err,
|
||||||
RrdpSyncError::Rrdp(RrdpError::PublishContentMissing)
|
RrdpSyncError::Rrdp(RrdpError::PublishContentMissing)
|
||||||
@ -2726,7 +2795,7 @@ mod tests {
|
|||||||
r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{sid}" serial="1"><publish uri="rsync://example.net/repo/a.cer">!!!</publish></snapshot>"#
|
r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{sid}" serial="1"><publish uri="rsync://example.net/repo/a.cer">!!!</publish></snapshot>"#
|
||||||
)
|
)
|
||||||
.into_bytes();
|
.into_bytes();
|
||||||
let err = apply_snapshot(&store, notif_uri, &xml, sid, 1).unwrap_err();
|
let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
err,
|
err,
|
||||||
RrdpSyncError::Rrdp(RrdpError::PublishBase64(_))
|
RrdpSyncError::Rrdp(RrdpError::PublishBase64(_))
|
||||||
@ -2741,9 +2810,8 @@ mod tests {
|
|||||||
let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
|
let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap();
|
||||||
|
|
||||||
let total = RRDP_SNAPSHOT_APPLY_BATCH_SIZE + 7;
|
let total = RRDP_SNAPSHOT_APPLY_BATCH_SIZE + 7;
|
||||||
let mut xml = format!(
|
let mut xml =
|
||||||
r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{sid}" serial="1">"#
|
format!(r#"<snapshot xmlns="{RRDP_XMLNS}" version="1" session_id="{sid}" serial="1">"#);
|
||||||
);
|
|
||||||
for i in 0..total {
|
for i in 0..total {
|
||||||
let uri = format!("rsync://example.net/repo/{i:04}.roa");
|
let uri = format!("rsync://example.net/repo/{i:04}.roa");
|
||||||
let bytes = format!("payload-{i}").into_bytes();
|
let bytes = format!("payload-{i}").into_bytes();
|
||||||
@ -2752,8 +2820,8 @@ mod tests {
|
|||||||
}
|
}
|
||||||
xml.push_str("</snapshot>");
|
xml.push_str("</snapshot>");
|
||||||
|
|
||||||
let published =
|
let published = apply_snapshot(&store, notif_uri, None, xml.as_bytes(), sid, 1)
|
||||||
apply_snapshot(&store, notif_uri, xml.as_bytes(), sid, 1).expect("apply snapshot");
|
.expect("apply snapshot");
|
||||||
assert_eq!(published, total);
|
assert_eq!(published, total);
|
||||||
|
|
||||||
for idx in [0usize, RRDP_SNAPSHOT_APPLY_BATCH_SIZE - 1, total - 1] {
|
for idx in [0usize, RRDP_SNAPSHOT_APPLY_BATCH_SIZE - 1, total - 1] {
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
use crate::blob_store::RawObjectStore;
|
use crate::blob_store::RawObjectStore;
|
||||||
|
use crate::current_repo_index::CurrentRepoIndexHandle;
|
||||||
use crate::data_model::manifest::{ManifestDecodeError, ManifestObject, ManifestValidateError};
|
use crate::data_model::manifest::{ManifestDecodeError, ManifestObject, ManifestValidateError};
|
||||||
use crate::data_model::signed_object::SignedObjectVerifyError;
|
use crate::data_model::signed_object::SignedObjectVerifyError;
|
||||||
use crate::policy::{CaFailedFetchPolicy, Policy};
|
use crate::policy::{CaFailedFetchPolicy, Policy};
|
||||||
@ -299,11 +300,53 @@ pub fn process_manifest_publication_point_fresh_after_repo_sync(
|
|||||||
repo_sync_ok: bool,
|
repo_sync_ok: bool,
|
||||||
repo_sync_error: Option<&str>,
|
repo_sync_error: Option<&str>,
|
||||||
) -> Result<FreshValidatedPublicationPoint, ManifestFreshError> {
|
) -> Result<FreshValidatedPublicationPoint, ManifestFreshError> {
|
||||||
if repo_sync_ok {
|
process_manifest_publication_point_fresh_after_repo_sync_with_timing(
|
||||||
try_build_fresh_publication_point(
|
|
||||||
store,
|
store,
|
||||||
manifest_rsync_uri,
|
manifest_rsync_uri,
|
||||||
publication_point_rsync_uri,
|
publication_point_rsync_uri,
|
||||||
|
None,
|
||||||
|
issuer_ca_der,
|
||||||
|
issuer_ca_rsync_uri,
|
||||||
|
validation_time,
|
||||||
|
repo_sync_ok,
|
||||||
|
repo_sync_error,
|
||||||
|
)
|
||||||
|
.map(|(fresh, _timing)| fresh)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Default)]
|
||||||
|
pub struct FreshPublicationPointTimingBreakdown {
|
||||||
|
pub manifest_load_ms: u64,
|
||||||
|
pub manifest_decode_ms: u64,
|
||||||
|
pub replay_guard_ms: u64,
|
||||||
|
pub manifest_entries_ms: u64,
|
||||||
|
pub pack_files_ms: u64,
|
||||||
|
pub ee_path_validate_ms: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn process_manifest_publication_point_fresh_after_repo_sync_with_timing(
|
||||||
|
store: &RocksStore,
|
||||||
|
manifest_rsync_uri: &str,
|
||||||
|
publication_point_rsync_uri: &str,
|
||||||
|
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||||
|
issuer_ca_der: &[u8],
|
||||||
|
issuer_ca_rsync_uri: Option<&str>,
|
||||||
|
validation_time: time::OffsetDateTime,
|
||||||
|
repo_sync_ok: bool,
|
||||||
|
repo_sync_error: Option<&str>,
|
||||||
|
) -> Result<
|
||||||
|
(
|
||||||
|
FreshValidatedPublicationPoint,
|
||||||
|
FreshPublicationPointTimingBreakdown,
|
||||||
|
),
|
||||||
|
ManifestFreshError,
|
||||||
|
> {
|
||||||
|
if repo_sync_ok {
|
||||||
|
try_build_fresh_publication_point_with_timing(
|
||||||
|
store,
|
||||||
|
manifest_rsync_uri,
|
||||||
|
publication_point_rsync_uri,
|
||||||
|
current_repo_index,
|
||||||
issuer_ca_der,
|
issuer_ca_der,
|
||||||
issuer_ca_rsync_uri,
|
issuer_ca_rsync_uri,
|
||||||
validation_time,
|
validation_time,
|
||||||
@ -469,10 +512,10 @@ pub fn load_current_instance_vcir_publication_point(
|
|||||||
if !seen.insert(uri.clone()) {
|
if !seen.insert(uri.clone()) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let entry_bytes = store
|
let entry_bytes = store.get_blob_bytes(&artifact.sha256)?.ok_or_else(|| {
|
||||||
.get_blob_bytes(&artifact.sha256)?
|
ManifestReuseError::MissingArtifactRaw {
|
||||||
.ok_or_else(|| ManifestReuseError::MissingArtifactRaw {
|
|
||||||
rsync_uri: uri.clone(),
|
rsync_uri: uri.clone(),
|
||||||
|
}
|
||||||
})?;
|
})?;
|
||||||
files.push(PackFile::from_bytes_compute_sha256(uri, entry_bytes));
|
files.push(PackFile::from_bytes_compute_sha256(uri, entry_bytes));
|
||||||
}
|
}
|
||||||
@ -541,6 +584,36 @@ pub(crate) fn try_build_fresh_publication_point(
|
|||||||
issuer_ca_rsync_uri: Option<&str>,
|
issuer_ca_rsync_uri: Option<&str>,
|
||||||
validation_time: time::OffsetDateTime,
|
validation_time: time::OffsetDateTime,
|
||||||
) -> Result<FreshValidatedPublicationPoint, ManifestFreshError> {
|
) -> Result<FreshValidatedPublicationPoint, ManifestFreshError> {
|
||||||
|
try_build_fresh_publication_point_with_timing(
|
||||||
|
store,
|
||||||
|
manifest_rsync_uri,
|
||||||
|
publication_point_rsync_uri,
|
||||||
|
None,
|
||||||
|
issuer_ca_der,
|
||||||
|
issuer_ca_rsync_uri,
|
||||||
|
validation_time,
|
||||||
|
)
|
||||||
|
.map(|(fresh, _timing)| fresh)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn try_build_fresh_publication_point_with_timing(
|
||||||
|
store: &RocksStore,
|
||||||
|
manifest_rsync_uri: &str,
|
||||||
|
publication_point_rsync_uri: &str,
|
||||||
|
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||||
|
issuer_ca_der: &[u8],
|
||||||
|
issuer_ca_rsync_uri: Option<&str>,
|
||||||
|
validation_time: time::OffsetDateTime,
|
||||||
|
) -> Result<
|
||||||
|
(
|
||||||
|
FreshValidatedPublicationPoint,
|
||||||
|
FreshPublicationPointTimingBreakdown,
|
||||||
|
),
|
||||||
|
ManifestFreshError,
|
||||||
|
> {
|
||||||
|
let mut timing = FreshPublicationPointTimingBreakdown::default();
|
||||||
|
let current_index_guard = current_repo_index.and_then(|handle| handle.lock().ok());
|
||||||
|
|
||||||
if !rsync_uri_is_under_publication_point(manifest_rsync_uri, publication_point_rsync_uri) {
|
if !rsync_uri_is_under_publication_point(manifest_rsync_uri, publication_point_rsync_uri) {
|
||||||
return Err(ManifestFreshError::ManifestOutsidePublicationPoint {
|
return Err(ManifestFreshError::ManifestOutsidePublicationPoint {
|
||||||
manifest_rsync_uri: manifest_rsync_uri.to_string(),
|
manifest_rsync_uri: manifest_rsync_uri.to_string(),
|
||||||
@ -548,17 +621,37 @@ pub(crate) fn try_build_fresh_publication_point(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let manifest_bytes = store
|
let manifest_load_started = std::time::Instant::now();
|
||||||
|
let manifest_bytes = if let Some(index) = current_index_guard.as_ref() {
|
||||||
|
let current = index.get_by_uri(manifest_rsync_uri).ok_or_else(|| {
|
||||||
|
ManifestFreshError::MissingManifest {
|
||||||
|
manifest_rsync_uri: manifest_rsync_uri.to_string(),
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
store
|
||||||
|
.get_blob_bytes(¤t.current_hash_hex)
|
||||||
|
.map_err(|e| ManifestFreshError::MissingManifest {
|
||||||
|
manifest_rsync_uri: format!("{manifest_rsync_uri} ({e})"),
|
||||||
|
})?
|
||||||
|
.ok_or_else(|| ManifestFreshError::MissingManifest {
|
||||||
|
manifest_rsync_uri: manifest_rsync_uri.to_string(),
|
||||||
|
})?
|
||||||
|
} else {
|
||||||
|
store
|
||||||
.load_current_object_bytes_by_uri(manifest_rsync_uri)
|
.load_current_object_bytes_by_uri(manifest_rsync_uri)
|
||||||
.map_err(|e| ManifestFreshError::MissingManifest {
|
.map_err(|e| ManifestFreshError::MissingManifest {
|
||||||
manifest_rsync_uri: format!("{manifest_rsync_uri} ({e})"),
|
manifest_rsync_uri: format!("{manifest_rsync_uri} ({e})"),
|
||||||
})?
|
})?
|
||||||
.ok_or_else(|| ManifestFreshError::MissingManifest {
|
.ok_or_else(|| ManifestFreshError::MissingManifest {
|
||||||
manifest_rsync_uri: manifest_rsync_uri.to_string(),
|
manifest_rsync_uri: manifest_rsync_uri.to_string(),
|
||||||
})?;
|
})?
|
||||||
|
};
|
||||||
|
timing.manifest_load_ms = manifest_load_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
|
let manifest_decode_started = std::time::Instant::now();
|
||||||
let manifest =
|
let manifest =
|
||||||
decode_and_validate_manifest_with_current_time(&manifest_bytes, validation_time)?;
|
decode_and_validate_manifest_with_current_time(&manifest_bytes, validation_time)?;
|
||||||
|
timing.manifest_decode_ms = manifest_decode_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
let this_update = manifest
|
let this_update = manifest
|
||||||
.manifest
|
.manifest
|
||||||
@ -582,6 +675,7 @@ pub(crate) fn try_build_fresh_publication_point(
|
|||||||
// - If manifestNumber is lower, treat as rollback and reject.
|
// - If manifestNumber is lower, treat as rollback and reject.
|
||||||
// - If manifestNumber is higher, require thisUpdate to be more recent than the previously
|
// - If manifestNumber is higher, require thisUpdate to be more recent than the previously
|
||||||
// validated thisUpdate.
|
// validated thisUpdate.
|
||||||
|
let replay_guard_started = std::time::Instant::now();
|
||||||
if let Some(old_vcir) = store.get_vcir(manifest_rsync_uri).ok().flatten() {
|
if let Some(old_vcir) = store.get_vcir(manifest_rsync_uri).ok().flatten() {
|
||||||
if old_vcir.manifest_rsync_uri == manifest_rsync_uri {
|
if old_vcir.manifest_rsync_uri == manifest_rsync_uri {
|
||||||
let new_num = manifest.manifest.manifest_number.bytes_be.as_slice();
|
let new_num = manifest.manifest.manifest_number.bytes_be.as_slice();
|
||||||
@ -633,34 +727,79 @@ pub(crate) fn try_build_fresh_publication_point(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
timing.replay_guard_ms = replay_guard_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
|
let manifest_entries_started = std::time::Instant::now();
|
||||||
let entries = manifest
|
let entries = manifest
|
||||||
.manifest
|
.manifest
|
||||||
.parse_files()
|
.parse_files()
|
||||||
.map_err(ManifestDecodeError::Validate)?;
|
.map_err(ManifestDecodeError::Validate)?;
|
||||||
|
timing.manifest_entries_ms = manifest_entries_started.elapsed().as_millis() as u64;
|
||||||
let mut files = Vec::with_capacity(manifest.manifest.file_count());
|
let mut files = Vec::with_capacity(manifest.manifest.file_count());
|
||||||
|
let pack_files_started = std::time::Instant::now();
|
||||||
|
let external_raw_store = store.external_raw_store_ref().cloned().map(std::sync::Arc::new);
|
||||||
for entry in &entries {
|
for entry in &entries {
|
||||||
let rsync_uri =
|
let rsync_uri =
|
||||||
join_rsync_dir_and_file(publication_point_rsync_uri, entry.file_name.as_str());
|
join_rsync_dir_and_file(publication_point_rsync_uri, entry.file_name.as_str());
|
||||||
let bytes = store
|
let current_object = if let Some(index) = current_index_guard.as_ref() {
|
||||||
.load_current_object_bytes_by_uri(&rsync_uri)
|
let current =
|
||||||
|
index
|
||||||
|
.get_by_uri(&rsync_uri)
|
||||||
|
.ok_or_else(|| ManifestFreshError::MissingFile {
|
||||||
|
rsync_uri: rsync_uri.clone(),
|
||||||
|
})?;
|
||||||
|
crate::storage::CurrentObjectWithHash {
|
||||||
|
current_hash_hex: current.current_hash_hex.clone(),
|
||||||
|
current_hash: current.current_hash,
|
||||||
|
bytes: Vec::new(),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
store
|
||||||
|
.load_current_object_with_hash_by_uri(&rsync_uri)
|
||||||
.map_err(|_e| ManifestFreshError::MissingFile {
|
.map_err(|_e| ManifestFreshError::MissingFile {
|
||||||
rsync_uri: rsync_uri.clone(),
|
rsync_uri: rsync_uri.clone(),
|
||||||
})?
|
})?
|
||||||
.ok_or_else(|| ManifestFreshError::MissingFile {
|
.ok_or_else(|| ManifestFreshError::MissingFile {
|
||||||
rsync_uri: rsync_uri.clone(),
|
rsync_uri: rsync_uri.clone(),
|
||||||
})?;
|
})?
|
||||||
|
};
|
||||||
|
|
||||||
let computed = sha2::Sha256::digest(&bytes);
|
if current_object.current_hash != entry.hash_bytes {
|
||||||
if computed.as_slice() != entry.hash_bytes.as_ref() {
|
|
||||||
return Err(ManifestFreshError::HashMismatch { rsync_uri });
|
return Err(ManifestFreshError::HashMismatch { rsync_uri });
|
||||||
}
|
}
|
||||||
|
|
||||||
files.push(PackFile::from_bytes_compute_sha256(rsync_uri, bytes));
|
if let (Some(_), Some(raw_store)) = (current_index_guard.as_ref(), external_raw_store.as_ref()) {
|
||||||
|
files.push(PackFile::from_lazy_external_raw_store(
|
||||||
|
rsync_uri,
|
||||||
|
current_object.current_hash_hex,
|
||||||
|
current_object.current_hash,
|
||||||
|
raw_store.clone(),
|
||||||
|
));
|
||||||
|
} else {
|
||||||
|
let bytes = if current_object.bytes.is_empty() {
|
||||||
|
store
|
||||||
|
.get_blob_bytes(¤t_object.current_hash_hex)
|
||||||
|
.map_err(|_e| ManifestFreshError::MissingFile {
|
||||||
|
rsync_uri: rsync_uri.clone(),
|
||||||
|
})?
|
||||||
|
.ok_or_else(|| ManifestFreshError::MissingFile {
|
||||||
|
rsync_uri: rsync_uri.clone(),
|
||||||
|
})?
|
||||||
|
} else {
|
||||||
|
current_object.bytes
|
||||||
|
};
|
||||||
|
files.push(PackFile::from_bytes_with_sha256(
|
||||||
|
rsync_uri,
|
||||||
|
bytes,
|
||||||
|
current_object.current_hash,
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
timing.pack_files_ms = pack_files_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
// RFC 6488 §3: manifest (signed object) validity includes a valid EE cert path.
|
// RFC 6488 §3: manifest (signed object) validity includes a valid EE cert path.
|
||||||
// We validate this after §6.4/§6.5 so the issuer CRL can be selected from the publication point.
|
// We validate this after §6.4/§6.5 so the issuer CRL can be selected from the publication point.
|
||||||
|
let ee_path_validate_started = std::time::Instant::now();
|
||||||
validate_manifest_embedded_ee_cert_path(
|
validate_manifest_embedded_ee_cert_path(
|
||||||
&manifest,
|
&manifest,
|
||||||
&files,
|
&files,
|
||||||
@ -668,8 +807,10 @@ pub(crate) fn try_build_fresh_publication_point(
|
|||||||
issuer_ca_rsync_uri,
|
issuer_ca_rsync_uri,
|
||||||
validation_time,
|
validation_time,
|
||||||
)?;
|
)?;
|
||||||
|
timing.ee_path_validate_ms = ee_path_validate_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
Ok(FreshValidatedPublicationPoint {
|
Ok((
|
||||||
|
FreshValidatedPublicationPoint {
|
||||||
manifest_rsync_uri: manifest_rsync_uri.to_string(),
|
manifest_rsync_uri: manifest_rsync_uri.to_string(),
|
||||||
publication_point_rsync_uri: publication_point_rsync_uri.to_string(),
|
publication_point_rsync_uri: publication_point_rsync_uri.to_string(),
|
||||||
manifest_number_be: manifest.manifest.manifest_number.bytes_be.clone(),
|
manifest_number_be: manifest.manifest.manifest_number.bytes_be.clone(),
|
||||||
@ -678,7 +819,9 @@ pub(crate) fn try_build_fresh_publication_point(
|
|||||||
verified_at: PackTime::from_utc_offset_datetime(now),
|
verified_at: PackTime::from_utc_offset_datetime(now),
|
||||||
manifest_bytes,
|
manifest_bytes,
|
||||||
files,
|
files,
|
||||||
})
|
},
|
||||||
|
timing,
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn cmp_minimal_be_unsigned(a: &[u8], b: &[u8]) -> Ordering {
|
fn cmp_minimal_be_unsigned(a: &[u8], b: &[u8]) -> Ordering {
|
||||||
@ -735,10 +878,13 @@ fn validate_manifest_embedded_ee_cert_path(
|
|||||||
for u in crldp_uris {
|
for u in crldp_uris {
|
||||||
let s = u.as_str();
|
let s = u.as_str();
|
||||||
if let Some(f) = crl_files.iter().find(|f| f.rsync_uri == s) {
|
if let Some(f) = crl_files.iter().find(|f| f.rsync_uri == s) {
|
||||||
|
let crl_bytes = f
|
||||||
|
.bytes()
|
||||||
|
.map_err(|e| ManifestFreshError::MissingFile { rsync_uri: format!("{s} ({e})") })?;
|
||||||
let _validated = validate_ee_cert_path(
|
let _validated = validate_ee_cert_path(
|
||||||
ee_der,
|
ee_der,
|
||||||
issuer_ca_der,
|
issuer_ca_der,
|
||||||
f.bytes.as_slice(),
|
crl_bytes,
|
||||||
issuer_ca_rsync_uri,
|
issuer_ca_rsync_uri,
|
||||||
Some(f.rsync_uri.as_str()),
|
Some(f.rsync_uri.as_str()),
|
||||||
validation_time,
|
validation_time,
|
||||||
@ -759,6 +905,7 @@ fn validate_manifest_embedded_ee_cert_path(
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use crate::current_repo_index::CurrentRepoIndex;
|
||||||
use crate::data_model::manifest::ManifestObject;
|
use crate::data_model::manifest::ManifestObject;
|
||||||
use crate::storage::{
|
use crate::storage::{
|
||||||
PackFile, PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult,
|
PackFile, PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult,
|
||||||
@ -853,6 +1000,12 @@ mod tests {
|
|||||||
.expect("put repository view entry");
|
.expect("put repository view entry");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn put_raw_only(store: &RocksStore, rsync_uri: &str, bytes: Vec<u8>, object_type: &str) {
|
||||||
|
store
|
||||||
|
.put_raw_by_hash_entry(&raw_by_hash_entry(rsync_uri, bytes, object_type))
|
||||||
|
.expect("put raw_by_hash entry");
|
||||||
|
}
|
||||||
|
|
||||||
fn sample_current_instance_vcir(
|
fn sample_current_instance_vcir(
|
||||||
manifest_rsync_uri: &str,
|
manifest_rsync_uri: &str,
|
||||||
publication_point_rsync_uri: &str,
|
publication_point_rsync_uri: &str,
|
||||||
@ -1034,6 +1187,132 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn try_build_fresh_publication_point_detects_hash_mismatch_via_repository_view_hash() {
|
||||||
|
let temp = tempfile::tempdir().expect("tempdir");
|
||||||
|
let store = RocksStore::open(temp.path()).expect("open rocksdb");
|
||||||
|
let (
|
||||||
|
manifest,
|
||||||
|
manifest_bytes,
|
||||||
|
manifest_rsync_uri,
|
||||||
|
publication_point_rsync_uri,
|
||||||
|
validation_time,
|
||||||
|
) = load_manifest_fixture();
|
||||||
|
put_current_object(&store, &manifest_rsync_uri, manifest_bytes, "mft");
|
||||||
|
|
||||||
|
let non_crl_entries = manifest
|
||||||
|
.manifest
|
||||||
|
.parse_files()
|
||||||
|
.expect("parse files")
|
||||||
|
.into_iter()
|
||||||
|
.filter(|entry| !entry.file_name.ends_with(".crl"))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let first = &non_crl_entries[0];
|
||||||
|
let second = &non_crl_entries[1];
|
||||||
|
|
||||||
|
let first_uri = format!("{publication_point_rsync_uri}{}", first.file_name);
|
||||||
|
let second_path = manifest_fixture_path()
|
||||||
|
.parent()
|
||||||
|
.unwrap()
|
||||||
|
.join(second.file_name.as_str());
|
||||||
|
let wrong_bytes = std::fs::read(&second_path).expect("read wrong fixture file");
|
||||||
|
let object_type = first_uri.rsplit('.').next().unwrap_or("bin");
|
||||||
|
put_current_object(&store, &first_uri, wrong_bytes, object_type);
|
||||||
|
|
||||||
|
for entry in non_crl_entries.iter().skip(1) {
|
||||||
|
let file_path = manifest_fixture_path()
|
||||||
|
.parent()
|
||||||
|
.unwrap()
|
||||||
|
.join(entry.file_name.as_str());
|
||||||
|
let bytes = std::fs::read(&file_path).expect("read fixture file");
|
||||||
|
let rsync_uri = format!("{publication_point_rsync_uri}{}", entry.file_name);
|
||||||
|
let object_type = rsync_uri.rsplit('.').next().unwrap_or("bin");
|
||||||
|
put_current_object(&store, &rsync_uri, bytes, object_type);
|
||||||
|
}
|
||||||
|
|
||||||
|
let err = try_build_fresh_publication_point(
|
||||||
|
&store,
|
||||||
|
&manifest_rsync_uri,
|
||||||
|
&publication_point_rsync_uri,
|
||||||
|
&issuer_ca_fixture_der(),
|
||||||
|
Some(issuer_ca_rsync_uri()),
|
||||||
|
validation_time,
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
matches!(err, ManifestFreshError::HashMismatch { .. }),
|
||||||
|
"{err}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn try_build_fresh_publication_point_uses_current_repo_index_without_repository_view() {
|
||||||
|
let temp = tempfile::tempdir().expect("tempdir");
|
||||||
|
let store = RocksStore::open(temp.path()).expect("open rocksdb");
|
||||||
|
let (
|
||||||
|
manifest,
|
||||||
|
manifest_bytes,
|
||||||
|
manifest_rsync_uri,
|
||||||
|
publication_point_rsync_uri,
|
||||||
|
validation_time,
|
||||||
|
) = load_manifest_fixture();
|
||||||
|
|
||||||
|
put_raw_only(&store, &manifest_rsync_uri, manifest_bytes.clone(), "mft");
|
||||||
|
let current_index = CurrentRepoIndex::shared();
|
||||||
|
let mut entries = vec![crate::storage::RepositoryViewEntry {
|
||||||
|
rsync_uri: manifest_rsync_uri.clone(),
|
||||||
|
current_hash: Some(hex::encode(sha2::Sha256::digest(&manifest_bytes))),
|
||||||
|
repository_source: Some("https://example.test/notification.xml".to_string()),
|
||||||
|
object_type: Some("mft".to_string()),
|
||||||
|
state: crate::storage::RepositoryViewState::Present,
|
||||||
|
}];
|
||||||
|
|
||||||
|
for entry in manifest.manifest.parse_files().expect("parse files") {
|
||||||
|
let file_path = manifest_fixture_path()
|
||||||
|
.parent()
|
||||||
|
.unwrap()
|
||||||
|
.join(entry.file_name.as_str());
|
||||||
|
let bytes = std::fs::read(&file_path).expect("read fixture file");
|
||||||
|
let rsync_uri = format!("{publication_point_rsync_uri}{}", entry.file_name);
|
||||||
|
let object_type = rsync_uri.rsplit('.').next().unwrap_or("bin").to_string();
|
||||||
|
put_raw_only(&store, &rsync_uri, bytes.clone(), &object_type);
|
||||||
|
entries.push(crate::storage::RepositoryViewEntry {
|
||||||
|
rsync_uri,
|
||||||
|
current_hash: Some(hex::encode(sha2::Sha256::digest(&bytes))),
|
||||||
|
repository_source: Some("https://example.test/notification.xml".to_string()),
|
||||||
|
object_type: Some(object_type),
|
||||||
|
state: crate::storage::RepositoryViewState::Present,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
current_index
|
||||||
|
.lock()
|
||||||
|
.expect("index lock")
|
||||||
|
.apply_repository_view_entries(&entries)
|
||||||
|
.expect("apply current index");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.get_repository_view_entry(&manifest_rsync_uri)
|
||||||
|
.expect("get repository view")
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
|
||||||
|
let (fresh, _timing) = try_build_fresh_publication_point_with_timing(
|
||||||
|
&store,
|
||||||
|
&manifest_rsync_uri,
|
||||||
|
&publication_point_rsync_uri,
|
||||||
|
Some(¤t_index),
|
||||||
|
&issuer_ca_fixture_der(),
|
||||||
|
Some(issuer_ca_rsync_uri()),
|
||||||
|
validation_time,
|
||||||
|
)
|
||||||
|
.expect("fresh publication point via current index");
|
||||||
|
|
||||||
|
assert_eq!(fresh.manifest_rsync_uri, manifest_rsync_uri);
|
||||||
|
assert_eq!(fresh.files.len(), manifest.manifest.file_count());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn validate_manifest_embedded_ee_cert_path_rejects_missing_crl_files() {
|
fn validate_manifest_embedded_ee_cert_path_rejects_missing_crl_files() {
|
||||||
let (manifest, _, _, publication_point_rsync_uri, validation_time) =
|
let (manifest, _, _, publication_point_rsync_uri, validation_time) =
|
||||||
@ -1139,7 +1418,7 @@ mod tests {
|
|||||||
assert_eq!(point.manifest_bytes, manifest_bytes);
|
assert_eq!(point.manifest_bytes, manifest_bytes);
|
||||||
assert_eq!(point.files.len(), 1);
|
assert_eq!(point.files.len(), 1);
|
||||||
assert_eq!(point.files[0].rsync_uri, locked_uri);
|
assert_eq!(point.files[0].rsync_uri, locked_uri);
|
||||||
assert_eq!(point.files[0].bytes, locked_bytes);
|
assert_eq!(point.files[0].bytes_cloned().expect("locked bytes"), locked_bytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@ -218,9 +218,12 @@ pub fn process_publication_point_for_issuer<P: PublicationPointData>(
|
|||||||
.iter()
|
.iter()
|
||||||
.filter(|f| f.rsync_uri.ends_with(".crl"))
|
.filter(|f| f.rsync_uri.ends_with(".crl"))
|
||||||
.map(|f| {
|
.map(|f| {
|
||||||
|
let bytes = f
|
||||||
|
.bytes_cloned()
|
||||||
|
.expect("snapshot CRL bytes must be loadable");
|
||||||
(
|
(
|
||||||
f.rsync_uri.clone(),
|
f.rsync_uri.clone(),
|
||||||
CachedIssuerCrl::Pending(f.bytes.clone()),
|
CachedIssuerCrl::Pending(bytes),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
@ -518,6 +521,9 @@ pub fn process_publication_point_snapshot_for_issuer(
|
|||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
enum ObjectValidateError {
|
enum ObjectValidateError {
|
||||||
|
#[error("object bytes load failed: {0}")]
|
||||||
|
BytesLoad(String),
|
||||||
|
|
||||||
#[error("ROA decode failed: {0}")]
|
#[error("ROA decode failed: {0}")]
|
||||||
RoaDecode(#[from] RoaDecodeError),
|
RoaDecode(#[from] RoaDecodeError),
|
||||||
|
|
||||||
@ -584,7 +590,7 @@ fn process_roa_with_issuer(
|
|||||||
let _decode = timing
|
let _decode = timing
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|t| t.span_phase("objects_roa_decode_and_validate_total"));
|
.map(|t| t.span_phase("objects_roa_decode_and_validate_total"));
|
||||||
let roa = RoaObject::decode_der(&file.bytes)?;
|
let roa = RoaObject::decode_der(file.bytes().map_err(ObjectValidateError::BytesLoad)?)?;
|
||||||
drop(_decode);
|
drop(_decode);
|
||||||
|
|
||||||
let _ee_profile = timing
|
let _ee_profile = timing
|
||||||
@ -692,7 +698,7 @@ fn process_aspa_with_issuer(
|
|||||||
let _decode = timing
|
let _decode = timing
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|t| t.span_phase("objects_aspa_decode_and_validate_total"));
|
.map(|t| t.span_phase("objects_aspa_decode_and_validate_total"));
|
||||||
let aspa = AspaObject::decode_der(&file.bytes)?;
|
let aspa = AspaObject::decode_der(file.bytes().map_err(ObjectValidateError::BytesLoad)?)?;
|
||||||
drop(_decode);
|
drop(_decode);
|
||||||
|
|
||||||
let _ee_profile = timing
|
let _ee_profile = timing
|
||||||
|
|||||||
@ -73,6 +73,7 @@ pub fn run_publication_point_once(
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: false,
|
rsync_dedup: false,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@ -3,6 +3,7 @@ use url::Url;
|
|||||||
use crate::analysis::timing::TimingHandle;
|
use crate::analysis::timing::TimingHandle;
|
||||||
use crate::audit::PublicationPointAudit;
|
use crate::audit::PublicationPointAudit;
|
||||||
use crate::audit_downloads::DownloadLogHandle;
|
use crate::audit_downloads::DownloadLogHandle;
|
||||||
|
use crate::current_repo_index::{CurrentRepoIndexHandle, CurrentRepoObject};
|
||||||
use crate::data_model::ta::TrustAnchor;
|
use crate::data_model::ta::TrustAnchor;
|
||||||
use crate::parallel::config::ParallelPhase1Config;
|
use crate::parallel::config::ParallelPhase1Config;
|
||||||
use crate::parallel::repo_runtime::{Phase1RepoSyncRuntime, RepoSyncRuntime};
|
use crate::parallel::repo_runtime::{Phase1RepoSyncRuntime, RepoSyncRuntime};
|
||||||
@ -84,6 +85,7 @@ pub struct RunTreeFromTalAuditOutput {
|
|||||||
pub publication_points: Vec<PublicationPointAudit>,
|
pub publication_points: Vec<PublicationPointAudit>,
|
||||||
pub downloads: Vec<crate::audit::AuditDownloadEvent>,
|
pub downloads: Vec<crate::audit::AuditDownloadEvent>,
|
||||||
pub download_stats: crate::audit::AuditDownloadStats,
|
pub download_stats: crate::audit::AuditDownloadStats,
|
||||||
|
pub current_repo_objects: Vec<CurrentRepoObject>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
@ -93,6 +95,14 @@ pub struct TalRootDiscovery {
|
|||||||
pub root_handle: CaInstanceHandle,
|
pub root_handle: CaInstanceHandle,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn snapshot_current_repo_objects(
|
||||||
|
current_repo_index: Option<&CurrentRepoIndexHandle>,
|
||||||
|
) -> Vec<CurrentRepoObject> {
|
||||||
|
current_repo_index
|
||||||
|
.and_then(|handle| handle.lock().ok().map(|idx| idx.snapshot_objects()))
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
fn make_live_runner<'a>(
|
fn make_live_runner<'a>(
|
||||||
store: &'a crate::storage::RocksStore,
|
store: &'a crate::storage::RocksStore,
|
||||||
policy: &'a crate::policy::Policy,
|
policy: &'a crate::policy::Policy,
|
||||||
@ -101,6 +111,7 @@ fn make_live_runner<'a>(
|
|||||||
validation_time: time::OffsetDateTime,
|
validation_time: time::OffsetDateTime,
|
||||||
timing: Option<TimingHandle>,
|
timing: Option<TimingHandle>,
|
||||||
download_log: Option<DownloadLogHandle>,
|
download_log: Option<DownloadLogHandle>,
|
||||||
|
current_repo_index: Option<CurrentRepoIndexHandle>,
|
||||||
repo_sync_runtime: Option<Arc<dyn RepoSyncRuntime>>,
|
repo_sync_runtime: Option<Arc<dyn RepoSyncRuntime>>,
|
||||||
) -> Rpkiv1PublicationPointRunner<'a> {
|
) -> Rpkiv1PublicationPointRunner<'a> {
|
||||||
Rpkiv1PublicationPointRunner {
|
Rpkiv1PublicationPointRunner {
|
||||||
@ -117,6 +128,7 @@ fn make_live_runner<'a>(
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index,
|
||||||
repo_sync_runtime,
|
repo_sync_runtime,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -130,15 +142,17 @@ fn build_phase1_repo_sync_runtime<H, R>(
|
|||||||
timing: Option<TimingHandle>,
|
timing: Option<TimingHandle>,
|
||||||
download_log: Option<DownloadLogHandle>,
|
download_log: Option<DownloadLogHandle>,
|
||||||
tal_inputs: Vec<crate::parallel::types::TalInputSpec>,
|
tal_inputs: Vec<crate::parallel::types::TalInputSpec>,
|
||||||
) -> Result<Arc<dyn RepoSyncRuntime>, RunTreeFromTalError>
|
) -> Result<(Arc<dyn RepoSyncRuntime>, CurrentRepoIndexHandle), RunTreeFromTalError>
|
||||||
where
|
where
|
||||||
H: Fetcher + Clone + 'static,
|
H: Fetcher + Clone + 'static,
|
||||||
R: crate::fetch::rsync::RsyncFetcher + Clone + 'static,
|
R: crate::fetch::rsync::RsyncFetcher + Clone + 'static,
|
||||||
{
|
{
|
||||||
let coordinator = GlobalRunCoordinator::new(parallel_config.clone(), tal_inputs);
|
let coordinator = GlobalRunCoordinator::new(parallel_config.clone(), tal_inputs);
|
||||||
|
let current_repo_index = coordinator.current_repo_index_handle();
|
||||||
let rsync_fetcher_arc = Arc::new(rsync_fetcher.clone());
|
let rsync_fetcher_arc = Arc::new(rsync_fetcher.clone());
|
||||||
let executor = LiveRepoTransportExecutor::new(
|
let executor = LiveRepoTransportExecutor::new(
|
||||||
Arc::clone(&store),
|
Arc::clone(&store),
|
||||||
|
current_repo_index.clone(),
|
||||||
Arc::new(http_fetcher.clone()),
|
Arc::new(http_fetcher.clone()),
|
||||||
Arc::clone(&rsync_fetcher_arc),
|
Arc::clone(&rsync_fetcher_arc),
|
||||||
timing,
|
timing,
|
||||||
@ -149,12 +163,13 @@ where
|
|||||||
let resolver: Arc<dyn Fn(&str) -> String + Send + Sync> =
|
let resolver: Arc<dyn Fn(&str) -> String + Send + Sync> =
|
||||||
Arc::new(move |base: &str| rsync_fetcher_arc.dedup_key(base));
|
Arc::new(move |base: &str| rsync_fetcher_arc.dedup_key(base));
|
||||||
let _ = policy; // policy reserved for later runtime-level decisions
|
let _ = policy; // policy reserved for later runtime-level decisions
|
||||||
Ok(Arc::new(Phase1RepoSyncRuntime::new(
|
let runtime = Arc::new(Phase1RepoSyncRuntime::new(
|
||||||
coordinator,
|
coordinator,
|
||||||
pool,
|
pool,
|
||||||
resolver,
|
resolver,
|
||||||
policy.sync_preference,
|
policy.sync_preference,
|
||||||
)))
|
));
|
||||||
|
Ok((runtime, current_repo_index))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn root_discovery_from_tal_input(
|
fn root_discovery_from_tal_input(
|
||||||
@ -171,21 +186,16 @@ fn root_discovery_from_tal_input(
|
|||||||
let tal_bytes = std::fs::read(path).map_err(|e| {
|
let tal_bytes = std::fs::read(path).map_err(|e| {
|
||||||
FromTalError::TalFetch(format!("read TAL file failed: {}: {e}", path.display()))
|
FromTalError::TalFetch(format!("read TAL file failed: {}: {e}", path.display()))
|
||||||
})?;
|
})?;
|
||||||
let tal = crate::data_model::tal::Tal::decode_bytes(&tal_bytes).map_err(FromTalError::from)?;
|
let tal = crate::data_model::tal::Tal::decode_bytes(&tal_bytes)
|
||||||
|
.map_err(FromTalError::from)?;
|
||||||
discover_root_ca_instance_from_tal_with_fetchers(http_fetcher, rsync_fetcher, tal, None)
|
discover_root_ca_instance_from_tal_with_fetchers(http_fetcher, rsync_fetcher, tal, None)
|
||||||
}
|
}
|
||||||
TalSource::FilePathWithTa { tal_path, ta_path } => {
|
TalSource::FilePathWithTa { tal_path, ta_path } => {
|
||||||
let tal_bytes = std::fs::read(tal_path).map_err(|e| {
|
let tal_bytes = std::fs::read(tal_path).map_err(|e| {
|
||||||
FromTalError::TalFetch(format!(
|
FromTalError::TalFetch(format!("read TAL file failed: {}: {e}", tal_path.display()))
|
||||||
"read TAL file failed: {}: {e}",
|
|
||||||
tal_path.display()
|
|
||||||
))
|
|
||||||
})?;
|
})?;
|
||||||
let ta_der = std::fs::read(ta_path).map_err(|e| {
|
let ta_der = std::fs::read(ta_path).map_err(|e| {
|
||||||
FromTalError::TaFetch(format!(
|
FromTalError::TaFetch(format!("read TA file failed: {}: {e}", ta_path.display()))
|
||||||
"read TA file failed: {}: {e}",
|
|
||||||
ta_path.display()
|
|
||||||
))
|
|
||||||
})?;
|
})?;
|
||||||
discover_root_ca_instance_from_tal_and_ta_der(&tal_bytes, &ta_der, None)
|
discover_root_ca_instance_from_tal_and_ta_der(&tal_bytes, &ta_der, None)
|
||||||
}
|
}
|
||||||
@ -269,6 +279,7 @@ pub fn run_tree_from_tal_url_serial(
|
|||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
|
None,
|
||||||
);
|
);
|
||||||
|
|
||||||
let root = root_handle_from_trust_anchor(
|
let root = root_handle_from_trust_anchor(
|
||||||
@ -303,6 +314,7 @@ pub fn run_tree_from_tal_url_serial_audit(
|
|||||||
None,
|
None,
|
||||||
Some(download_log.clone()),
|
Some(download_log.clone()),
|
||||||
None,
|
None,
|
||||||
|
None,
|
||||||
);
|
);
|
||||||
|
|
||||||
let root = root_handle_from_trust_anchor(
|
let root = root_handle_from_trust_anchor(
|
||||||
@ -325,6 +337,7 @@ pub fn run_tree_from_tal_url_serial_audit(
|
|||||||
publication_points,
|
publication_points,
|
||||||
downloads,
|
downloads,
|
||||||
download_stats,
|
download_stats,
|
||||||
|
current_repo_objects: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -352,6 +365,7 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing(
|
|||||||
Some(timing.clone()),
|
Some(timing.clone()),
|
||||||
Some(download_log.clone()),
|
Some(download_log.clone()),
|
||||||
None,
|
None,
|
||||||
|
None,
|
||||||
);
|
);
|
||||||
|
|
||||||
let root = root_handle_from_trust_anchor(
|
let root = root_handle_from_trust_anchor(
|
||||||
@ -375,6 +389,7 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing(
|
|||||||
publication_points,
|
publication_points,
|
||||||
downloads,
|
downloads,
|
||||||
download_stats,
|
download_stats,
|
||||||
|
current_repo_objects: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -394,7 +409,7 @@ where
|
|||||||
{
|
{
|
||||||
let discovery = discover_root_ca_instance_from_tal_url(http_fetcher, tal_url)?;
|
let discovery = discover_root_ca_instance_from_tal_url(http_fetcher, tal_url)?;
|
||||||
let download_log = DownloadLogHandle::new();
|
let download_log = DownloadLogHandle::new();
|
||||||
let runtime = build_phase1_repo_sync_runtime(
|
let (runtime, current_repo_index) = build_phase1_repo_sync_runtime(
|
||||||
Arc::clone(&store),
|
Arc::clone(&store),
|
||||||
policy,
|
policy,
|
||||||
http_fetcher,
|
http_fetcher,
|
||||||
@ -404,6 +419,7 @@ where
|
|||||||
Some(download_log.clone()),
|
Some(download_log.clone()),
|
||||||
vec![TalInputSpec::from_url(tal_url.to_string())],
|
vec![TalInputSpec::from_url(tal_url.to_string())],
|
||||||
)?;
|
)?;
|
||||||
|
let current_repo_index_for_output = current_repo_index.clone();
|
||||||
let runner = make_live_runner(
|
let runner = make_live_runner(
|
||||||
store.as_ref(),
|
store.as_ref(),
|
||||||
policy,
|
policy,
|
||||||
@ -412,6 +428,7 @@ where
|
|||||||
validation_time,
|
validation_time,
|
||||||
None,
|
None,
|
||||||
Some(download_log.clone()),
|
Some(download_log.clone()),
|
||||||
|
Some(current_repo_index),
|
||||||
Some(runtime),
|
Some(runtime),
|
||||||
);
|
);
|
||||||
|
|
||||||
@ -434,6 +451,7 @@ where
|
|||||||
publication_points,
|
publication_points,
|
||||||
downloads,
|
downloads,
|
||||||
download_stats,
|
download_stats,
|
||||||
|
current_repo_objects: snapshot_current_repo_objects(Some(¤t_repo_index_for_output)),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -453,7 +471,8 @@ where
|
|||||||
H: Fetcher + Clone + 'static,
|
H: Fetcher + Clone + 'static,
|
||||||
R: crate::fetch::rsync::RsyncFetcher + Clone + 'static,
|
R: crate::fetch::rsync::RsyncFetcher + Clone + 'static,
|
||||||
{
|
{
|
||||||
let discovery = discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri)?;
|
let discovery =
|
||||||
|
discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri)?;
|
||||||
let download_log = DownloadLogHandle::new();
|
let download_log = DownloadLogHandle::new();
|
||||||
let derived_tal_id = derive_tal_id(&discovery);
|
let derived_tal_id = derive_tal_id(&discovery);
|
||||||
let tal_inputs = vec![TalInputSpec {
|
let tal_inputs = vec![TalInputSpec {
|
||||||
@ -468,7 +487,7 @@ where
|
|||||||
ta_der: ta_der.to_vec(),
|
ta_der: ta_der.to_vec(),
|
||||||
},
|
},
|
||||||
}];
|
}];
|
||||||
let runtime = build_phase1_repo_sync_runtime(
|
let (runtime, current_repo_index) = build_phase1_repo_sync_runtime(
|
||||||
Arc::clone(&store),
|
Arc::clone(&store),
|
||||||
policy,
|
policy,
|
||||||
http_fetcher,
|
http_fetcher,
|
||||||
@ -478,6 +497,7 @@ where
|
|||||||
Some(download_log.clone()),
|
Some(download_log.clone()),
|
||||||
tal_inputs,
|
tal_inputs,
|
||||||
)?;
|
)?;
|
||||||
|
let current_repo_index_for_output = current_repo_index.clone();
|
||||||
let runner = make_live_runner(
|
let runner = make_live_runner(
|
||||||
store.as_ref(),
|
store.as_ref(),
|
||||||
policy,
|
policy,
|
||||||
@ -486,6 +506,7 @@ where
|
|||||||
validation_time,
|
validation_time,
|
||||||
None,
|
None,
|
||||||
Some(download_log.clone()),
|
Some(download_log.clone()),
|
||||||
|
Some(current_repo_index),
|
||||||
Some(runtime),
|
Some(runtime),
|
||||||
);
|
);
|
||||||
|
|
||||||
@ -508,6 +529,7 @@ where
|
|||||||
publication_points,
|
publication_points,
|
||||||
downloads,
|
downloads,
|
||||||
download_stats,
|
download_stats,
|
||||||
|
current_repo_objects: snapshot_current_repo_objects(Some(¤t_repo_index_for_output)),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -526,21 +548,25 @@ where
|
|||||||
R: crate::fetch::rsync::RsyncFetcher + Clone + 'static,
|
R: crate::fetch::rsync::RsyncFetcher + Clone + 'static,
|
||||||
{
|
{
|
||||||
if tal_inputs.is_empty() {
|
if tal_inputs.is_empty() {
|
||||||
return Err(RunTreeFromTalError::Replay("multi-TAL run requires at least one TAL input".to_string()));
|
return Err(RunTreeFromTalError::Replay(
|
||||||
|
"multi-TAL run requires at least one TAL input".to_string(),
|
||||||
|
));
|
||||||
}
|
}
|
||||||
let roots = discover_multiple_roots_from_tal_inputs(&tal_inputs, http_fetcher, rsync_fetcher)?;
|
let roots = discover_multiple_roots_from_tal_inputs(&tal_inputs, http_fetcher, rsync_fetcher)?;
|
||||||
let primary = roots
|
let primary = roots.first().cloned().ok_or_else(|| {
|
||||||
.first()
|
RunTreeFromTalError::Replay("multi-TAL root discovery returned no roots".to_string())
|
||||||
.cloned()
|
})?;
|
||||||
.ok_or_else(|| RunTreeFromTalError::Replay("multi-TAL root discovery returned no roots".to_string()))?;
|
let discoveries = roots
|
||||||
let discoveries = roots.iter().map(|item| item.discovery.clone()).collect::<Vec<_>>();
|
.iter()
|
||||||
|
.map(|item| item.discovery.clone())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
let root_handles = roots
|
let root_handles = roots
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|item| item.root_handle)
|
.map(|item| item.root_handle)
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
let download_log = DownloadLogHandle::new();
|
let download_log = DownloadLogHandle::new();
|
||||||
let runtime = build_phase1_repo_sync_runtime(
|
let (runtime, current_repo_index) = build_phase1_repo_sync_runtime(
|
||||||
Arc::clone(&store),
|
Arc::clone(&store),
|
||||||
policy,
|
policy,
|
||||||
http_fetcher,
|
http_fetcher,
|
||||||
@ -550,6 +576,7 @@ where
|
|||||||
Some(download_log.clone()),
|
Some(download_log.clone()),
|
||||||
tal_inputs,
|
tal_inputs,
|
||||||
)?;
|
)?;
|
||||||
|
let current_repo_index_for_output = current_repo_index.clone();
|
||||||
let runner = make_live_runner(
|
let runner = make_live_runner(
|
||||||
store.as_ref(),
|
store.as_ref(),
|
||||||
policy,
|
policy,
|
||||||
@ -558,6 +585,7 @@ where
|
|||||||
validation_time,
|
validation_time,
|
||||||
None,
|
None,
|
||||||
Some(download_log.clone()),
|
Some(download_log.clone()),
|
||||||
|
Some(current_repo_index),
|
||||||
Some(runtime),
|
Some(runtime),
|
||||||
);
|
);
|
||||||
|
|
||||||
@ -574,6 +602,7 @@ where
|
|||||||
publication_points,
|
publication_points,
|
||||||
downloads,
|
downloads,
|
||||||
download_stats,
|
download_stats,
|
||||||
|
current_repo_objects: snapshot_current_repo_objects(Some(¤t_repo_index_for_output)),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -605,6 +634,7 @@ pub fn run_tree_from_tal_and_ta_der_serial(
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -652,6 +682,7 @@ pub fn run_tree_from_tal_bytes_serial_audit(
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -675,6 +706,7 @@ pub fn run_tree_from_tal_bytes_serial_audit(
|
|||||||
publication_points,
|
publication_points,
|
||||||
downloads,
|
downloads,
|
||||||
download_stats,
|
download_stats,
|
||||||
|
current_repo_objects: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -714,6 +746,7 @@ pub fn run_tree_from_tal_bytes_serial_audit_with_timing(
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -739,6 +772,7 @@ pub fn run_tree_from_tal_bytes_serial_audit_with_timing(
|
|||||||
publication_points,
|
publication_points,
|
||||||
downloads,
|
downloads,
|
||||||
download_stats,
|
download_stats,
|
||||||
|
current_repo_objects: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -771,6 +805,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit(
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -794,6 +829,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit(
|
|||||||
publication_points,
|
publication_points,
|
||||||
downloads,
|
downloads,
|
||||||
download_stats,
|
download_stats,
|
||||||
|
current_repo_objects: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -829,6 +865,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing(
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -853,6 +890,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing(
|
|||||||
publication_points,
|
publication_points,
|
||||||
downloads,
|
downloads,
|
||||||
download_stats,
|
download_stats,
|
||||||
|
current_repo_objects: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -894,6 +932,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial(
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -947,6 +986,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit(
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -970,6 +1010,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit(
|
|||||||
publication_points,
|
publication_points,
|
||||||
downloads,
|
downloads,
|
||||||
download_stats,
|
download_stats,
|
||||||
|
current_repo_objects: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1015,6 +1056,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing(
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -1039,6 +1081,7 @@ pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing(
|
|||||||
publication_points,
|
publication_points,
|
||||||
downloads,
|
downloads,
|
||||||
download_stats,
|
download_stats,
|
||||||
|
current_repo_objects: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1066,6 +1109,7 @@ fn build_payload_replay_runner<'a>(
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1094,6 +1138,7 @@ fn build_payload_delta_replay_runner<'a>(
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1122,6 +1167,7 @@ fn build_payload_delta_replay_current_store_runner<'a>(
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1238,6 +1284,7 @@ fn run_payload_delta_replay_audit_inner(
|
|||||||
publication_points,
|
publication_points,
|
||||||
downloads,
|
downloads,
|
||||||
download_stats,
|
download_stats,
|
||||||
|
current_repo_objects: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1381,6 +1428,7 @@ fn run_payload_delta_replay_step_audit_inner(
|
|||||||
publication_points,
|
publication_points,
|
||||||
downloads,
|
downloads,
|
||||||
download_stats,
|
download_stats,
|
||||||
|
current_repo_objects: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1442,23 +1490,15 @@ mod multi_tal_tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn discover_multiple_roots_from_tal_inputs_builds_multiple_root_handles() {
|
fn discover_multiple_roots_from_tal_inputs_builds_multiple_root_handles() {
|
||||||
let apnic_tal = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal")
|
let apnic_tal =
|
||||||
.expect("read apnic tal");
|
std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal").expect("read apnic tal");
|
||||||
let apnic_ta = std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta");
|
let apnic_ta = std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta");
|
||||||
let arin_tal = std::fs::read("tests/fixtures/tal/arin.tal").expect("read arin tal");
|
let arin_tal = std::fs::read("tests/fixtures/tal/arin.tal").expect("read arin tal");
|
||||||
let arin_ta = std::fs::read("tests/fixtures/ta/arin-ta.cer").expect("read arin ta");
|
let arin_ta = std::fs::read("tests/fixtures/ta/arin-ta.cer").expect("read arin ta");
|
||||||
|
|
||||||
let tal_inputs = vec![
|
let tal_inputs = vec![
|
||||||
TalInputSpec::from_ta_der(
|
TalInputSpec::from_ta_der("https://example.test/apnic.tal", apnic_tal, apnic_ta),
|
||||||
"https://example.test/apnic.tal",
|
TalInputSpec::from_ta_der("https://example.test/arin.tal", arin_tal, arin_ta),
|
||||||
apnic_tal,
|
|
||||||
apnic_ta,
|
|
||||||
),
|
|
||||||
TalInputSpec::from_ta_der(
|
|
||||||
"https://example.test/arin.tal",
|
|
||||||
arin_tal,
|
|
||||||
arin_ta,
|
|
||||||
),
|
|
||||||
];
|
];
|
||||||
|
|
||||||
let roots = discover_multiple_roots_from_tal_inputs(
|
let roots = discover_multiple_roots_from_tal_inputs(
|
||||||
@ -1631,16 +1671,14 @@ mod replay_api_tests {
|
|||||||
let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db");
|
let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db");
|
||||||
let (tal_bytes, ta_der, archive_root, locks_path, validation_time) =
|
let (tal_bytes, ta_der, archive_root, locks_path, validation_time) =
|
||||||
apnic_multi_rir_replay_inputs();
|
apnic_multi_rir_replay_inputs();
|
||||||
assert!(
|
if !archive_root.is_dir() || !locks_path.is_file() {
|
||||||
archive_root.is_dir(),
|
eprintln!(
|
||||||
"payload replay archive missing: {}",
|
"skipping multi-rir payload replay api test; missing fixtures: archive={} locks={}",
|
||||||
archive_root.display()
|
archive_root.display(),
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
locks_path.is_file(),
|
|
||||||
"payload replay locks missing: {}",
|
|
||||||
locks_path.display()
|
locks_path.display()
|
||||||
);
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
let out = run_tree_from_tal_and_ta_der_payload_replay_serial_audit(
|
let out = run_tree_from_tal_and_ta_der_payload_replay_serial_audit(
|
||||||
&store,
|
&store,
|
||||||
|
|||||||
@ -4,6 +4,7 @@ use crate::audit::{
|
|||||||
sha256_hex, sha256_hex_from_32,
|
sha256_hex, sha256_hex_from_32,
|
||||||
};
|
};
|
||||||
use crate::audit_downloads::DownloadLogHandle;
|
use crate::audit_downloads::DownloadLogHandle;
|
||||||
|
use crate::current_repo_index::CurrentRepoIndexHandle;
|
||||||
use crate::data_model::aspa::AspaObject;
|
use crate::data_model::aspa::AspaObject;
|
||||||
use crate::data_model::crl::RpkixCrl;
|
use crate::data_model::crl::RpkixCrl;
|
||||||
use crate::data_model::manifest::ManifestObject;
|
use crate::data_model::manifest::ManifestObject;
|
||||||
@ -14,8 +15,8 @@ use crate::data_model::router_cert::{
|
|||||||
BgpsecRouterCertificateProfileError,
|
BgpsecRouterCertificateProfileError,
|
||||||
};
|
};
|
||||||
use crate::fetch::rsync::RsyncFetcher;
|
use crate::fetch::rsync::RsyncFetcher;
|
||||||
use crate::policy::Policy;
|
|
||||||
use crate::parallel::repo_runtime::{RepoSyncRuntime, RepoSyncRuntimeOutcome};
|
use crate::parallel::repo_runtime::{RepoSyncRuntime, RepoSyncRuntimeOutcome};
|
||||||
|
use crate::policy::Policy;
|
||||||
use crate::replay::archive::ReplayArchiveIndex;
|
use crate::replay::archive::ReplayArchiveIndex;
|
||||||
use crate::replay::delta_archive::ReplayDeltaArchiveIndex;
|
use crate::replay::delta_archive::ReplayDeltaArchiveIndex;
|
||||||
use crate::report::{RfcRef, Warning};
|
use crate::report::{RfcRef, Warning};
|
||||||
@ -35,7 +36,7 @@ use crate::validation::ca_path::{
|
|||||||
};
|
};
|
||||||
use crate::validation::manifest::{
|
use crate::validation::manifest::{
|
||||||
ManifestFreshError, PublicationPointData, PublicationPointSource,
|
ManifestFreshError, PublicationPointData, PublicationPointSource,
|
||||||
process_manifest_publication_point_fresh_after_repo_sync,
|
process_manifest_publication_point_fresh_after_repo_sync_with_timing,
|
||||||
};
|
};
|
||||||
use crate::validation::objects::{
|
use crate::validation::objects::{
|
||||||
AspaAttestation, RouterKeyPayload, Vrp, process_publication_point_for_issuer,
|
AspaAttestation, RouterKeyPayload, Vrp, process_publication_point_for_issuer,
|
||||||
@ -53,6 +54,26 @@ use serde_json::json;
|
|||||||
use x509_parser::prelude::FromDer;
|
use x509_parser::prelude::FromDer;
|
||||||
use x509_parser::x509::SubjectPublicKeyInfo;
|
use x509_parser::x509::SubjectPublicKeyInfo;
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Default)]
|
||||||
|
struct BuildVcirTimingBreakdown {
|
||||||
|
select_crl_ms: u64,
|
||||||
|
current_ca_decode_ms: u64,
|
||||||
|
local_outputs_ms: u64,
|
||||||
|
child_entries_ms: u64,
|
||||||
|
related_artifacts_ms: u64,
|
||||||
|
struct_build_ms: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Default)]
|
||||||
|
struct PersistVcirTimingBreakdown {
|
||||||
|
embedded_collect_ms: u64,
|
||||||
|
embedded_store_ms: u64,
|
||||||
|
build_vcir_ms: u64,
|
||||||
|
previous_load_ms: u64,
|
||||||
|
replace_vcir_ms: u64,
|
||||||
|
build_vcir: BuildVcirTimingBreakdown,
|
||||||
|
}
|
||||||
|
|
||||||
pub struct Rpkiv1PublicationPointRunner<'a> {
|
pub struct Rpkiv1PublicationPointRunner<'a> {
|
||||||
pub store: &'a RocksStore,
|
pub store: &'a RocksStore,
|
||||||
pub policy: &'a Policy,
|
pub policy: &'a Policy,
|
||||||
@ -78,6 +99,7 @@ pub struct Rpkiv1PublicationPointRunner<'a> {
|
|||||||
/// same `rsync_base_uri` (observed in APNIC full sync timing reports).
|
/// same `rsync_base_uri` (observed in APNIC full sync timing reports).
|
||||||
pub rsync_dedup: bool,
|
pub rsync_dedup: bool,
|
||||||
pub rsync_repo_cache: Mutex<HashMap<String, bool>>, // rsync_base_uri -> rsync_ok
|
pub rsync_repo_cache: Mutex<HashMap<String, bool>>, // rsync_base_uri -> rsync_ok
|
||||||
|
pub current_repo_index: Option<CurrentRepoIndexHandle>,
|
||||||
pub repo_sync_runtime: Option<Arc<dyn RepoSyncRuntime>>,
|
pub repo_sync_runtime: Option<Arc<dyn RepoSyncRuntime>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -196,7 +218,12 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
|||||||
warnings: repo_warnings,
|
warnings: repo_warnings,
|
||||||
} = runtime.sync_publication_point_repo(ca)?;
|
} = runtime.sync_publication_point_repo(ca)?;
|
||||||
warnings.extend(repo_warnings);
|
warnings.extend(repo_warnings);
|
||||||
(repo_sync_ok, repo_sync_err, repo_sync_source, repo_sync_phase)
|
(
|
||||||
|
repo_sync_ok,
|
||||||
|
repo_sync_err,
|
||||||
|
repo_sync_source,
|
||||||
|
repo_sync_phase,
|
||||||
|
)
|
||||||
} else if skip_sync_due_to_dedup {
|
} else if skip_sync_due_to_dedup {
|
||||||
let source = if effective_notification_uri.is_some() {
|
let source = if effective_notification_uri.is_some() {
|
||||||
Some("rrdp_dedup_skip".to_string())
|
Some("rrdp_dedup_skip".to_string())
|
||||||
@ -332,15 +359,17 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let snapshot_prepare_started = std::time::Instant::now();
|
||||||
let fresh_publication_point = {
|
let fresh_publication_point = {
|
||||||
let _manifest_total = self
|
let _manifest_total = self
|
||||||
.timing
|
.timing
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|t| t.span_phase("manifest_processing_total"));
|
.map(|t| t.span_phase("manifest_processing_total"));
|
||||||
process_manifest_publication_point_fresh_after_repo_sync(
|
process_manifest_publication_point_fresh_after_repo_sync_with_timing(
|
||||||
self.store,
|
self.store,
|
||||||
&ca.manifest_rsync_uri,
|
&ca.manifest_rsync_uri,
|
||||||
&ca.publication_point_rsync_uri,
|
&ca.publication_point_rsync_uri,
|
||||||
|
self.current_repo_index.as_ref(),
|
||||||
&ca.ca_certificate_der,
|
&ca.ca_certificate_der,
|
||||||
ca.ca_certificate_rsync_uri.as_deref(),
|
ca.ca_certificate_rsync_uri.as_deref(),
|
||||||
self.validation_time,
|
self.validation_time,
|
||||||
@ -348,9 +377,11 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
|||||||
repo_sync_err.as_deref(),
|
repo_sync_err.as_deref(),
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
|
let snapshot_prepare_ms = snapshot_prepare_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
match fresh_publication_point {
|
match fresh_publication_point {
|
||||||
Ok(fresh_point) => {
|
Ok((fresh_point, snapshot_prepare_timing)) => {
|
||||||
|
let objects_processing_started = std::time::Instant::now();
|
||||||
let mut objects = {
|
let mut objects = {
|
||||||
let _objects_total = self
|
let _objects_total = self
|
||||||
.timing
|
.timing
|
||||||
@ -367,7 +398,9 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
|||||||
self.timing.as_ref(),
|
self.timing.as_ref(),
|
||||||
)
|
)
|
||||||
};
|
};
|
||||||
|
let objects_processing_ms = objects_processing_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
|
let child_discovery_started = std::time::Instant::now();
|
||||||
let out = {
|
let out = {
|
||||||
let _child_disc_total = self
|
let _child_disc_total = self
|
||||||
.timing
|
.timing
|
||||||
@ -391,15 +424,19 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
|||||||
(Vec::new(), Vec::new(), Vec::new())
|
(Vec::new(), Vec::new(), Vec::new())
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let child_discovery_ms = child_discovery_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
objects.router_keys.extend(discovered_router_keys);
|
objects.router_keys.extend(discovered_router_keys);
|
||||||
objects
|
objects
|
||||||
.local_outputs_cache
|
.local_outputs_cache
|
||||||
.extend(build_router_key_local_outputs(ca, &objects.router_keys));
|
.extend(build_router_key_local_outputs(ca, &objects.router_keys));
|
||||||
|
|
||||||
|
let snapshot_pack_started = std::time::Instant::now();
|
||||||
let pack = fresh_point.to_publication_point_snapshot();
|
let pack = fresh_point.to_publication_point_snapshot();
|
||||||
|
let snapshot_pack_ms = snapshot_pack_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
persist_vcir_for_fresh_result(
|
let persist_vcir_started = std::time::Instant::now();
|
||||||
|
let persist_vcir_timing = persist_vcir_for_fresh_result_with_timing(
|
||||||
self.store,
|
self.store,
|
||||||
ca,
|
ca,
|
||||||
&pack,
|
&pack,
|
||||||
@ -410,7 +447,9 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
|||||||
self.validation_time,
|
self.validation_time,
|
||||||
)
|
)
|
||||||
.map_err(|e| format!("persist VCIR failed: {e}"))?;
|
.map_err(|e| format!("persist VCIR failed: {e}"))?;
|
||||||
|
let persist_vcir_ms = persist_vcir_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
|
let audit_build_started = std::time::Instant::now();
|
||||||
let audit = build_publication_point_audit_from_snapshot(
|
let audit = build_publication_point_audit_from_snapshot(
|
||||||
ca,
|
ca,
|
||||||
PublicationPointSource::Fresh,
|
PublicationPointSource::Fresh,
|
||||||
@ -423,6 +462,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
|||||||
&objects,
|
&objects,
|
||||||
&child_audits,
|
&child_audits,
|
||||||
);
|
);
|
||||||
|
let audit_build_ms = audit_build_started.elapsed().as_millis() as u64;
|
||||||
let result = PublicationPointRunResult {
|
let result = PublicationPointRunResult {
|
||||||
source: PublicationPointSource::Fresh,
|
source: PublicationPointSource::Fresh,
|
||||||
snapshot: Some(pack),
|
snapshot: Some(pack),
|
||||||
@ -442,6 +482,30 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
|||||||
"repo_sync_phase": repo_sync_phase,
|
"repo_sync_phase": repo_sync_phase,
|
||||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||||
"total_duration_ms": total_duration_ms,
|
"total_duration_ms": total_duration_ms,
|
||||||
|
"post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms),
|
||||||
|
"snapshot_prepare_ms": snapshot_prepare_ms,
|
||||||
|
"snapshot_manifest_load_ms": snapshot_prepare_timing.manifest_load_ms,
|
||||||
|
"snapshot_manifest_decode_ms": snapshot_prepare_timing.manifest_decode_ms,
|
||||||
|
"snapshot_replay_guard_ms": snapshot_prepare_timing.replay_guard_ms,
|
||||||
|
"snapshot_manifest_entries_ms": snapshot_prepare_timing.manifest_entries_ms,
|
||||||
|
"snapshot_pack_files_ms": snapshot_prepare_timing.pack_files_ms,
|
||||||
|
"snapshot_ee_path_validate_ms": snapshot_prepare_timing.ee_path_validate_ms,
|
||||||
|
"objects_processing_ms": objects_processing_ms,
|
||||||
|
"child_discovery_ms": child_discovery_ms,
|
||||||
|
"snapshot_pack_ms": snapshot_pack_ms,
|
||||||
|
"persist_vcir_ms": persist_vcir_ms,
|
||||||
|
"persist_embedded_collect_ms": persist_vcir_timing.embedded_collect_ms,
|
||||||
|
"persist_embedded_store_ms": persist_vcir_timing.embedded_store_ms,
|
||||||
|
"persist_build_vcir_ms": persist_vcir_timing.build_vcir_ms,
|
||||||
|
"persist_previous_load_ms": persist_vcir_timing.previous_load_ms,
|
||||||
|
"persist_replace_vcir_ms": persist_vcir_timing.replace_vcir_ms,
|
||||||
|
"persist_select_crl_ms": persist_vcir_timing.build_vcir.select_crl_ms,
|
||||||
|
"persist_current_ca_decode_ms": persist_vcir_timing.build_vcir.current_ca_decode_ms,
|
||||||
|
"persist_local_outputs_ms": persist_vcir_timing.build_vcir.local_outputs_ms,
|
||||||
|
"persist_child_entries_ms": persist_vcir_timing.build_vcir.child_entries_ms,
|
||||||
|
"persist_related_artifacts_ms": persist_vcir_timing.build_vcir.related_artifacts_ms,
|
||||||
|
"persist_vcir_struct_ms": persist_vcir_timing.build_vcir.struct_build_ms,
|
||||||
|
"audit_build_ms": audit_build_ms,
|
||||||
"warning_count": result.warnings.len(),
|
"warning_count": result.warnings.len(),
|
||||||
"vrp_count": result.objects.vrps.len(),
|
"vrp_count": result.objects.vrps.len(),
|
||||||
"vap_count": result.objects.aspas.len(),
|
"vap_count": result.objects.aspas.len(),
|
||||||
@ -461,6 +525,30 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
|||||||
"repo_sync_phase": repo_sync_phase,
|
"repo_sync_phase": repo_sync_phase,
|
||||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||||
"total_duration_ms": total_duration_ms,
|
"total_duration_ms": total_duration_ms,
|
||||||
|
"post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms),
|
||||||
|
"snapshot_prepare_ms": snapshot_prepare_ms,
|
||||||
|
"snapshot_manifest_load_ms": snapshot_prepare_timing.manifest_load_ms,
|
||||||
|
"snapshot_manifest_decode_ms": snapshot_prepare_timing.manifest_decode_ms,
|
||||||
|
"snapshot_replay_guard_ms": snapshot_prepare_timing.replay_guard_ms,
|
||||||
|
"snapshot_manifest_entries_ms": snapshot_prepare_timing.manifest_entries_ms,
|
||||||
|
"snapshot_pack_files_ms": snapshot_prepare_timing.pack_files_ms,
|
||||||
|
"snapshot_ee_path_validate_ms": snapshot_prepare_timing.ee_path_validate_ms,
|
||||||
|
"objects_processing_ms": objects_processing_ms,
|
||||||
|
"child_discovery_ms": child_discovery_ms,
|
||||||
|
"snapshot_pack_ms": snapshot_pack_ms,
|
||||||
|
"persist_vcir_ms": persist_vcir_ms,
|
||||||
|
"persist_embedded_collect_ms": persist_vcir_timing.embedded_collect_ms,
|
||||||
|
"persist_embedded_store_ms": persist_vcir_timing.embedded_store_ms,
|
||||||
|
"persist_build_vcir_ms": persist_vcir_timing.build_vcir_ms,
|
||||||
|
"persist_previous_load_ms": persist_vcir_timing.previous_load_ms,
|
||||||
|
"persist_replace_vcir_ms": persist_vcir_timing.replace_vcir_ms,
|
||||||
|
"persist_select_crl_ms": persist_vcir_timing.build_vcir.select_crl_ms,
|
||||||
|
"persist_current_ca_decode_ms": persist_vcir_timing.build_vcir.current_ca_decode_ms,
|
||||||
|
"persist_local_outputs_ms": persist_vcir_timing.build_vcir.local_outputs_ms,
|
||||||
|
"persist_child_entries_ms": persist_vcir_timing.build_vcir.child_entries_ms,
|
||||||
|
"persist_related_artifacts_ms": persist_vcir_timing.build_vcir.related_artifacts_ms,
|
||||||
|
"persist_vcir_struct_ms": persist_vcir_timing.build_vcir.struct_build_ms,
|
||||||
|
"audit_build_ms": audit_build_ms,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@ -479,6 +567,10 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
|||||||
"repo_sync_phase": repo_sync_phase,
|
"repo_sync_phase": repo_sync_phase,
|
||||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||||
"total_duration_ms": total_duration_ms,
|
"total_duration_ms": total_duration_ms,
|
||||||
|
"post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms),
|
||||||
|
"snapshot_prepare_ms": snapshot_prepare_ms,
|
||||||
|
"projection_ms": 0,
|
||||||
|
"audit_build_ms": 0,
|
||||||
"error": fresh_err.to_string(),
|
"error": fresh_err.to_string(),
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
@ -498,6 +590,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
|||||||
Err(format!("{fresh_err}"))
|
Err(format!("{fresh_err}"))
|
||||||
}
|
}
|
||||||
crate::policy::CaFailedFetchPolicy::ReuseCurrentInstanceVcir => {
|
crate::policy::CaFailedFetchPolicy::ReuseCurrentInstanceVcir => {
|
||||||
|
let projection_started = std::time::Instant::now();
|
||||||
let projection = project_current_instance_vcir_on_failed_fetch(
|
let projection = project_current_instance_vcir_on_failed_fetch(
|
||||||
self.store,
|
self.store,
|
||||||
ca,
|
ca,
|
||||||
@ -505,7 +598,9 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
|||||||
self.validation_time,
|
self.validation_time,
|
||||||
)
|
)
|
||||||
.map_err(|e| format!("failed fetch VCIR projection failed: {e}"))?;
|
.map_err(|e| format!("failed fetch VCIR projection failed: {e}"))?;
|
||||||
|
let projection_ms = projection_started.elapsed().as_millis() as u64;
|
||||||
warnings.extend(projection.warnings.clone());
|
warnings.extend(projection.warnings.clone());
|
||||||
|
let audit_build_started = std::time::Instant::now();
|
||||||
let audit = build_publication_point_audit_from_vcir(
|
let audit = build_publication_point_audit_from_vcir(
|
||||||
ca,
|
ca,
|
||||||
projection.source,
|
projection.source,
|
||||||
@ -519,6 +614,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
|||||||
&projection.objects,
|
&projection.objects,
|
||||||
&projection.child_audits,
|
&projection.child_audits,
|
||||||
);
|
);
|
||||||
|
let audit_build_ms = audit_build_started.elapsed().as_millis() as u64;
|
||||||
let result = PublicationPointRunResult {
|
let result = PublicationPointRunResult {
|
||||||
source: projection.source,
|
source: projection.source,
|
||||||
snapshot: projection.snapshot,
|
snapshot: projection.snapshot,
|
||||||
@ -538,6 +634,10 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
|||||||
"repo_sync_phase": repo_sync_phase,
|
"repo_sync_phase": repo_sync_phase,
|
||||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||||
"total_duration_ms": total_duration_ms,
|
"total_duration_ms": total_duration_ms,
|
||||||
|
"post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms),
|
||||||
|
"snapshot_prepare_ms": snapshot_prepare_ms,
|
||||||
|
"projection_ms": projection_ms,
|
||||||
|
"audit_build_ms": audit_build_ms,
|
||||||
"warning_count": result.warnings.len(),
|
"warning_count": result.warnings.len(),
|
||||||
"vrp_count": result.objects.vrps.len(),
|
"vrp_count": result.objects.vrps.len(),
|
||||||
"vap_count": result.objects.aspas.len(),
|
"vap_count": result.objects.aspas.len(),
|
||||||
@ -604,6 +704,10 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
|||||||
"repo_sync_phase": repo_sync_phase,
|
"repo_sync_phase": repo_sync_phase,
|
||||||
"repo_sync_duration_ms": repo_sync_duration_ms,
|
"repo_sync_duration_ms": repo_sync_duration_ms,
|
||||||
"total_duration_ms": total_duration_ms,
|
"total_duration_ms": total_duration_ms,
|
||||||
|
"post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms),
|
||||||
|
"snapshot_prepare_ms": snapshot_prepare_ms,
|
||||||
|
"projection_ms": projection_ms,
|
||||||
|
"audit_build_ms": audit_build_ms,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@ -715,13 +819,16 @@ fn discover_children_from_fresh_snapshot_with_audit<P: PublicationPointData>(
|
|||||||
let mut crl_cache: std::collections::HashMap<String, CachedIssuerCrl> = locked_files
|
let mut crl_cache: std::collections::HashMap<String, CachedIssuerCrl> = locked_files
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|f| f.rsync_uri.ends_with(".crl"))
|
.filter(|f| f.rsync_uri.ends_with(".crl"))
|
||||||
.map(|f| {
|
.map(|f| -> Result<(String, CachedIssuerCrl), String> {
|
||||||
(
|
let bytes = f
|
||||||
|
.bytes_cloned()
|
||||||
|
.map_err(|e| format!("snapshot CRL bytes load failed: {e}"))?;
|
||||||
|
Ok((
|
||||||
f.rsync_uri.clone(),
|
f.rsync_uri.clone(),
|
||||||
CachedIssuerCrl::Pending(f.bytes.clone()),
|
CachedIssuerCrl::Pending(bytes),
|
||||||
)
|
))
|
||||||
})
|
})
|
||||||
.collect();
|
.collect::<Result<_, String>>()?;
|
||||||
|
|
||||||
let mut out: Vec<DiscoveredChildCaInstance> = Vec::new();
|
let mut out: Vec<DiscoveredChildCaInstance> = Vec::new();
|
||||||
let mut audits: Vec<ObjectAuditEntry> = Vec::new();
|
let mut audits: Vec<ObjectAuditEntry> = Vec::new();
|
||||||
@ -798,7 +905,9 @@ fn discover_children_from_fresh_snapshot_with_audit<P: PublicationPointData>(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
cer_seen = cer_seen.saturating_add(1);
|
cer_seen = cer_seen.saturating_add(1);
|
||||||
let child_der = f.bytes.as_slice();
|
let child_der = f
|
||||||
|
.bytes()
|
||||||
|
.map_err(|e| format!("child certificate bytes load failed: {e}"))?;
|
||||||
|
|
||||||
let tdecode = std::time::Instant::now();
|
let tdecode = std::time::Instant::now();
|
||||||
let child_cert = match crate::data_model::rc::ResourceCertificate::decode_der(child_der) {
|
let child_cert = match crate::data_model::rc::ResourceCertificate::decode_der(child_der) {
|
||||||
@ -1232,7 +1341,10 @@ fn select_issuer_crl_from_snapshot<'a>(
|
|||||||
for u in crldp_uris {
|
for u in crldp_uris {
|
||||||
let s = u.as_str();
|
let s = u.as_str();
|
||||||
if let Some(f) = pack.files.iter().find(|f| f.rsync_uri == s) {
|
if let Some(f) = pack.files.iter().find(|f| f.rsync_uri == s) {
|
||||||
return Ok((f.rsync_uri.as_str(), f.bytes.as_slice()));
|
let bytes = f
|
||||||
|
.bytes()
|
||||||
|
.map_err(|e| format!("snapshot CRL bytes load failed: {e}"))?;
|
||||||
|
return Ok((f.rsync_uri.as_str(), bytes));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1285,7 +1397,8 @@ fn repo_sync_failure_phase_label(
|
|||||||
original_notification_uri: Option<&str>,
|
original_notification_uri: Option<&str>,
|
||||||
effective_notification_uri: Option<&str>,
|
effective_notification_uri: Option<&str>,
|
||||||
) -> &'static str {
|
) -> &'static str {
|
||||||
if attempted_rrdp && original_notification_uri.is_some() && effective_notification_uri.is_some() {
|
if attempted_rrdp && original_notification_uri.is_some() && effective_notification_uri.is_some()
|
||||||
|
{
|
||||||
"rrdp_failed_rsync_failed"
|
"rrdp_failed_rsync_failed"
|
||||||
} else if attempted_rrdp
|
} else if attempted_rrdp
|
||||||
&& original_notification_uri.is_some()
|
&& original_notification_uri.is_some()
|
||||||
@ -1367,7 +1480,7 @@ fn build_publication_point_audit_from_snapshot(
|
|||||||
if !f.rsync_uri.ends_with(".crl") {
|
if !f.rsync_uri.ends_with(".crl") {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let ok = RpkixCrl::decode_der(&f.bytes).is_ok();
|
let ok = f.bytes().ok().and_then(|bytes| RpkixCrl::decode_der(bytes).ok()).is_some();
|
||||||
audit_by_uri.insert(
|
audit_by_uri.insert(
|
||||||
f.rsync_uri.clone(),
|
f.rsync_uri.clone(),
|
||||||
ObjectAuditEntry {
|
ObjectAuditEntry {
|
||||||
@ -2129,15 +2242,42 @@ fn persist_vcir_for_fresh_result(
|
|||||||
discovered_children: &[DiscoveredChildCaInstance],
|
discovered_children: &[DiscoveredChildCaInstance],
|
||||||
validation_time: time::OffsetDateTime,
|
validation_time: time::OffsetDateTime,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
if objects.stats.publication_point_dropped {
|
persist_vcir_for_fresh_result_with_timing(
|
||||||
return Ok(());
|
store,
|
||||||
}
|
ca,
|
||||||
|
pack,
|
||||||
let embedded_evidence = collect_vcir_embedded_evidence(pack, objects)?;
|
objects,
|
||||||
persist_vcir_non_repository_evidence(store, ca, &embedded_evidence)
|
warnings,
|
||||||
.map_err(|e| format!("store VCIR audit evidence failed: {e}"))?;
|
child_audits,
|
||||||
|
discovered_children,
|
||||||
let vcir = build_vcir_from_fresh_result(
|
validation_time,
|
||||||
|
)
|
||||||
|
.map(|_timing| ())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn persist_vcir_for_fresh_result_with_timing(
|
||||||
|
store: &RocksStore,
|
||||||
|
ca: &CaInstanceHandle,
|
||||||
|
pack: &PublicationPointSnapshot,
|
||||||
|
objects: &crate::validation::objects::ObjectsOutput,
|
||||||
|
warnings: &[Warning],
|
||||||
|
child_audits: &[ObjectAuditEntry],
|
||||||
|
discovered_children: &[DiscoveredChildCaInstance],
|
||||||
|
validation_time: time::OffsetDateTime,
|
||||||
|
) -> Result<PersistVcirTimingBreakdown, String> {
|
||||||
|
let mut timing = PersistVcirTimingBreakdown::default();
|
||||||
|
|
||||||
|
if objects.stats.publication_point_dropped {
|
||||||
|
return Ok(timing);
|
||||||
|
}
|
||||||
|
|
||||||
|
let embedded_store_started = std::time::Instant::now();
|
||||||
|
persist_vcir_non_repository_evidence(store, ca)
|
||||||
|
.map_err(|e| format!("store VCIR audit evidence failed: {e}"))?;
|
||||||
|
timing.embedded_store_ms = embedded_store_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
|
let build_vcir_started = std::time::Instant::now();
|
||||||
|
let (vcir, build_vcir_timing) = build_vcir_from_fresh_result_with_timing(
|
||||||
ca,
|
ca,
|
||||||
pack,
|
pack,
|
||||||
objects,
|
objects,
|
||||||
@ -2145,17 +2285,23 @@ fn persist_vcir_for_fresh_result(
|
|||||||
child_audits,
|
child_audits,
|
||||||
discovered_children,
|
discovered_children,
|
||||||
validation_time,
|
validation_time,
|
||||||
&embedded_evidence,
|
|
||||||
)?;
|
)?;
|
||||||
|
timing.build_vcir_ms = build_vcir_started.elapsed().as_millis() as u64;
|
||||||
|
timing.build_vcir = build_vcir_timing;
|
||||||
|
|
||||||
|
let previous_load_started = std::time::Instant::now();
|
||||||
let previous = store
|
let previous = store
|
||||||
.get_vcir(&pack.manifest_rsync_uri)
|
.get_vcir(&pack.manifest_rsync_uri)
|
||||||
.map_err(|e| format!("load existing VCIR failed: {e}"))?;
|
.map_err(|e| format!("load existing VCIR failed: {e}"))?;
|
||||||
|
timing.previous_load_ms = previous_load_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
|
let replace_vcir_started = std::time::Instant::now();
|
||||||
store
|
store
|
||||||
.replace_vcir_and_audit_rule_indexes(previous.as_ref(), &vcir)
|
.replace_vcir_and_audit_rule_indexes(previous.as_ref(), &vcir)
|
||||||
.map_err(|e| format!("store VCIR and audit rule index failed: {e}"))?;
|
.map_err(|e| format!("store VCIR and audit rule index failed: {e}"))?;
|
||||||
|
timing.replace_vcir_ms = replace_vcir_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
Ok(())
|
Ok(timing)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_vcir_from_fresh_result(
|
fn build_vcir_from_fresh_result(
|
||||||
@ -2166,21 +2312,56 @@ fn build_vcir_from_fresh_result(
|
|||||||
child_audits: &[ObjectAuditEntry],
|
child_audits: &[ObjectAuditEntry],
|
||||||
discovered_children: &[DiscoveredChildCaInstance],
|
discovered_children: &[DiscoveredChildCaInstance],
|
||||||
validation_time: time::OffsetDateTime,
|
validation_time: time::OffsetDateTime,
|
||||||
embedded_evidence: &[VcirEmbeddedEvidence],
|
|
||||||
) -> Result<ValidatedCaInstanceResult, String> {
|
) -> Result<ValidatedCaInstanceResult, String> {
|
||||||
|
build_vcir_from_fresh_result_with_timing(
|
||||||
|
ca,
|
||||||
|
pack,
|
||||||
|
objects,
|
||||||
|
warnings,
|
||||||
|
child_audits,
|
||||||
|
discovered_children,
|
||||||
|
validation_time,
|
||||||
|
)
|
||||||
|
.map(|(vcir, _timing)| vcir)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_vcir_from_fresh_result_with_timing(
|
||||||
|
ca: &CaInstanceHandle,
|
||||||
|
pack: &PublicationPointSnapshot,
|
||||||
|
objects: &crate::validation::objects::ObjectsOutput,
|
||||||
|
warnings: &[Warning],
|
||||||
|
child_audits: &[ObjectAuditEntry],
|
||||||
|
discovered_children: &[DiscoveredChildCaInstance],
|
||||||
|
validation_time: time::OffsetDateTime,
|
||||||
|
) -> Result<(ValidatedCaInstanceResult, BuildVcirTimingBreakdown), String> {
|
||||||
|
let mut timing = BuildVcirTimingBreakdown::default();
|
||||||
|
|
||||||
|
let select_crl_started = std::time::Instant::now();
|
||||||
let current_crl = select_manifest_current_crl_from_snapshot(pack)?;
|
let current_crl = select_manifest_current_crl_from_snapshot(pack)?;
|
||||||
|
timing.select_crl_ms = select_crl_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
|
let current_ca_decode_started = std::time::Instant::now();
|
||||||
let ca_cert = ResourceCertificate::decode_der(&ca.ca_certificate_der)
|
let ca_cert = ResourceCertificate::decode_der(&ca.ca_certificate_der)
|
||||||
.map_err(|e| format!("decode current CA certificate failed: {e}"))?;
|
.map_err(|e| format!("decode current CA certificate failed: {e}"))?;
|
||||||
|
timing.current_ca_decode_ms = current_ca_decode_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
|
let local_outputs_started = std::time::Instant::now();
|
||||||
let local_outputs = build_vcir_local_outputs(ca, pack, objects)?;
|
let local_outputs = build_vcir_local_outputs(ca, pack, objects)?;
|
||||||
|
timing.local_outputs_ms = local_outputs_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
|
let child_entries_started = std::time::Instant::now();
|
||||||
let child_entries = build_vcir_child_entries(discovered_children, validation_time)?;
|
let child_entries = build_vcir_child_entries(discovered_children, validation_time)?;
|
||||||
|
timing.child_entries_ms = child_entries_started.elapsed().as_millis() as u64;
|
||||||
|
|
||||||
|
let related_artifacts_started = std::time::Instant::now();
|
||||||
let related_artifacts = build_vcir_related_artifacts(
|
let related_artifacts = build_vcir_related_artifacts(
|
||||||
ca,
|
ca,
|
||||||
pack,
|
pack,
|
||||||
current_crl.file.rsync_uri.as_str(),
|
current_crl.file.rsync_uri.as_str(),
|
||||||
objects,
|
objects,
|
||||||
child_audits,
|
child_audits,
|
||||||
embedded_evidence,
|
|
||||||
);
|
);
|
||||||
|
timing.related_artifacts_ms = related_artifacts_started.elapsed().as_millis() as u64;
|
||||||
let accepted_object_count = related_artifacts
|
let accepted_object_count = related_artifacts
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|artifact| artifact.validation_status == VcirArtifactValidationStatus::Accepted)
|
.filter(|artifact| artifact.validation_status == VcirArtifactValidationStatus::Accepted)
|
||||||
@ -2207,6 +2388,7 @@ fn build_vcir_from_fresh_result(
|
|||||||
.ok_or_else(|| "current CA certificate missing AuthorityKeyIdentifier".to_string())?,
|
.ok_or_else(|| "current CA certificate missing AuthorityKeyIdentifier".to_string())?,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let struct_build_started = std::time::Instant::now();
|
||||||
let vcir = ValidatedCaInstanceResult {
|
let vcir = ValidatedCaInstanceResult {
|
||||||
manifest_rsync_uri: pack.manifest_rsync_uri.clone(),
|
manifest_rsync_uri: pack.manifest_rsync_uri.clone(),
|
||||||
parent_manifest_rsync_uri: ca.parent_manifest_rsync_uri.clone(),
|
parent_manifest_rsync_uri: ca.parent_manifest_rsync_uri.clone(),
|
||||||
@ -2264,7 +2446,8 @@ fn build_vcir_from_fresh_result(
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
vcir.validate_internal().map_err(|e| e.to_string())?;
|
vcir.validate_internal().map_err(|e| e.to_string())?;
|
||||||
Ok(vcir)
|
timing.struct_build_ms = struct_build_started.elapsed().as_millis() as u64;
|
||||||
|
Ok((vcir, timing))
|
||||||
}
|
}
|
||||||
|
|
||||||
struct CurrentCrlRef<'a> {
|
struct CurrentCrlRef<'a> {
|
||||||
@ -2290,7 +2473,10 @@ fn select_manifest_current_crl_from_snapshot(
|
|||||||
.iter()
|
.iter()
|
||||||
.find(|candidate| candidate.rsync_uri == *uri)
|
.find(|candidate| candidate.rsync_uri == *uri)
|
||||||
{
|
{
|
||||||
let crl = RpkixCrl::decode_der(&file.bytes)
|
let crl = RpkixCrl::decode_der(
|
||||||
|
file.bytes()
|
||||||
|
.map_err(|e| format!("load current CRL bytes for VCIR failed: {e}"))?,
|
||||||
|
)
|
||||||
.map_err(|e| format!("decode current CRL for VCIR failed: {e}"))?;
|
.map_err(|e| format!("decode current CRL for VCIR failed: {e}"))?;
|
||||||
return Ok(CurrentCrlRef { file, crl });
|
return Ok(CurrentCrlRef { file, crl });
|
||||||
}
|
}
|
||||||
@ -2329,7 +2515,10 @@ fn build_vcir_local_outputs(
|
|||||||
for file in &pack.files {
|
for file in &pack.files {
|
||||||
let source_object_hash = sha256_hex_from_32(&file.sha256);
|
let source_object_hash = sha256_hex_from_32(&file.sha256);
|
||||||
if accepted_roa_uris.contains(file.rsync_uri.as_str()) {
|
if accepted_roa_uris.contains(file.rsync_uri.as_str()) {
|
||||||
let roa = RoaObject::decode_der(&file.bytes)
|
let roa = RoaObject::decode_der(
|
||||||
|
file.bytes()
|
||||||
|
.map_err(|e| format!("load accepted ROA bytes for VCIR failed: {e}"))?,
|
||||||
|
)
|
||||||
.map_err(|e| format!("decode accepted ROA for VCIR failed: {e}"))?;
|
.map_err(|e| format!("decode accepted ROA for VCIR failed: {e}"))?;
|
||||||
let ee = &roa.signed_object.signed_data.certificates[0];
|
let ee = &roa.signed_object.signed_data.certificates[0];
|
||||||
let source_ee_cert_hash = sha256_hex(ee.raw_der.as_slice());
|
let source_ee_cert_hash = sha256_hex(ee.raw_der.as_slice());
|
||||||
@ -2368,7 +2557,10 @@ fn build_vcir_local_outputs(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
} else if accepted_aspa_uris.contains(file.rsync_uri.as_str()) {
|
} else if accepted_aspa_uris.contains(file.rsync_uri.as_str()) {
|
||||||
let aspa = AspaObject::decode_der(&file.bytes)
|
let aspa = AspaObject::decode_der(
|
||||||
|
file.bytes()
|
||||||
|
.map_err(|e| format!("load accepted ASPA bytes for VCIR failed: {e}"))?,
|
||||||
|
)
|
||||||
.map_err(|e| format!("decode accepted ASPA for VCIR failed: {e}"))?;
|
.map_err(|e| format!("decode accepted ASPA for VCIR failed: {e}"))?;
|
||||||
let ee = &aspa.signed_object.signed_data.certificates[0];
|
let ee = &aspa.signed_object.signed_data.certificates[0];
|
||||||
let source_ee_cert_hash = sha256_hex(ee.raw_der.as_slice());
|
let source_ee_cert_hash = sha256_hex(ee.raw_der.as_slice());
|
||||||
@ -2489,98 +2681,9 @@ fn build_vcir_child_entries(
|
|||||||
Ok(out)
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
|
||||||
struct VcirEmbeddedEvidence {
|
|
||||||
artifact: VcirRelatedArtifact,
|
|
||||||
raw_entry: RawByHashEntry,
|
|
||||||
}
|
|
||||||
|
|
||||||
fn collect_vcir_embedded_evidence(
|
|
||||||
pack: &PublicationPointSnapshot,
|
|
||||||
objects: &crate::validation::objects::ObjectsOutput,
|
|
||||||
) -> Result<Vec<VcirEmbeddedEvidence>, String> {
|
|
||||||
let mut evidence = Vec::new();
|
|
||||||
let mut seen_hashes = HashSet::new();
|
|
||||||
|
|
||||||
let manifest = ManifestObject::decode_der(&pack.manifest_bytes)
|
|
||||||
.map_err(|e| format!("decode manifest for VCIR embedded evidence failed: {e}"))?;
|
|
||||||
if let Some(ee) = manifest.signed_object.signed_data.certificates.first() {
|
|
||||||
let ee_hash = sha256_hex(ee.raw_der.as_slice());
|
|
||||||
if seen_hashes.insert(ee_hash.clone()) {
|
|
||||||
evidence.push(VcirEmbeddedEvidence {
|
|
||||||
artifact: VcirRelatedArtifact {
|
|
||||||
artifact_role: VcirArtifactRole::EeCert,
|
|
||||||
artifact_kind: VcirArtifactKind::Cer,
|
|
||||||
uri: None,
|
|
||||||
sha256: ee_hash.clone(),
|
|
||||||
object_type: Some("cer".to_string()),
|
|
||||||
validation_status: VcirArtifactValidationStatus::Accepted,
|
|
||||||
},
|
|
||||||
raw_entry: embedded_raw_entry(ee_hash, ee.raw_der.to_vec()),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let accepted_uris: HashSet<&str> = objects
|
|
||||||
.audit
|
|
||||||
.iter()
|
|
||||||
.filter(|entry| matches!(entry.result, AuditObjectResult::Ok))
|
|
||||||
.map(|entry| entry.rsync_uri.as_str())
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
for file in &pack.files {
|
|
||||||
let Some(kind) = signed_object_kind_from_uri(file.rsync_uri.as_str()) else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
if !accepted_uris.contains(file.rsync_uri.as_str()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let ee_der = match kind {
|
|
||||||
VcirArtifactKind::Roa => RoaObject::decode_der(&file.bytes)
|
|
||||||
.map_err(|e| format!("decode accepted ROA for VCIR embedded evidence failed: {e}"))?
|
|
||||||
.signed_object
|
|
||||||
.signed_data
|
|
||||||
.certificates
|
|
||||||
.first()
|
|
||||||
.map(|cert| cert.raw_der.to_vec()),
|
|
||||||
VcirArtifactKind::Aspa => AspaObject::decode_der(&file.bytes)
|
|
||||||
.map_err(|e| {
|
|
||||||
format!("decode accepted ASPA for VCIR embedded evidence failed: {e}")
|
|
||||||
})?
|
|
||||||
.signed_object
|
|
||||||
.signed_data
|
|
||||||
.certificates
|
|
||||||
.first()
|
|
||||||
.map(|cert| cert.raw_der.to_vec()),
|
|
||||||
_ => None,
|
|
||||||
};
|
|
||||||
let Some(ee_der) = ee_der else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let ee_hash = sha256_hex(ee_der.as_slice());
|
|
||||||
if !seen_hashes.insert(ee_hash.clone()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
evidence.push(VcirEmbeddedEvidence {
|
|
||||||
artifact: VcirRelatedArtifact {
|
|
||||||
artifact_role: VcirArtifactRole::EeCert,
|
|
||||||
artifact_kind: VcirArtifactKind::Cer,
|
|
||||||
uri: None,
|
|
||||||
sha256: ee_hash.clone(),
|
|
||||||
object_type: Some("cer".to_string()),
|
|
||||||
validation_status: VcirArtifactValidationStatus::Accepted,
|
|
||||||
},
|
|
||||||
raw_entry: embedded_raw_entry(ee_hash, ee_der),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(evidence)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn persist_vcir_non_repository_evidence(
|
fn persist_vcir_non_repository_evidence(
|
||||||
store: &RocksStore,
|
store: &RocksStore,
|
||||||
ca: &CaInstanceHandle,
|
ca: &CaInstanceHandle,
|
||||||
embedded_evidence: &[VcirEmbeddedEvidence],
|
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
let current_ca_hash = sha256_hex(&ca.ca_certificate_der);
|
let current_ca_hash = sha256_hex(&ca.ca_certificate_der);
|
||||||
let mut current_ca_entry =
|
let mut current_ca_entry =
|
||||||
@ -2591,10 +2694,6 @@ fn persist_vcir_non_repository_evidence(
|
|||||||
current_ca_entry.object_type = Some("cer".to_string());
|
current_ca_entry.object_type = Some("cer".to_string());
|
||||||
current_ca_entry.encoding = Some("der".to_string());
|
current_ca_entry.encoding = Some("der".to_string());
|
||||||
upsert_raw_by_hash_entry(store, current_ca_entry)?;
|
upsert_raw_by_hash_entry(store, current_ca_entry)?;
|
||||||
|
|
||||||
for evidence in embedded_evidence {
|
|
||||||
upsert_raw_by_hash_entry(store, evidence.raw_entry.clone())?;
|
|
||||||
}
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2641,32 +2740,12 @@ fn upsert_raw_by_hash_entry(store: &RocksStore, entry: RawByHashEntry) -> Result
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn embedded_raw_entry(sha256_hex: String, bytes: Vec<u8>) -> RawByHashEntry {
|
|
||||||
let mut entry = RawByHashEntry::from_bytes(sha256_hex, bytes);
|
|
||||||
entry.object_type = Some("cer".to_string());
|
|
||||||
entry.encoding = Some("der".to_string());
|
|
||||||
entry
|
|
||||||
}
|
|
||||||
|
|
||||||
fn signed_object_kind_from_uri(uri: &str) -> Option<VcirArtifactKind> {
|
|
||||||
if uri.ends_with(".roa") {
|
|
||||||
Some(VcirArtifactKind::Roa)
|
|
||||||
} else if uri.ends_with(".asa") {
|
|
||||||
Some(VcirArtifactKind::Aspa)
|
|
||||||
} else if uri.ends_with(".gbr") {
|
|
||||||
Some(VcirArtifactKind::Gbr)
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn build_vcir_related_artifacts(
|
fn build_vcir_related_artifacts(
|
||||||
ca: &CaInstanceHandle,
|
ca: &CaInstanceHandle,
|
||||||
pack: &PublicationPointSnapshot,
|
pack: &PublicationPointSnapshot,
|
||||||
current_crl_rsync_uri: &str,
|
current_crl_rsync_uri: &str,
|
||||||
objects: &crate::validation::objects::ObjectsOutput,
|
objects: &crate::validation::objects::ObjectsOutput,
|
||||||
child_audits: &[ObjectAuditEntry],
|
child_audits: &[ObjectAuditEntry],
|
||||||
embedded_evidence: &[VcirEmbeddedEvidence],
|
|
||||||
) -> Vec<VcirRelatedArtifact> {
|
) -> Vec<VcirRelatedArtifact> {
|
||||||
let mut audit_by_uri: HashMap<&str, AuditObjectResult> = HashMap::new();
|
let mut audit_by_uri: HashMap<&str, AuditObjectResult> = HashMap::new();
|
||||||
for entry in child_audits.iter().chain(objects.audit.iter()) {
|
for entry in child_audits.iter().chain(objects.audit.iter()) {
|
||||||
@ -2711,11 +2790,6 @@ fn build_vcir_related_artifacts(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
artifacts.extend(
|
|
||||||
embedded_evidence
|
|
||||||
.iter()
|
|
||||||
.map(|evidence| evidence.artifact.clone()),
|
|
||||||
);
|
|
||||||
artifacts
|
artifacts
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -3498,7 +3572,7 @@ authorityKeyIdentifier = keyid:always
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn collect_and_persist_vcir_embedded_evidence_for_real_signed_objects() {
|
fn persist_vcir_non_repository_evidence_stores_current_ca_cert_only() {
|
||||||
let (pack, issuer_ca_der, validation_time) =
|
let (pack, issuer_ca_der, validation_time) =
|
||||||
cernet_publication_point_snapshot_for_vcir_tests();
|
cernet_publication_point_snapshot_for_vcir_tests();
|
||||||
let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca");
|
let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca");
|
||||||
@ -3519,13 +3593,6 @@ authorityKeyIdentifier = keyid:always
|
|||||||
"expected local outputs from signed objects"
|
"expected local outputs from signed objects"
|
||||||
);
|
);
|
||||||
|
|
||||||
let evidence =
|
|
||||||
collect_vcir_embedded_evidence(&pack, &objects).expect("collect embedded evidence");
|
|
||||||
assert!(
|
|
||||||
evidence.len() >= 2,
|
|
||||||
"expected manifest EE and signed-object EE evidence"
|
|
||||||
);
|
|
||||||
|
|
||||||
let store_dir = tempfile::tempdir().expect("store dir");
|
let store_dir = tempfile::tempdir().expect("store dir");
|
||||||
let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
|
let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
|
||||||
let ca = CaInstanceHandle {
|
let ca = CaInstanceHandle {
|
||||||
@ -3543,7 +3610,7 @@ authorityKeyIdentifier = keyid:always
|
|||||||
publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(),
|
publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(),
|
||||||
rrdp_notification_uri: None,
|
rrdp_notification_uri: None,
|
||||||
};
|
};
|
||||||
persist_vcir_non_repository_evidence(&store, &ca, &evidence)
|
persist_vcir_non_repository_evidence(&store, &ca)
|
||||||
.expect("persist embedded evidence");
|
.expect("persist embedded evidence");
|
||||||
|
|
||||||
let issuer_hash = sha256_hex(&issuer_ca_der);
|
let issuer_hash = sha256_hex(&issuer_ca_der);
|
||||||
@ -3557,15 +3624,17 @@ authorityKeyIdentifier = keyid:always
|
|||||||
.iter()
|
.iter()
|
||||||
.any(|uri| uri.ends_with("BfycW4hQb3wNP4YsiJW-1n6fjro.cer"))
|
.any(|uri| uri.ends_with("BfycW4hQb3wNP4YsiJW-1n6fjro.cer"))
|
||||||
);
|
);
|
||||||
for entry in &evidence {
|
let first_output = objects
|
||||||
|
.local_outputs_cache
|
||||||
|
.first()
|
||||||
|
.expect("first local output");
|
||||||
assert!(
|
assert!(
|
||||||
store
|
store
|
||||||
.get_raw_by_hash_entry(&entry.raw_entry.sha256_hex)
|
.get_raw_by_hash_entry(&first_output.source_ee_cert_hash)
|
||||||
.expect("load evidence raw entry")
|
.expect("load source ee raw")
|
||||||
.is_some()
|
.is_none()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn build_router_key_local_outputs_encodes_router_key_payloads() {
|
fn build_router_key_local_outputs_encodes_router_key_payloads() {
|
||||||
@ -3794,24 +3863,12 @@ authorityKeyIdentifier = keyid:always
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
let embedded = vec![VcirEmbeddedEvidence {
|
|
||||||
artifact: VcirRelatedArtifact {
|
|
||||||
artifact_role: VcirArtifactRole::EeCert,
|
|
||||||
artifact_kind: VcirArtifactKind::Cer,
|
|
||||||
uri: None,
|
|
||||||
sha256: sha256_hex(b"embedded-ee"),
|
|
||||||
object_type: Some("cer".to_string()),
|
|
||||||
validation_status: VcirArtifactValidationStatus::Accepted,
|
|
||||||
},
|
|
||||||
raw_entry: embedded_raw_entry(sha256_hex(b"embedded-ee"), vec![1u8, 2, 3]),
|
|
||||||
}];
|
|
||||||
let artifacts = build_vcir_related_artifacts(
|
let artifacts = build_vcir_related_artifacts(
|
||||||
&ca,
|
&ca,
|
||||||
&pack,
|
&pack,
|
||||||
"rsync://example.test/repo/issuer/issuer.crl",
|
"rsync://example.test/repo/issuer/issuer.crl",
|
||||||
&objects,
|
&objects,
|
||||||
&[],
|
&[],
|
||||||
&embedded,
|
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
artifacts
|
artifacts
|
||||||
@ -3841,9 +3898,12 @@ authorityKeyIdentifier = keyid:always
|
|||||||
assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref()
|
assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref()
|
||||||
== Some("rsync://example.test/repo/issuer/extra.bin")
|
== Some("rsync://example.test/repo/issuer/extra.bin")
|
||||||
&& artifact.artifact_kind == VcirArtifactKind::Other));
|
&& artifact.artifact_kind == VcirArtifactKind::Other));
|
||||||
assert!(artifacts.iter().any(
|
assert!(
|
||||||
|artifact| artifact.uri.is_none() && artifact.sha256 == sha256_hex(b"embedded-ee")
|
!artifacts
|
||||||
));
|
.iter()
|
||||||
|
.any(|artifact| artifact.uri.is_none() && artifact.sha256 == sha256_hex(b"embedded-ee")),
|
||||||
|
"embedded EE cert artifacts should no longer be persisted separately"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -4046,6 +4106,7 @@ authorityKeyIdentifier = keyid:always
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: false,
|
rsync_dedup: false,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -4108,17 +4169,6 @@ authorityKeyIdentifier = keyid:always
|
|||||||
.expect("audit rule index exists");
|
.expect("audit rule index exists");
|
||||||
assert_eq!(audit_rule.manifest_rsync_uri, manifest_rsync_uri);
|
assert_eq!(audit_rule.manifest_rsync_uri, manifest_rsync_uri);
|
||||||
assert_eq!(audit_rule.output_id, first_vrp.output_id);
|
assert_eq!(audit_rule.output_id, first_vrp.output_id);
|
||||||
assert!(vcir.related_artifacts.iter().any(|artifact| {
|
|
||||||
artifact.artifact_role == VcirArtifactRole::EeCert
|
|
||||||
&& artifact.artifact_kind == VcirArtifactKind::Cer
|
|
||||||
}));
|
|
||||||
let ee_entry = store
|
|
||||||
.get_raw_by_hash_entry(&first_vrp.source_ee_cert_hash)
|
|
||||||
.expect("get source ee raw")
|
|
||||||
.expect("source ee raw exists");
|
|
||||||
assert_eq!(ee_entry.object_type.as_deref(), Some("cer"));
|
|
||||||
assert_eq!(ee_entry.encoding.as_deref(), Some("der"));
|
|
||||||
|
|
||||||
let trace = crate::audit_trace::trace_rule_to_root(
|
let trace = crate::audit_trace::trace_rule_to_root(
|
||||||
&store,
|
&store,
|
||||||
crate::storage::AuditRuleKind::Roa,
|
crate::storage::AuditRuleKind::Roa,
|
||||||
@ -4215,6 +4265,7 @@ authorityKeyIdentifier = keyid:always
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -4323,6 +4374,7 @@ authorityKeyIdentifier = keyid:always
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -4434,6 +4486,7 @@ authorityKeyIdentifier = keyid:always
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -4517,6 +4570,7 @@ authorityKeyIdentifier = keyid:always
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: false,
|
rsync_dedup: false,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
let first = ok_runner
|
let first = ok_runner
|
||||||
@ -4543,6 +4597,7 @@ authorityKeyIdentifier = keyid:always
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: false,
|
rsync_dedup: false,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
let second = bad_runner
|
let second = bad_runner
|
||||||
@ -5728,6 +5783,7 @@ authorityKeyIdentifier = keyid:always
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: false,
|
rsync_dedup: false,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
let first = runner_rrdp
|
let first = runner_rrdp
|
||||||
@ -5757,6 +5813,7 @@ authorityKeyIdentifier = keyid:always
|
|||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: Mutex::new(HashMap::new()),
|
rsync_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
repo_sync_runtime: None,
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
let third = runner_rsync
|
let third = runner_rsync
|
||||||
|
|||||||
@ -297,6 +297,13 @@ fn decode_validate(obj_type: ObjType, bytes: &[u8]) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn landing_packfile_cbor_put(store: &RocksStore, obj_type: ObjType, sample: &str, bytes: &[u8]) {
|
fn landing_packfile_cbor_put(store: &RocksStore, obj_type: ObjType, sample: &str, bytes: &[u8]) {
|
||||||
|
#[derive(serde::Serialize)]
|
||||||
|
struct BenchPackFile<'a> {
|
||||||
|
rsync_uri: &'a str,
|
||||||
|
bytes: &'a [u8],
|
||||||
|
sha256: [u8; 32],
|
||||||
|
}
|
||||||
|
|
||||||
let rsync_uri = format!(
|
let rsync_uri = format!(
|
||||||
"rsync://bench.invalid/{}/{}.{}",
|
"rsync://bench.invalid/{}/{}.{}",
|
||||||
obj_type.as_str(),
|
obj_type.as_str(),
|
||||||
@ -304,7 +311,13 @@ fn landing_packfile_cbor_put(store: &RocksStore, obj_type: ObjType, sample: &str
|
|||||||
obj_type.ext()
|
obj_type.ext()
|
||||||
);
|
);
|
||||||
let pf = PackFile::from_bytes_compute_sha256(rsync_uri, bytes.to_vec());
|
let pf = PackFile::from_bytes_compute_sha256(rsync_uri, bytes.to_vec());
|
||||||
let encoded = serde_cbor::to_vec(std::hint::black_box(&pf)).expect("cbor encode packfile");
|
let bench_pf = BenchPackFile {
|
||||||
|
rsync_uri: &pf.rsync_uri,
|
||||||
|
bytes: pf.bytes().expect("load packfile bytes"),
|
||||||
|
sha256: pf.sha256,
|
||||||
|
};
|
||||||
|
let encoded =
|
||||||
|
serde_cbor::to_vec(std::hint::black_box(&bench_pf)).expect("cbor encode packfile");
|
||||||
let key = format!("bench:packfile:{}:{}", obj_type.as_str(), sample);
|
let key = format!("bench:packfile:{}:{}", obj_type.as_str(), sample);
|
||||||
let sha256_hex = hex::encode(sha2::Sha256::digest(&encoded));
|
let sha256_hex = hex::encode(sha2::Sha256::digest(&encoded));
|
||||||
let mut entry = RawByHashEntry::from_bytes(sha256_hex, encoded);
|
let mut entry = RawByHashEntry::from_bytes(sha256_hex, encoded);
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
use std::cell::RefCell;
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use rpki::fetch::http::{BlockingHttpFetcher, HttpFetcherConfig};
|
use rpki::fetch::http::{BlockingHttpFetcher, HttpFetcherConfig};
|
||||||
@ -52,7 +52,7 @@ impl RsyncFetcher for AlwaysFailRsyncFetcher {
|
|||||||
struct CountingDenyUriFetcher {
|
struct CountingDenyUriFetcher {
|
||||||
inner: BlockingHttpFetcher,
|
inner: BlockingHttpFetcher,
|
||||||
deny_uri: String,
|
deny_uri: String,
|
||||||
counts: std::rc::Rc<RefCell<HashMap<String, u64>>>,
|
counts: Arc<Mutex<HashMap<String, u64>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CountingDenyUriFetcher {
|
impl CountingDenyUriFetcher {
|
||||||
@ -60,18 +60,28 @@ impl CountingDenyUriFetcher {
|
|||||||
Self {
|
Self {
|
||||||
inner,
|
inner,
|
||||||
deny_uri,
|
deny_uri,
|
||||||
counts: std::rc::Rc::new(RefCell::new(HashMap::new())),
|
counts: Arc::new(Mutex::new(HashMap::new())),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn count(&self, uri: &str) -> u64 {
|
fn count(&self, uri: &str) -> u64 {
|
||||||
*self.counts.borrow().get(uri).unwrap_or(&0)
|
*self
|
||||||
|
.counts
|
||||||
|
.lock()
|
||||||
|
.expect("lock counts")
|
||||||
|
.get(uri)
|
||||||
|
.unwrap_or(&0)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Fetcher for CountingDenyUriFetcher {
|
impl Fetcher for CountingDenyUriFetcher {
|
||||||
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
|
fn fetch(&self, uri: &str) -> Result<Vec<u8>, String> {
|
||||||
*self.counts.borrow_mut().entry(uri.to_string()).or_insert(0) += 1;
|
*self
|
||||||
|
.counts
|
||||||
|
.lock()
|
||||||
|
.expect("lock counts")
|
||||||
|
.entry(uri.to_string())
|
||||||
|
.or_insert(0) += 1;
|
||||||
if uri == self.deny_uri {
|
if uri == self.deny_uri {
|
||||||
return Err(format!("snapshot fetch denied: {uri}"));
|
return Err(format!("snapshot fetch denied: {uri}"));
|
||||||
}
|
}
|
||||||
|
|||||||
@ -88,7 +88,7 @@ impl LiveStats {
|
|||||||
|
|
||||||
if f.rsync_uri.ends_with(".crl") {
|
if f.rsync_uri.ends_with(".crl") {
|
||||||
self.crl_total += 1;
|
self.crl_total += 1;
|
||||||
if RpkixCrl::decode_der(&f.bytes).is_ok() {
|
if f.bytes().ok().and_then(|b| RpkixCrl::decode_der(b).ok()).is_some() {
|
||||||
self.crl_decode_ok += 1;
|
self.crl_decode_ok += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -176,6 +176,8 @@ fn apnic_tree_full_stats_serial() {
|
|||||||
rrdp_repo_cache: std::sync::Mutex::new(std::collections::HashMap::new()),
|
rrdp_repo_cache: std::sync::Mutex::new(std::collections::HashMap::new()),
|
||||||
rsync_dedup: true,
|
rsync_dedup: true,
|
||||||
rsync_repo_cache: std::sync::Mutex::new(std::collections::HashMap::new()),
|
rsync_repo_cache: std::sync::Mutex::new(std::collections::HashMap::new()),
|
||||||
|
current_repo_index: None,
|
||||||
|
repo_sync_runtime: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
let stats = RefCell::new(LiveStats::default());
|
let stats = RefCell::new(LiveStats::default());
|
||||||
|
|||||||
@ -88,8 +88,6 @@ fn cli_run_offline_mode_writes_cir_and_static_pool() {
|
|||||||
let repo_dir = tempfile::tempdir().expect("repo tempdir");
|
let repo_dir = tempfile::tempdir().expect("repo tempdir");
|
||||||
let out_dir = tempfile::tempdir().expect("out tempdir");
|
let out_dir = tempfile::tempdir().expect("out tempdir");
|
||||||
let cir_path = out_dir.path().join("result.cir");
|
let cir_path = out_dir.path().join("result.cir");
|
||||||
let static_root = out_dir.path().join("static");
|
|
||||||
|
|
||||||
let policy_path = out_dir.path().join("policy.toml");
|
let policy_path = out_dir.path().join("policy.toml");
|
||||||
std::fs::write(&policy_path, "sync_preference = \"rsync_only\"\n").expect("write policy");
|
std::fs::write(&policy_path, "sync_preference = \"rsync_only\"\n").expect("write policy");
|
||||||
|
|
||||||
@ -117,8 +115,6 @@ fn cli_run_offline_mode_writes_cir_and_static_pool() {
|
|||||||
"--cir-enable".to_string(),
|
"--cir-enable".to_string(),
|
||||||
"--cir-out".to_string(),
|
"--cir-out".to_string(),
|
||||||
cir_path.to_string_lossy().to_string(),
|
cir_path.to_string_lossy().to_string(),
|
||||||
"--cir-static-root".to_string(),
|
|
||||||
static_root.to_string_lossy().to_string(),
|
|
||||||
"--cir-tal-uri".to_string(),
|
"--cir-tal-uri".to_string(),
|
||||||
"https://example.test/root.tal".to_string(),
|
"https://example.test/root.tal".to_string(),
|
||||||
];
|
];
|
||||||
@ -134,21 +130,6 @@ fn cli_run_offline_mode_writes_cir_and_static_pool() {
|
|||||||
.iter()
|
.iter()
|
||||||
.any(|item| item.rsync_uri.contains("apnic-rpki-root-iana-origin.cer"))
|
.any(|item| item.rsync_uri.contains("apnic-rpki-root-iana-origin.cer"))
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut file_count = 0usize;
|
|
||||||
let mut stack = vec![static_root.clone()];
|
|
||||||
while let Some(path) = stack.pop() {
|
|
||||||
for entry in std::fs::read_dir(path).expect("read_dir") {
|
|
||||||
let entry = entry.expect("entry");
|
|
||||||
let path = entry.path();
|
|
||||||
if path.is_dir() {
|
|
||||||
stack.push(path);
|
|
||||||
} else {
|
|
||||||
file_count += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assert!(file_count >= 1);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@ -19,11 +19,10 @@ fn wrapper_script() -> std::path::PathBuf {
|
|||||||
#[test]
|
#[test]
|
||||||
fn multi_rir_case_info_resolves_all_five_rirs_and_timings() {
|
fn multi_rir_case_info_resolves_all_five_rirs_and_timings() {
|
||||||
let bundle_root = multi_rir_bundle_root();
|
let bundle_root = multi_rir_bundle_root();
|
||||||
assert!(
|
if !bundle_root.is_dir() {
|
||||||
bundle_root.is_dir(),
|
eprintln!("skipping multi-rir case info test; bundle root missing: {}", bundle_root.display());
|
||||||
"bundle root missing: {}",
|
return;
|
||||||
bundle_root.display()
|
}
|
||||||
);
|
|
||||||
|
|
||||||
let expected = [
|
let expected = [
|
||||||
("afrinic", "afrinic", "afrinic.tal", "afrinic-ta.cer"),
|
("afrinic", "afrinic", "afrinic.tal", "afrinic-ta.cer"),
|
||||||
@ -210,11 +209,13 @@ stderr={}",
|
|||||||
#[test]
|
#[test]
|
||||||
fn multi_rir_wrapper_describe_mode_works_for_ripe() {
|
fn multi_rir_wrapper_describe_mode_works_for_ripe() {
|
||||||
let bundle_root = multi_rir_bundle_root();
|
let bundle_root = multi_rir_bundle_root();
|
||||||
assert!(
|
if !bundle_root.is_dir() {
|
||||||
bundle_root.is_dir(),
|
eprintln!(
|
||||||
"bundle root missing: {}",
|
"skipping multi-rir wrapper describe test; bundle root missing: {}",
|
||||||
bundle_root.display()
|
bundle_root.display()
|
||||||
);
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
let out = Command::new(wrapper_script())
|
let out = Command::new(wrapper_script())
|
||||||
.env("BUNDLE_ROOT", &bundle_root)
|
.env("BUNDLE_ROOT", &bundle_root)
|
||||||
|
|||||||
@ -28,7 +28,9 @@ fn run_case(
|
|||||||
"--parallel-phase1".to_string(),
|
"--parallel-phase1".to_string(),
|
||||||
"--disable-rrdp".to_string(),
|
"--disable-rrdp".to_string(),
|
||||||
"--rsync-local-dir".to_string(),
|
"--rsync-local-dir".to_string(),
|
||||||
fixture("tests/fixtures/repository").to_string_lossy().to_string(),
|
fixture("tests/fixtures/repository")
|
||||||
|
.to_string_lossy()
|
||||||
|
.to_string(),
|
||||||
"--validation-time".to_string(),
|
"--validation-time".to_string(),
|
||||||
"2026-04-07T00:00:00Z".to_string(),
|
"2026-04-07T00:00:00Z".to_string(),
|
||||||
"--max-depth".to_string(),
|
"--max-depth".to_string(),
|
||||||
@ -67,14 +69,19 @@ fn multi_tal_parallel_output_matches_union_of_single_tal_outputs() {
|
|||||||
let arin_tal = fixture("tests/fixtures/tal/arin.tal");
|
let arin_tal = fixture("tests/fixtures/tal/arin.tal");
|
||||||
let arin_ta = fixture("tests/fixtures/ta/arin-ta.cer");
|
let arin_ta = fixture("tests/fixtures/ta/arin-ta.cer");
|
||||||
|
|
||||||
let (apnic_report, apnic_vrps, apnic_vaps, _) =
|
let (apnic_report, apnic_vrps, apnic_vaps, _) = run_case("apnic", &[(&apnic_tal, &apnic_ta)]);
|
||||||
run_case("apnic", &[(&apnic_tal, &apnic_ta)]);
|
|
||||||
let (arin_report, arin_vrps, arin_vaps, _) = run_case("arin", &[(&arin_tal, &arin_ta)]);
|
let (arin_report, arin_vrps, arin_vaps, _) = run_case("arin", &[(&arin_tal, &arin_ta)]);
|
||||||
let (multi_report, multi_vrps, multi_vaps, multi_ccr) =
|
let (multi_report, multi_vrps, multi_vaps, multi_ccr) =
|
||||||
run_case("multi", &[(&apnic_tal, &apnic_ta), (&arin_tal, &arin_ta)]);
|
run_case("multi", &[(&apnic_tal, &apnic_ta), (&arin_tal, &arin_ta)]);
|
||||||
|
|
||||||
let expected_vrps = apnic_vrps.union(&arin_vrps).cloned().collect::<BTreeSet<_>>();
|
let expected_vrps = apnic_vrps
|
||||||
let expected_vaps = apnic_vaps.union(&arin_vaps).cloned().collect::<BTreeSet<_>>();
|
.union(&arin_vrps)
|
||||||
|
.cloned()
|
||||||
|
.collect::<BTreeSet<_>>();
|
||||||
|
let expected_vaps = apnic_vaps
|
||||||
|
.union(&arin_vaps)
|
||||||
|
.cloned()
|
||||||
|
.collect::<BTreeSet<_>>();
|
||||||
|
|
||||||
assert_eq!(multi_vrps, expected_vrps);
|
assert_eq!(multi_vrps, expected_vrps);
|
||||||
assert_eq!(multi_vaps, expected_vaps);
|
assert_eq!(multi_vaps, expected_vaps);
|
||||||
|
|||||||
@ -113,7 +113,7 @@ fn build_cernet_pack_and_validation_time() -> (
|
|||||||
.iter()
|
.iter()
|
||||||
.find(|f| f.rsync_uri.ends_with(".crl"))
|
.find(|f| f.rsync_uri.ends_with(".crl"))
|
||||||
.expect("crl present in snapshot");
|
.expect("crl present in snapshot");
|
||||||
let crl = RpkixCrl::decode_der(&crl_file.bytes).expect("decode crl");
|
let crl = RpkixCrl::decode_der(crl_file.bytes().expect("load crl bytes")).expect("decode crl");
|
||||||
|
|
||||||
let mut t = manifest.manifest.this_update;
|
let mut t = manifest.manifest.this_update;
|
||||||
if issuer_ca.tbs.validity_not_before > t {
|
if issuer_ca.tbs.validity_not_before > t {
|
||||||
|
|||||||
@ -113,7 +113,7 @@ fn build_cernet_pack_and_validation_time() -> (
|
|||||||
.iter()
|
.iter()
|
||||||
.find(|f| f.rsync_uri.ends_with(".crl"))
|
.find(|f| f.rsync_uri.ends_with(".crl"))
|
||||||
.expect("crl present in snapshot");
|
.expect("crl present in snapshot");
|
||||||
let crl = RpkixCrl::decode_der(&crl_file.bytes).expect("decode crl");
|
let crl = RpkixCrl::decode_der(crl_file.bytes().expect("load crl bytes")).expect("decode crl");
|
||||||
|
|
||||||
// Choose a validation_time that is within:
|
// Choose a validation_time that is within:
|
||||||
// - manifest thisUpdate..nextUpdate (RFC 9286 §6.3)
|
// - manifest thisUpdate..nextUpdate (RFC 9286 §6.3)
|
||||||
@ -151,7 +151,9 @@ fn drop_object_policy_drops_only_failing_object() {
|
|||||||
.expect("another ROA present in snapshot");
|
.expect("another ROA present in snapshot");
|
||||||
let victim_uri = pack.files[tamper_idx].rsync_uri.clone();
|
let victim_uri = pack.files[tamper_idx].rsync_uri.clone();
|
||||||
|
|
||||||
let mut tampered = pack.files[tamper_idx].bytes.clone();
|
let mut tampered = pack.files[tamper_idx]
|
||||||
|
.bytes_cloned()
|
||||||
|
.expect("clone roa bytes");
|
||||||
let last = tampered.len() - 1;
|
let last = tampered.len() - 1;
|
||||||
tampered[last] ^= 0xFF;
|
tampered[last] ^= 0xFF;
|
||||||
pack.files[tamper_idx] = PackFile::from_bytes_compute_sha256(victim_uri.clone(), tampered);
|
pack.files[tamper_idx] = PackFile::from_bytes_compute_sha256(victim_uri.clone(), tampered);
|
||||||
@ -194,7 +196,9 @@ fn drop_publication_point_policy_drops_the_publication_point() {
|
|||||||
.expect("a ROA present in snapshot");
|
.expect("a ROA present in snapshot");
|
||||||
let victim_uri = pack.files[tamper_idx].rsync_uri.clone();
|
let victim_uri = pack.files[tamper_idx].rsync_uri.clone();
|
||||||
|
|
||||||
let mut tampered = pack.files[tamper_idx].bytes.clone();
|
let mut tampered = pack.files[tamper_idx]
|
||||||
|
.bytes_cloned()
|
||||||
|
.expect("clone roa bytes");
|
||||||
let last = tampered.len() - 1;
|
let last = tampered.len() - 1;
|
||||||
tampered[last] ^= 0xFF;
|
tampered[last] ^= 0xFF;
|
||||||
pack.files[tamper_idx] = PackFile::from_bytes_compute_sha256(victim_uri.clone(), tampered);
|
pack.files[tamper_idx] = PackFile::from_bytes_compute_sha256(victim_uri.clone(), tampered);
|
||||||
|
|||||||
@ -61,10 +61,8 @@ fn offline_serial_and_parallel_phase1_match_compare_views() {
|
|||||||
|
|
||||||
let (serial_vrps, serial_vaps) =
|
let (serial_vrps, serial_vaps) =
|
||||||
rpki::bundle::decode_ccr_compare_views(&serial_ccr, "apnic").expect("serial compare view");
|
rpki::bundle::decode_ccr_compare_views(&serial_ccr, "apnic").expect("serial compare view");
|
||||||
let (parallel_vrps, parallel_vaps) = rpki::bundle::decode_ccr_compare_views(
|
let (parallel_vrps, parallel_vaps) =
|
||||||
¶llel_ccr,
|
rpki::bundle::decode_ccr_compare_views(¶llel_ccr, "apnic")
|
||||||
"apnic",
|
|
||||||
)
|
|
||||||
.expect("parallel compare view");
|
.expect("parallel compare view");
|
||||||
|
|
||||||
assert_eq!(serial_vrps, parallel_vrps, "VRP compare views must match");
|
assert_eq!(serial_vrps, parallel_vrps, "VRP compare views must match");
|
||||||
|
|||||||
@ -17,11 +17,13 @@ fn signed_object_decode_accepts_real_arin_ber_indefinite_roa_from_multi_rir_bund
|
|||||||
let xml_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(
|
let xml_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(
|
||||||
"../../rpki/target/live/20260316-112341-multi-final3/arin/base-payload-archive/v1/captures/arin-base-arin/rrdp/repos/8bd1405242614aed9f13321d266fe3193db0ca842e0cbffda2b3df05481c331b/4a394319-7460-4141-a416-1addb69284ff/snapshot-160090-880434ae2a6f0e5fff224391c65a22ab037e09ac1d3ebad0ceda18558b49b13e.xml",
|
"../../rpki/target/live/20260316-112341-multi-final3/arin/base-payload-archive/v1/captures/arin-base-arin/rrdp/repos/8bd1405242614aed9f13321d266fe3193db0ca842e0cbffda2b3df05481c331b/4a394319-7460-4141-a416-1addb69284ff/snapshot-160090-880434ae2a6f0e5fff224391c65a22ab037e09ac1d3ebad0ceda18558b49b13e.xml",
|
||||||
);
|
);
|
||||||
assert!(
|
if !xml_path.is_file() {
|
||||||
xml_path.is_file(),
|
eprintln!(
|
||||||
"xml path missing: {}",
|
"skipping BER-indefinite signed object fixture test; xml path missing: {}",
|
||||||
xml_path.display()
|
xml_path.display()
|
||||||
);
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
let xml = std::fs::read_to_string(&xml_path).expect("read snapshot xml");
|
let xml = std::fs::read_to_string(&xml_path).expect("read snapshot xml");
|
||||||
let uri = "rsync://rpki.arin.net/repository/arin-rpki-ta/5e4a23ea-e80a-403e-b08c-2171da2157d3/2a246947-2d62-4a6c-ba05-87187f0099b2/9ed5ce80-224e-46ab-94f1-1afce8ccf13f/0b13beb5-6bbb-3994-a254-02c5b10175c5.roa";
|
let uri = "rsync://rpki.arin.net/repository/arin-rpki-ta/5e4a23ea-e80a-403e-b08c-2171da2157d3/2a246947-2d62-4a6c-ba05-87187f0099b2/9ed5ce80-224e-46ab-94f1-1afce8ccf13f/0b13beb5-6bbb-3994-a254-02c5b10175c5.roa";
|
||||||
let der = extract_publish_bytes(&xml, uri);
|
let der = extract_publish_bytes(&xml, uri);
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user