20260427 拆分独立repo-bytes.db数据库文件,cir materialize 移除旧兼容

This commit is contained in:
2026-04-27 16:01:33 +08:00
parent 944ea6ca00
commit eaa375c5ec
53 changed files with 950 additions and 595 deletions

View File

@@ -1,5 +1,7 @@
use rpki::ccr::{
compute_state_hash, decode_content_info, dump_content_info_json_value,
CcrContentInfo, CcrDigestAlgorithm, ManifestInstance, ManifestState, RoaPayloadSet,
RoaPayloadState, RouterKey, RouterKeySet, RouterKeyState, TrustAnchorState, compute_state_hash,
decode_content_info, dump_content_info_json_value,
encode::{
encode_aspa_payload_state_payload_der, encode_content_info,
encode_manifest_state_payload_der, encode_roa_payload_state_payload_der,
@@ -8,8 +10,6 @@ use rpki::ccr::{
verify::{
verify_against_report_json_path, verify_against_vcir_store, verify_content_info_bytes,
},
CcrContentInfo, CcrDigestAlgorithm, ManifestInstance, ManifestState, RoaPayloadSet,
RoaPayloadState, RouterKey, RouterKeySet, RouterKeyState, TrustAnchorState,
};
use rpki::data_model::common::BigUnsigned;
use rpki::storage::{

View File

@@ -1,4 +1,3 @@
use std::collections::BTreeSet;
use std::path::PathBuf;
use std::process::Command;
@@ -15,7 +14,7 @@ fn skip_heavy_blackbox_test() -> bool {
}
#[test]
fn cir_full_and_delta_pair_reuses_shared_static_pool() {
fn cir_full_and_delta_pair_reuses_shared_repo_bytes_db() {
if skip_heavy_blackbox_test() {
return;
}
@@ -25,8 +24,6 @@ fn cir_full_and_delta_pair_reuses_shared_static_pool() {
let out = out_dir.path().join("cir-pair");
let fixture_root = out_dir.path().join("fixture");
std::fs::create_dir_all(&fixture_root).unwrap();
let static_payload_root = fixture_root.join("payloads");
std::fs::create_dir_all(&static_payload_root).unwrap();
let base_locks = fixture_root.join("base-locks.json");
let delta_locks = fixture_root.join("locks-delta.json");
std::fs::write(&base_locks, br#"{"validationTime":"2026-03-16T11:49:15Z"}"#).unwrap();
@@ -130,7 +127,7 @@ MODE=""
cir=""
ccr=""
report=""
static_root=""
repo_bytes_db=""
while [[ $# -gt 0 ]]; do
case "$1" in
--payload-replay-archive) MODE="full"; shift 2 ;;
@@ -138,22 +135,19 @@ while [[ $# -gt 0 ]]; do
--cir-out) cir="$2"; shift 2 ;;
--ccr-out) ccr="$2"; shift 2 ;;
--report-json) report="$2"; shift 2 ;;
--cir-static-root) static_root="$2"; shift 2 ;;
--repo-bytes-db) repo_bytes_db="$2"; shift 2 ;;
*) shift ;;
esac
done
mkdir -p "$(dirname "$cir")" "$(dirname "$ccr")" "$(dirname "$report")" "$static_root/20260316/{{ab,cd,ef}}/00"
mkdir -p "$(dirname "$cir")" "$(dirname "$ccr")" "$(dirname "$report")" "$repo_bytes_db"
if [[ "$MODE" == "full" ]]; then
cp "{full_cir}" "$cir"
cp "{full_ccr}" "$ccr"
cp "{full_report}" "$report"
install -D -m 0644 "{payload_root}/full-object" "$static_root/20260316/ab/cd/{full_hash}"
else
cp "{delta_cir}" "$cir"
cp "{delta_ccr}" "$ccr"
cp "{delta_report}" "$report"
install -D -m 0644 "{payload_root}/full-object" "$static_root/20260316/ab/cd/{full_hash}"
install -D -m 0644 "{payload_root}/delta-object" "$static_root/20260316/ef/00/{delta_hash}"
fi
"#,
full_cir = full_cir_path.display(),
@@ -162,15 +156,10 @@ fi
delta_ccr = delta_ccr_path.display(),
full_report = full_report_path.display(),
delta_report = delta_report_path.display(),
payload_root = static_payload_root.display(),
full_hash = full_obj_hash,
delta_hash = delta_obj_hash,
),
)
.unwrap();
std::fs::set_permissions(&stub, std::os::unix::fs::PermissionsExt::from_mode(0o755)).unwrap();
std::fs::write(static_payload_root.join("full-object"), b"full-object").unwrap();
std::fs::write(static_payload_root.join("delta-object"), b"delta-object").unwrap();
let proc = Command::new(script)
.args([
@@ -222,30 +211,13 @@ fi
rpki::cir::decode_cir(&std::fs::read(out.join("delta-001").join("input.cir")).unwrap())
.expect("decode delta cir");
let mut hashes = BTreeSet::new();
for item in &full_cir.objects {
hashes.insert(hex::encode(&item.sha256));
}
for item in &delta_cir.objects {
hashes.insert(hex::encode(&item.sha256));
}
assert!(!full_cir.objects.is_empty());
assert!(!delta_cir.objects.is_empty());
let static_file_count = walk(out.join("static")).len();
assert_eq!(static_file_count, hashes.len());
assert!(out.join("summary.json").is_file());
let summary: serde_json::Value =
serde_json::from_slice(&std::fs::read(out.join("summary.json")).unwrap()).unwrap();
assert_eq!(summary["repoBytesDbPath"], "repo-bytes.db");
assert_eq!(summary["repoBytesDbExists"], true);
assert!(out.join("full").join("result.ccr").is_file());
assert!(out.join("delta-001").join("result.ccr").is_file());
}
fn walk(path: std::path::PathBuf) -> Vec<std::path::PathBuf> {
let mut out = Vec::new();
if path.is_file() {
out.push(path);
} else if path.is_dir() {
for entry in std::fs::read_dir(path).unwrap() {
out.extend(walk(entry.unwrap().path()));
}
}
out
}

View File

@@ -1,6 +1,7 @@
use std::path::PathBuf;
use std::process::Command;
use rpki::blob_store::ExternalRepoBytesDb;
use rpki::ccr::{
CcrContentInfo, CcrDigestAlgorithm, RpkiCanonicalCacheRepresentation, TrustAnchorState,
encode_content_info,
@@ -15,7 +16,7 @@ fn cir_drop_report_counts_dropped_roa_objects_and_vrps() {
let cir_path = td.path().join("input.cir");
let ccr_path = td.path().join("output.ccr");
let report_path = td.path().join("report.json");
let static_root = td.path().join("static");
let repo_bytes_db = td.path().join("repo-bytes.db");
let json_out = td.path().join("drop.json");
let md_out = td.path().join("drop.md");
@@ -26,12 +27,10 @@ fn cir_drop_report_counts_dropped_roa_objects_and_vrps() {
use sha2::{Digest, Sha256};
hex::encode(Sha256::digest(&roa_bytes))
};
let dir = static_root
.join("20260409")
.join(&hash[0..2])
.join(&hash[2..4]);
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join(&hash), &roa_bytes).unwrap();
ExternalRepoBytesDb::open(&repo_bytes_db)
.expect("open repo bytes")
.put_blob_bytes_batch(&[(hash.clone(), roa_bytes.clone())])
.expect("write repo bytes");
let cir = CanonicalInputRepresentation {
version: CIR_VERSION_V1,
@@ -96,8 +95,8 @@ fn cir_drop_report_counts_dropped_roa_objects_and_vrps() {
ccr_path.to_string_lossy().as_ref(),
"--report-json",
report_path.to_string_lossy().as_ref(),
"--static-root",
static_root.to_string_lossy().as_ref(),
"--repo-bytes-db",
repo_bytes_db.to_string_lossy().as_ref(),
"--json-out",
json_out.to_string_lossy().as_ref(),
"--md-out",

View File

@@ -1,9 +1,10 @@
use std::path::{Path, PathBuf};
use std::process::Command;
use rpki::blob_store::ExternalRepoBytesDb;
use rpki::cir::{
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal, encode_cir,
materialize_cir,
materialize_cir_from_repo_bytes,
};
fn skip_heavy_script_replay_test() -> bool {
@@ -55,12 +56,13 @@ fn build_ta_only_cir() -> (CanonicalInputRepresentation, Vec<u8>) {
)
}
fn write_static(root: &Path, date: &str, bytes: &[u8]) {
fn write_repo_bytes(repo_bytes_db: &Path, bytes: &[u8]) {
use sha2::{Digest, Sha256};
let hash = hex::encode(Sha256::digest(bytes));
let dir = root.join(date).join(&hash[0..2]).join(&hash[2..4]);
std::fs::create_dir_all(&dir).expect("mkdir static");
std::fs::write(dir.join(hash), bytes).expect("write static object");
ExternalRepoBytesDb::open(repo_bytes_db)
.expect("open repo bytes")
.put_blob_bytes_batch(&[(hash, bytes.to_vec())])
.expect("write repo bytes");
}
fn prepare_reference_ccr(
@@ -120,15 +122,15 @@ fn cir_replay_matrix_script_matches_reference_for_all_participants() {
}
let td = tempfile::tempdir().expect("tempdir");
let static_root = td.path().join("static");
let repo_bytes_db = td.path().join("repo-bytes.db");
let cir_path = td.path().join("sample.cir");
let mirror_root = td.path().join("mirror");
let out_dir = td.path().join("matrix-out");
let (cir, ta_bytes) = build_ta_only_cir();
std::fs::write(&cir_path, encode_cir(&cir).expect("encode cir")).expect("write cir");
write_static(&static_root, "20260407", &ta_bytes);
materialize_cir(&cir, &static_root, &mirror_root, true).expect("materialize");
write_repo_bytes(&repo_bytes_db, &ta_bytes);
materialize_cir_from_repo_bytes(&cir, &repo_bytes_db, &mirror_root, true).expect("materialize");
let reference_ccr = prepare_reference_ccr(td.path(), &cir, &mirror_root);
let script =
@@ -146,8 +148,8 @@ fn cir_replay_matrix_script_matches_reference_for_all_participants() {
.args([
"--cir",
cir_path.to_string_lossy().as_ref(),
"--static-root",
static_root.to_string_lossy().as_ref(),
"--repo-bytes-db",
repo_bytes_db.to_string_lossy().as_ref(),
"--out-dir",
out_dir.to_string_lossy().as_ref(),
"--reference-ccr",

View File

@@ -1,9 +1,10 @@
use std::path::{Path, PathBuf};
use std::process::Command;
use rpki::blob_store::ExternalRepoBytesDb;
use rpki::cir::{
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal, encode_cir,
materialize_cir,
materialize_cir_from_repo_bytes,
};
fn skip_heavy_script_replay_test() -> bool {
@@ -55,12 +56,13 @@ fn build_ta_only_cir() -> (CanonicalInputRepresentation, Vec<u8>) {
)
}
fn write_static(root: &Path, date: &str, bytes: &[u8]) {
fn write_repo_bytes(repo_bytes_db: &Path, bytes: &[u8]) {
use sha2::{Digest, Sha256};
let hash = hex::encode(Sha256::digest(bytes));
let dir = root.join(date).join(&hash[0..2]).join(&hash[2..4]);
std::fs::create_dir_all(&dir).expect("mkdir static");
std::fs::write(dir.join(hash), bytes).expect("write static object");
ExternalRepoBytesDb::open(repo_bytes_db)
.expect("open repo bytes")
.put_blob_bytes_batch(&[(hash, bytes.to_vec())])
.expect("write repo bytes");
}
fn prepare_reference_ccr(
@@ -118,15 +120,15 @@ fn cir_routinator_script_matches_reference_on_ta_only_cir() {
return;
}
let td = tempfile::tempdir().expect("tempdir");
let static_root = td.path().join("static");
let repo_bytes_db = td.path().join("repo-bytes.db");
let cir_path = td.path().join("sample.cir");
let mirror_root = td.path().join("mirror");
let out_dir = td.path().join("routinator-out");
let (cir, ta_bytes) = build_ta_only_cir();
std::fs::write(&cir_path, encode_cir(&cir).expect("encode cir")).expect("write cir");
write_static(&static_root, "20260407", &ta_bytes);
materialize_cir(&cir, &static_root, &mirror_root, true).expect("materialize");
write_repo_bytes(&repo_bytes_db, &ta_bytes);
materialize_cir_from_repo_bytes(&cir, &repo_bytes_db, &mirror_root, true).expect("materialize");
let reference_ccr = prepare_reference_ccr(td.path(), &cir, &mirror_root);
let script =
@@ -144,8 +146,8 @@ fn cir_routinator_script_matches_reference_on_ta_only_cir() {
.args([
"--cir",
cir_path.to_string_lossy().as_ref(),
"--static-root",
static_root.to_string_lossy().as_ref(),
"--repo-bytes-db",
repo_bytes_db.to_string_lossy().as_ref(),
"--out-dir",
out_dir.to_string_lossy().as_ref(),
"--reference-ccr",
@@ -177,15 +179,15 @@ fn cir_rpki_client_script_matches_reference_on_ta_only_cir() {
return;
}
let td = tempfile::tempdir().expect("tempdir");
let static_root = td.path().join("static");
let repo_bytes_db = td.path().join("repo-bytes.db");
let cir_path = td.path().join("sample.cir");
let mirror_root = td.path().join("mirror");
let out_dir = td.path().join("rpki-client-out");
let (cir, ta_bytes) = build_ta_only_cir();
std::fs::write(&cir_path, encode_cir(&cir).expect("encode cir")).expect("write cir");
write_static(&static_root, "20260407", &ta_bytes);
materialize_cir(&cir, &static_root, &mirror_root, true).expect("materialize");
write_repo_bytes(&repo_bytes_db, &ta_bytes);
materialize_cir_from_repo_bytes(&cir, &repo_bytes_db, &mirror_root, true).expect("materialize");
let reference_ccr = prepare_reference_ccr(td.path(), &cir, &mirror_root);
let script =
@@ -203,8 +205,8 @@ fn cir_rpki_client_script_matches_reference_on_ta_only_cir() {
.args([
"--cir",
cir_path.to_string_lossy().as_ref(),
"--static-root",
static_root.to_string_lossy().as_ref(),
"--repo-bytes-db",
repo_bytes_db.to_string_lossy().as_ref(),
"--out-dir",
out_dir.to_string_lossy().as_ref(),
"--reference-ccr",

View File

@@ -132,7 +132,7 @@ MODE=""
cir=""
ccr=""
report=""
static_root=""
repo_bytes_db=""
while [[ $# -gt 0 ]]; do
case "$1" in
--payload-replay-archive) MODE="full"; shift 2 ;;
@@ -140,16 +140,15 @@ while [[ $# -gt 0 ]]; do
--cir-out) cir="$2"; shift 2 ;;
--ccr-out) ccr="$2"; shift 2 ;;
--report-json) report="$2"; shift 2 ;;
--cir-static-root) static_root="$2"; shift 2 ;;
--repo-bytes-db) repo_bytes_db="$2"; shift 2 ;;
*) shift ;;
esac
done
mkdir -p "$(dirname "$cir")" "$(dirname "$ccr")" "$(dirname "$report")" "$static_root/20260316/{{ab,cd,ef}}/00"
mkdir -p "$(dirname "$cir")" "$(dirname "$ccr")" "$(dirname "$report")" "$repo_bytes_db"
if [[ "$MODE" == "full" ]]; then
cp "{full_cir}" "$cir"
cp "{full_ccr}" "$ccr"
cp "{full_report}" "$report"
install -D -m 0644 "{payload_root}/full-object" "$static_root/20260316/ab/cd/{full_hash}"
else
if [[ "$cir" == *delta-001* ]]; then
cp "{delta_cir}" "$cir"
@@ -160,8 +159,6 @@ else
cp "{delta_ccr}" "$ccr"
cp "{delta_report}" "$report"
fi
install -D -m 0644 "{payload_root}/full-object" "$static_root/20260316/ab/cd/{full_hash}"
install -D -m 0644 "{payload_root}/delta-object" "$static_root/20260316/ef/00/{delta_hash}"
fi
"#,
full_cir = full_cir_path.display(),
@@ -170,9 +167,6 @@ fi
delta_ccr = delta_ccr_path.display(),
full_report = full_report_path.display(),
delta_report = delta_report_path.display(),
payload_root = static_payload_root.display(),
full_hash = full_hash,
delta_hash = delta_hash,
),
)
.unwrap();

View File

@@ -1,9 +1,10 @@
use std::path::{Path, PathBuf};
use std::process::Command;
use rpki::blob_store::ExternalRepoBytesDb;
use rpki::cir::{
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal, encode_cir,
materialize_cir,
materialize_cir_from_repo_bytes,
};
fn skip_heavy_script_replay_test() -> bool {
@@ -55,12 +56,13 @@ fn build_ta_only_cir() -> (CanonicalInputRepresentation, Vec<u8>) {
)
}
fn write_static(root: &Path, date: &str, bytes: &[u8]) {
fn write_repo_bytes(repo_bytes_db: &Path, bytes: &[u8]) {
use sha2::{Digest, Sha256};
let hash = hex::encode(Sha256::digest(bytes));
let dir = root.join(date).join(&hash[0..2]).join(&hash[2..4]);
std::fs::create_dir_all(&dir).expect("mkdir static");
std::fs::write(dir.join(hash), bytes).expect("write static object");
ExternalRepoBytesDb::open(repo_bytes_db)
.expect("open repo bytes")
.put_blob_bytes_batch(&[(hash, bytes.to_vec())])
.expect("write repo bytes");
}
fn prepare_reference_ccr(
@@ -110,7 +112,7 @@ fn prepare_reference_ccr(
fn prepare_sequence_root(td: &Path) -> PathBuf {
let sequence_root = td.join("sequence");
let static_root = sequence_root.join("static");
let repo_bytes_db = sequence_root.join("repo-bytes.db");
let mirror_root = td.join("mirror");
std::fs::create_dir_all(sequence_root.join("full")).unwrap();
std::fs::create_dir_all(sequence_root.join("delta-001")).unwrap();
@@ -129,8 +131,8 @@ fn prepare_sequence_root(td: &Path) -> PathBuf {
&cir_bytes,
)
.unwrap();
write_static(&static_root, "20260407", &ta_bytes);
materialize_cir(&cir, &static_root, &mirror_root, true).unwrap();
write_repo_bytes(&repo_bytes_db, &ta_bytes);
materialize_cir_from_repo_bytes(&cir, &repo_bytes_db, &mirror_root, true).unwrap();
let reference = prepare_reference_ccr(td, &cir, &mirror_root);
std::fs::copy(&reference, sequence_root.join("full").join("result.ccr")).unwrap();
std::fs::copy(
@@ -148,7 +150,7 @@ fn prepare_sequence_root(td: &Path) -> PathBuf {
std::fs::write(sequence_root.join("delta-002").join("report.json"), b"{}").unwrap();
let sequence = serde_json::json!({
"version": 1,
"staticRoot": "static",
"repoBytesDbPath": "repo-bytes.db",
"steps": [
{"stepId":"full","kind":"full","validationTime":"2026-04-07T00:00:00Z","cirPath":"full/input.cir","ccrPath":"full/result.ccr","reportPath":"full/report.json","previousStepId":null},
{"stepId":"delta-001","kind":"delta","validationTime":"2026-04-07T00:00:00Z","cirPath":"delta-001/input.cir","ccrPath":"delta-001/result.ccr","reportPath":"delta-001/report.json","previousStepId":"full"},

View File

@@ -1,9 +1,10 @@
use std::path::{Path, PathBuf};
use std::process::Command;
use rpki::blob_store::ExternalRepoBytesDb;
use rpki::cir::{
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal, encode_cir,
materialize_cir,
materialize_cir_from_repo_bytes,
};
fn skip_heavy_script_replay_test() -> bool {
@@ -55,12 +56,13 @@ fn build_ta_only_cir() -> (CanonicalInputRepresentation, Vec<u8>) {
)
}
fn write_static(root: &Path, date: &str, bytes: &[u8]) {
fn write_repo_bytes(repo_bytes_db: &Path, bytes: &[u8]) {
use sha2::{Digest, Sha256};
let hash = hex::encode(Sha256::digest(bytes));
let dir = root.join(date).join(&hash[0..2]).join(&hash[2..4]);
std::fs::create_dir_all(&dir).expect("mkdir static");
std::fs::write(dir.join(hash), bytes).expect("write static object");
ExternalRepoBytesDb::open(repo_bytes_db)
.expect("open repo bytes")
.put_blob_bytes_batch(&[(hash, bytes.to_vec())])
.expect("write repo bytes");
}
fn prepare_reference_ccr(
@@ -118,7 +120,7 @@ fn ours_sequence_replay_script_replays_all_steps() {
}
let td = tempfile::tempdir().expect("tempdir");
let sequence_root = td.path().join("sequence");
let static_root = sequence_root.join("static");
let repo_bytes_db = sequence_root.join("repo-bytes.db");
let mirror_root = td.path().join("mirror");
std::fs::create_dir_all(sequence_root.join("full")).unwrap();
std::fs::create_dir_all(sequence_root.join("delta-001")).unwrap();
@@ -137,8 +139,8 @@ fn ours_sequence_replay_script_replays_all_steps() {
&cir_bytes,
)
.unwrap();
write_static(&static_root, "20260407", &ta_bytes);
materialize_cir(&cir, &static_root, &mirror_root, true).unwrap();
write_repo_bytes(&repo_bytes_db, &ta_bytes);
materialize_cir_from_repo_bytes(&cir, &repo_bytes_db, &mirror_root, true).unwrap();
let reference = prepare_reference_ccr(td.path(), &cir, &mirror_root);
std::fs::copy(&reference, sequence_root.join("full").join("result.ccr")).unwrap();
std::fs::copy(
@@ -157,7 +159,7 @@ fn ours_sequence_replay_script_replays_all_steps() {
let sequence = serde_json::json!({
"version": 1,
"staticRoot": "static",
"repoBytesDbPath": "repo-bytes.db",
"steps": [
{"stepId":"full","kind":"full","validationTime":"2026-04-07T00:00:00Z","cirPath":"full/input.cir","ccrPath":"full/result.ccr","reportPath":"full/report.json","previousStepId":null},
{"stepId":"delta-001","kind":"delta","validationTime":"2026-04-07T00:00:00Z","cirPath":"delta-001/input.cir","ccrPath":"delta-001/result.ccr","reportPath":"delta-001/report.json","previousStepId":"full"},

View File

@@ -10,7 +10,7 @@ use rpki::storage::{
VcirArtifactValidationStatus, VcirAuditSummary, VcirCcrManifestProjection, VcirInstanceGate,
VcirRelatedArtifact, VcirSummary,
};
use rpki::validation::manifest::{process_manifest_publication_point, PublicationPointSource};
use rpki::validation::manifest::{PublicationPointSource, process_manifest_publication_point};
fn issuer_ca_fixture() -> Vec<u8> {
std::fs::read(

View File

@@ -11,8 +11,8 @@ use rpki::storage::{
VcirRelatedArtifact, VcirSummary,
};
use rpki::validation::manifest::{
process_manifest_publication_point, process_manifest_publication_point_after_repo_sync,
ManifestProcessError, PublicationPointSource,
ManifestProcessError, PublicationPointSource, process_manifest_publication_point,
process_manifest_publication_point_after_repo_sync,
};
fn issuer_ca_fixture_der() -> Vec<u8> {

View File

@@ -2,9 +2,9 @@ use rpki::parallel::config::ParallelPhase2Config;
use rpki::policy::{Policy, SignedObjectFailurePolicy};
use rpki::storage::{PackFile, PackTime};
use rpki::validation::objects::{
process_publication_point_for_issuer_parallel_roa_with_pool,
ParallelRoaWorkerPool, process_publication_point_for_issuer_parallel_roa_with_pool,
process_publication_point_snapshot_for_issuer,
process_publication_point_snapshot_for_issuer_parallel_roa, ParallelRoaWorkerPool,
process_publication_point_snapshot_for_issuer_parallel_roa,
};
use rpki::validation::publication_point::PublicationPointSnapshot;