NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260427 拆分独立repo-bytes.db数据库文件，cir materialize 移除旧兼容

Browse Source
This commit is contained in:
yuyr
2026-04-27 16:01:33 +08:00
parent 944ea6ca00
commit eaa375c5ec
53 changed files with 950 additions and 595 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
src/bin/cir_drop_report.rs
View File

@@ -1,13 +1,13 @@
use std::collections::{BTreeMap, BTreeSet};
use std::path::PathBuf;
use rpki::blob_store::{ExternalRawStoreDb, RawObjectStore};
use rpki::blob_store::ExternalRepoBytesDb;
use rpki::bundle::decode_ccr_compare_views;
use rpki::ccr::decode_content_info;
use rpki::cir::{decode_cir, resolve_static_pool_file};
use rpki::cir::decode_cir;
use rpki::data_model::roa::RoaObject;
const USAGE: &str = "Usage: cir_drop_report --cir <path> --ccr <path> --report-json <path> (--static-root <path> | --raw-store-db <path>) --json-out <path> --md-out <path>";
const USAGE: &str = "Usage: cir_drop_report --cir <path> --ccr <path> --report-json <path> --repo-bytes-db <path> --json-out <path> --md-out <path>";
#[derive(serde::Serialize)]
struct DroppedObjectRecord {
@@ -50,23 +50,11 @@ fn classify_reason(detail: Option<&str>, result: &str) -> String {
fn parse_args(
argv: &[String],
) -> Result<
(
PathBuf,
PathBuf,
PathBuf,
Option<PathBuf>,
Option<PathBuf>,
PathBuf,
PathBuf,
),
String,
> {
) -> Result<(PathBuf, PathBuf, PathBuf, PathBuf, PathBuf, PathBuf), String> {
let mut cir = None;
let mut ccr = None;
let mut report = None;
let mut static_root = None;
let mut raw_store_db = None;
let mut repo_bytes_db = None;
let mut json_out = None;
let mut md_out = None;
let mut i = 1usize;
@@ -86,16 +74,10 @@ fn parse_args(
argv.get(i).ok_or("--report-json requires a value")?,
));
}
"--static-root" => {
"--repo-bytes-db" => {
i += 1;
static_root = Some(PathBuf::from(
argv.get(i).ok_or("--static-root requires a value")?,
));
}
"--raw-store-db" => {
i += 1;
raw_store_db = Some(PathBuf::from(
argv.get(i).ok_or("--raw-store-db requires a value")?,
repo_bytes_db = Some(PathBuf::from(
argv.get(i).ok_or("--repo-bytes-db requires a value")?,
));
}
"--json-out" => {
@@ -119,8 +101,7 @@ fn parse_args(
cir.ok_or_else(|| format!("--cir is required\n\n{USAGE}"))?,
ccr.ok_or_else(|| format!("--ccr is required\n\n{USAGE}"))?,
report.ok_or_else(|| format!("--report-json is required\n\n{USAGE}"))?,
static_root,
raw_store_db,
repo_bytes_db.ok_or_else(|| format!("--repo-bytes-db is required\n\n{USAGE}"))?,
json_out.ok_or_else(|| format!("--json-out is required\n\n{USAGE}"))?,
md_out.ok_or_else(|| format!("--md-out is required\n\n{USAGE}"))?,
))
@@ -128,14 +109,7 @@ fn parse_args(
fn main() -> Result<(), String> {
let argv: Vec<String> = std::env::args().collect();
let (cir_path, ccr_path, report_path, static_root, raw_store_db, json_out, md_out) =
parse_args(&argv)?;
let backend_count = static_root.is_some() as u8 + raw_store_db.is_some() as u8;
if backend_count != 1 {
return Err(format!(
"must specify exactly one of --static-root or --raw-store-db\n\n{USAGE}"
));
}
let (cir_path, ccr_path, report_path, repo_bytes_db, json_out, md_out) = parse_args(&argv)?;
let cir = decode_cir(&std::fs::read(&cir_path).map_err(|e| format!("read cir failed: {e}"))?)
.map_err(|e| format!("decode cir failed: {e}"))?;
@@ -149,6 +123,8 @@ fn main() -> Result<(), String> {
&std::fs::read(&report_path).map_err(|e| format!("read report failed: {e}"))?,
)
.map_err(|e| format!("parse report failed: {e}"))?;
let repo_bytes = ExternalRepoBytesDb::open(&repo_bytes_db)
.map_err(|e| format!("open repo bytes db failed: {e}"))?;
let mut object_hash_by_uri = BTreeMap::new();
for object in &cir.objects {
@@ -189,18 +165,7 @@ fn main() -> Result<(), String> {
let mut derived_vrp_count = 0usize;
if kind == "roa" && !hash.is_empty() {
let bytes_opt = if let Some(static_root) = static_root.as_ref() {
match resolve_static_pool_file(static_root, &hash) {
Ok(path) => std::fs::read(&path).ok(),
Err(_) => None,
}
} else if let Some(raw_store_db) = raw_store_db.as_ref() {
ExternalRawStoreDb::open(raw_store_db)
.ok()
.and_then(|store| store.get_blob_bytes(&hash).ok().flatten())
} else {
None
};
let bytes_opt = repo_bytes.get_blob_bytes(&hash).ok().flatten();
match bytes_opt {
Some(bytes) => {
if let Ok(roa) = RoaObject::decode_der(&bytes) {

39
src/bin/cir_materialize.rs
View File

@@ -1,7 +1,7 @@
use std::path::PathBuf;
fn usage() -> &'static str {
"Usage: cir_materialize --cir <path> (--repo-bytes-db <path> | --raw-store-db <path> | --static-root <path>) --mirror-root <path> [--keep-db]"
"Usage: cir_materialize --cir <path> --repo-bytes-db <path> --mirror-root <path> [--keep-db]"
}
fn main() {
@@ -13,9 +13,7 @@ fn main() {
fn run(argv: Vec<String>) -> Result<(), String> {
let mut cir_path: Option<PathBuf> = None;
let mut static_root: Option<PathBuf> = None;
let mut repo_bytes_db: Option<PathBuf> = None;
let mut raw_store_db: Option<PathBuf> = None;
let mut mirror_root: Option<PathBuf> = None;
let mut keep_db = false;
@@ -27,24 +25,12 @@ fn run(argv: Vec<String>) -> Result<(), String> {
i += 1;
cir_path = Some(PathBuf::from(argv.get(i).ok_or("--cir requires a value")?));
}
"--static-root" => {
i += 1;
static_root = Some(PathBuf::from(
argv.get(i).ok_or("--static-root requires a value")?,
));
}
"--repo-bytes-db" => {
i += 1;
repo_bytes_db = Some(PathBuf::from(
argv.get(i).ok_or("--repo-bytes-db requires a value")?,
));
}
"--raw-store-db" => {
i += 1;
raw_store_db = Some(PathBuf::from(
argv.get(i).ok_or("--raw-store-db requires a value")?,
));
}
"--mirror-root" => {
i += 1;
mirror_root = Some(PathBuf::from(
@@ -58,33 +44,18 @@ fn run(argv: Vec<String>) -> Result<(), String> {
}
let cir_path = cir_path.ok_or_else(|| format!("--cir is required\n\n{}", usage()))?;
let repo_bytes_db =
repo_bytes_db.ok_or_else(|| format!("--repo-bytes-db is required\n\n{}", usage()))?;
let mirror_root =
mirror_root.ok_or_else(|| format!("--mirror-root is required\n\n{}", usage()))?;
let backend_count =
static_root.is_some() as u8 + raw_store_db.is_some() as u8 + repo_bytes_db.is_some() as u8;
if backend_count != 1 {
return Err(format!(
"must specify exactly one of --repo-bytes-db, --raw-store-db or --static-root\n\n{}",
usage()
));
}
let bytes = std::fs::read(&cir_path)
.map_err(|e| format!("read CIR failed: {}: {e}", cir_path.display()))?;
let cir = rpki::cir::decode_cir(&bytes).map_err(|e| e.to_string())?;
let result = if let Some(static_root) = static_root {
rpki::cir::materialize_cir(&cir, &static_root, &mirror_root, true)
.map_err(|e| e.to_string())
} else if let Some(repo_bytes_db) = repo_bytes_db {
let result =
rpki::cir::materialize_cir_from_repo_bytes(&cir, &repo_bytes_db, &mirror_root, true)
.map_err(|e| e.to_string())
} else if let Some(raw_store_db) = raw_store_db {
rpki::cir::materialize_cir_from_raw_store(&cir, &raw_store_db, &mirror_root, true)
.map_err(|e| e.to_string())
} else {
unreachable!("validated backend count")
};
.map_err(|e| e.to_string());
match result {
Ok(summary) => {
eprintln!(

22
src/bin/cir_ta_only_fixture.rs
View File

@@ -1,12 +1,12 @@
use std::path::PathBuf;
use rpki::blob_store::ExternalRepoBytesDb;
use rpki::cir::{
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal, encode_cir,
write_bytes_to_static_pool,
};
use sha2::Digest;
const USAGE: &str = "Usage: cir_ta_only_fixture --tal-path <path> --ta-path <path> --tal-uri <url> --validation-time <rfc3339> --cir-out <path> --static-root <path>";
const USAGE: &str = "Usage: cir_ta_only_fixture --tal-path <path> --ta-path <path> --tal-uri <url> --validation-time <rfc3339> --cir-out <path> --repo-bytes-db <path>";
fn parse_args(
argv: &[String],
@@ -26,7 +26,7 @@ fn parse_args(
let mut tal_uri = None;
let mut validation_time = None;
let mut cir_out = None;
let mut static_root = None;
let mut repo_bytes_db = None;
let mut i = 1usize;
while i < argv.len() {
match argv[i].as_str() {
@@ -63,10 +63,10 @@ fn parse_args(
argv.get(i).ok_or("--cir-out requires a value")?,
));
}
"--static-root" => {
"--repo-bytes-db" => {
i += 1;
static_root = Some(PathBuf::from(
argv.get(i).ok_or("--static-root requires a value")?,
repo_bytes_db = Some(PathBuf::from(
argv.get(i).ok_or("--repo-bytes-db requires a value")?,
));
}
"-h" | "--help" => return Err(USAGE.to_string()),
@@ -80,13 +80,13 @@ fn parse_args(
tal_uri.ok_or_else(|| format!("--tal-uri is required\n\n{USAGE}"))?,
validation_time.ok_or_else(|| format!("--validation-time is required\n\n{USAGE}"))?,
cir_out.ok_or_else(|| format!("--cir-out is required\n\n{USAGE}"))?,
static_root.ok_or_else(|| format!("--static-root is required\n\n{USAGE}"))?,
repo_bytes_db.ok_or_else(|| format!("--repo-bytes-db is required\n\n{USAGE}"))?,
))
}
fn main() -> Result<(), String> {
let argv: Vec<String> = std::env::args().collect();
let (tal_path, ta_path, tal_uri, validation_time, cir_out, static_root) = parse_args(&argv)?;
let (tal_path, ta_path, tal_uri, validation_time, cir_out, repo_bytes_db) = parse_args(&argv)?;
let tal_bytes = std::fs::read(&tal_path).map_err(|e| format!("read tal failed: {e}"))?;
let ta_bytes = std::fs::read(&ta_path).map_err(|e| format!("read ta failed: {e}"))?;
@@ -102,8 +102,10 @@ fn main() -> Result<(), String> {
let sha = sha2::Sha256::digest(&ta_bytes);
let hash_hex = hex::encode(sha);
write_bytes_to_static_pool(&static_root, validation_time.date(), &hash_hex, &ta_bytes)
.map_err(|e| format!("write static pool failed: {e}"))?;
ExternalRepoBytesDb::open(&repo_bytes_db)
.map_err(|e| format!("open repo bytes db failed: {e}"))?
.put_blob_bytes_batch(&[(hash_hex, ta_bytes.clone())])
.map_err(|e| format!("write repo bytes db failed: {e}"))?;
let cir = CanonicalInputRepresentation {
version: CIR_VERSION_V1,

50
src/bin/replay_bundle_record.rs
View File

@@ -737,25 +737,47 @@ mod tests {
if skip_heavy_blackbox_test() {
return;
}
let tal_path = PathBuf::from("tests/fixtures/tal/apnic-rfc7730-https.tal");
let ta_path = PathBuf::from("tests/fixtures/ta/apnic-ta.cer");
let replay_archive = PathBuf::from(
"/home/yuyr/dev/rust_playground/routinator/bench/multi_rir_demo/runs/20260316-112341-multi-final3/apnic/base-payload-archive",
);
let replay_locks = PathBuf::from(
"/home/yuyr/dev/rust_playground/routinator/bench/multi_rir_demo/runs/20260316-112341-multi-final3/apnic/base-locks.json",
);
let delta_archive = PathBuf::from(
"/home/yuyr/dev/rust_playground/routinator/bench/multi_rir_demo/runs/20260316-112341-multi-final3/apnic/payload-delta-archive",
);
let delta_locks = PathBuf::from(
"/home/yuyr/dev/rust_playground/routinator/bench/multi_rir_demo/runs/20260316-112341-multi-final3/apnic/locks-delta.json",
);
let required = [
tal_path.as_path(),
ta_path.as_path(),
replay_archive.as_path(),
replay_locks.as_path(),
delta_archive.as_path(),
delta_locks.as_path(),
];
if let Some(missing) = required.iter().find(|path| !path.exists()) {
eprintln!(
"skipping replay_bundle_record smoke test; fixture missing: {}",
missing.display()
);
return;
}
let dir = tempdir().expect("tempdir");
let out_dir = dir.path().join("bundle");
let out = run(Args {
rir: Some("apnic".to_string()),
out_dir: Some(out_dir.clone()),
tal_path: Some(PathBuf::from("tests/fixtures/tal/apnic-rfc7730-https.tal")),
ta_path: Some(PathBuf::from("tests/fixtures/ta/apnic-ta.cer")),
payload_replay_archive: Some(PathBuf::from(
"/home/yuyr/dev/rust_playground/routinator/bench/multi_rir_demo/runs/20260316-112341-multi-final3/apnic/base-payload-archive",
)),
payload_replay_locks: Some(PathBuf::from(
"/home/yuyr/dev/rust_playground/routinator/bench/multi_rir_demo/runs/20260316-112341-multi-final3/apnic/base-locks.json",
)),
payload_delta_archive: Some(PathBuf::from(
"/home/yuyr/dev/rust_playground/routinator/bench/multi_rir_demo/runs/20260316-112341-multi-final3/apnic/payload-delta-archive",
)),
payload_delta_locks: Some(PathBuf::from(
"/home/yuyr/dev/rust_playground/routinator/bench/multi_rir_demo/runs/20260316-112341-multi-final3/apnic/locks-delta.json",
)),
tal_path: Some(tal_path),
ta_path: Some(ta_path),
payload_replay_archive: Some(replay_archive),
payload_replay_locks: Some(replay_locks),
payload_delta_archive: Some(delta_archive),
payload_delta_locks: Some(delta_locks),
validation_time: None,
max_depth: Some(0),
max_instances: Some(1),