NASP/rpki
77
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: NASP:feature/142-custom-tal-installer
Branches Tags
NASP:main
NASP:feature/142-custom-tal-installer
NASP:dev_1.0.0_yuyr_parallel
NASP:dev_1.3_xuxt
NASP:dev_1.2_xuxt
NASP:dev_1.0.0_yuyr
NASP:dev_1.1_xuxt
NASP:dev_1.0_xuxt
..
compare: NASP:main
Branches Tags
NASP:feature/142-custom-tal-installer
NASP:dev_1.0.0_yuyr_parallel
NASP:dev_1.3_xuxt
NASP:dev_1.2_xuxt
NASP:dev_1.0.0_yuyr
NASP:main
NASP:dev_1.1_xuxt
NASP:dev_1.0_xuxt

1 Commits
feature/14 ... main

Author SHA1 Message Date
xuxt
f5b7da921a RC证书解析部分代码 (#2) 
Co-authored-by: xiuting.xu <xiutingxt.xu@gmail.com>
Reviewed-on: #2
Reviewed-by: yuyr <yuyr@zgclab.edu.cn>
Co-authored-by: xuxt <xuxt@zgclab.edu.cn>
Co-committed-by: xuxt <xuxt@zgclab.edu.cn>
 2026-02-03 10:01:32 +08:00
570 changed files with 1391 additions and 196159 deletions
Show Stats Expand all files Collapse all files

17
.dockerignore
View File

@ -1,17 +0,0 @@
target/
.git/
.gitignore
perf.*
**/* copy.excalidraw
ui/rpki-explorer/node_modules/
ui/rpki-explorer/dist/
ui/rpki-explorer/playwright-report/
ui/rpki-explorer/test-results/
ui/rpki-explorer/.vite/
deploy/arm64-compose/.env
target/
*.profraw
*.profdata
*.tar
*.tar.gz
*.zip

7
.gitignore vendored
View File

@ -1,9 +1,2 @@
target/
Cargo.lock
perf.*
specs/* copy.excalidraw
ui/rpki-explorer/node_modules/
ui/rpki-explorer/dist/
ui/rpki-explorer/playwright-report/
ui/rpki-explorer/test-results/
ui/rpki-explorer/.vite/

30
Cargo.toml
View File

@ -3,35 +3,13 @@ name = "rpki"
version = "0.1.0"
edition = "2024"
[features]
default = ["full"]
# Full build used by the main RP implementation (includes RocksDB-backed storage).
full = ["dep:rocksdb"]
profile = ["dep:pprof", "dep:flate2"]
[dependencies]
asn1-rs = "0.7.1"
der-parser = { version = "10.0.0", features = ["serialize"] }
der-parser = "10.0.0"
hex = "0.4.3"
base64 = "0.22.1"
sha2 = "0.10.8"
thiserror = "2.0.18"
time = "0.3.45"
ring = "0.17.14"
x509-parser = { version = "0.18.0", features = ["verify"] }
url = "2.5.8"
serde = { version = "1.0.218", features = ["derive"] }
serde_json = { version = "1.0.140", features = ["raw_value"] }
toml = "0.8.20"
rocksdb = { version = "0.22.0", optional = true, default-features = false, features = ["lz4"] }
serde_cbor = "0.11.2"
memmap2 = "0.9.10"
roxmltree = "0.20.0"
quick-xml = "0.37.2"
uuid = { version = "1.7.0", features = ["v4"] }
reqwest = { version = "0.12.12", default-features = false, features = ["blocking", "rustls-tls", "gzip", "brotli", "deflate"] }
pprof = { version = "0.14.1", optional = true, features = ["flamegraph", "prost-codec"] }
flate2 = { version = "1.0.35", optional = true }
tempfile = "3.16.0"
[dev-dependencies]
asn1-rs = "0.7.1"
asn1-rs-derive = "0.6.0"
asn1 = "0.23.0"

52
README.md
View File

@ -9,55 +9,3 @@ cargo test
cargo test -- --nocapture
```
# 覆盖率（cargo-llvm-cov）
安装工具：
```
rustup component add llvm-tools-preview
cargo install cargo-llvm-cov --locked
```
统计行覆盖率并要求 >=90%：
```
./scripts/coverage.sh
# 或
cargo llvm-cov --fail-under-lines 90
```
默认会复用现有插桩产物，不会先 clean。需要强制全量重编译时：
```
COVERAGE_FORCE_CLEAN=1 ./scripts/coverage.sh
```
说明：
- 默认行为适合本地重复确认覆盖率，避免每次都重编译整套插桩目标；
- 默认还会设置 `RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS=1`，跳过会拉起 shell replay pipeline 的重型集成测试，避免 coverage 期间额外触发 `target/release` 构建；
- 默认还会设置 `RPKI_SKIP_HEAVY_BLACKBOX_TESTS=1`，跳过更慢的 blackbox CLI / CIR record 脚本测试，进一步降低日常 coverage 成本；
- 默认还会设置 `RPKI_SKIP_HEAVY_CRYPTO_TESTS=1`，跳过需要大量 OpenSSL 生成证书/CRL 的重型密码学测试，进一步压缩日常 coverage 时长；
- 如需把这批脚本回放测试也纳入 coverage，可显式关闭该开关：
```
RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS=0 ./scripts/coverage.sh
```
如需连同第二批 blackbox 测试一起跑：
```
RPKI_SKIP_HEAVY_BLACKBOX_TESTS=0 ./scripts/coverage.sh
```
如需连同重型 OpenSSL 证书路径测试一起跑：
```
RPKI_SKIP_HEAVY_CRYPTO_TESTS=0 ./scripts/coverage.sh
```
- replay 脚本现在也支持通过环境变量注入现成二进制，避免找不到二进制时自动 `cargo build --release`：
- `RPKI_BIN`
- `CIR_MATERIALIZE_BIN`
- `CIR_EXTRACT_INPUTS_BIN`
- `CCR_TO_COMPARE_VIEWS_BIN`
- `COVERAGE_FORCE_CLEAN=1` 适合需要完全从零重建插桩目标时使用。

8
benchmark/ours_manifest_bench/Cargo.toml
View File

@ -1,8 +0,0 @@
[package]
name = "ours-manifest-bench"
version = "0.1.0"
edition = "2024"
[dependencies]
rpki = { path = "../..", default-features = false }

145
benchmark/ours_manifest_bench/src/main.rs
View File

@ -1,145 +0,0 @@
use rpki::data_model::manifest::ManifestObject;
use std::hint::black_box;
use std::path::PathBuf;
use std::time::Instant;
#[derive(Debug, Clone)]
struct Config {
sample: Option<String>,
manifest_path: Option<PathBuf>,
iterations: u64,
warmup_iterations: u64,
repeats: u32,
}
fn usage_and_exit() -> ! {
eprintln!(
"Usage:\n ours-manifest-bench (--sample <name> | --manifest <path>) [--iterations N] [--warmup-iterations N] [--repeats N]\n\nExamples:\n cargo run --release -- --sample small-01 --iterations 20000 --warmup-iterations 2000 --repeats 3\n cargo run --release -- --manifest ../../tests/benchmark/selected_der/small-01.mft"
);
std::process::exit(2);
}
fn parse_args() -> Config {
let mut sample: Option<String> = None;
let mut manifest_path: Option<PathBuf> = None;
let mut iterations: u64 = 20_000;
let mut warmup_iterations: u64 = 2_000;
let mut repeats: u32 = 3;
let mut args = std::env::args().skip(1);
while let Some(arg) = args.next() {
match arg.as_str() {
"--sample" => sample = Some(args.next().unwrap_or_else(|| usage_and_exit())),
"--manifest" => {
manifest_path = Some(PathBuf::from(args.next().unwrap_or_else(|| usage_and_exit())))
}
"--iterations" => {
iterations = args
.next()
.unwrap_or_else(|| usage_and_exit())
.parse()
.unwrap_or_else(|_| usage_and_exit())
}
"--warmup-iterations" => {
warmup_iterations = args
.next()
.unwrap_or_else(|| usage_and_exit())
.parse()
.unwrap_or_else(|_| usage_and_exit())
}
"--repeats" => {
repeats = args
.next()
.unwrap_or_else(|| usage_and_exit())
.parse()
.unwrap_or_else(|_| usage_and_exit())
}
"-h" | "--help" => usage_and_exit(),
_ => usage_and_exit(),
}
}
if sample.is_none() && manifest_path.is_none() {
usage_and_exit();
}
if sample.is_some() && manifest_path.is_some() {
usage_and_exit();
}
Config {
sample,
manifest_path,
iterations,
warmup_iterations,
repeats,
}
}
fn derive_manifest_path(sample: &str) -> PathBuf {
// Assumes current working directory is `rpki/benchmark/ours_manifest_bench`.
PathBuf::from(format!("../../tests/benchmark/selected_der/{sample}.mft"))
}
fn main() {
let cfg = parse_args();
let manifest_path = cfg
.manifest_path
.clone()
.unwrap_or_else(|| derive_manifest_path(cfg.sample.as_deref().unwrap()));
let bytes = std::fs::read(&manifest_path).unwrap_or_else(|e| {
eprintln!("read manifest fixture failed: {e}; path={}", manifest_path.display());
std::process::exit(1);
});
let decoded_once = ManifestObject::decode_der(&bytes).unwrap_or_else(|e| {
eprintln!("decode failed: {e}; path={}", manifest_path.display());
std::process::exit(1);
});
let file_count = decoded_once.manifest.file_count();
let mut round_ns_per_op: Vec<f64> = Vec::with_capacity(cfg.repeats as usize);
let mut round_ops_per_s: Vec<f64> = Vec::with_capacity(cfg.repeats as usize);
for _round in 0..cfg.repeats {
for _ in 0..cfg.warmup_iterations {
let obj = ManifestObject::decode_der(black_box(&bytes)).expect("warmup decode");
black_box(obj);
}
let start = Instant::now();
for _ in 0..cfg.iterations {
let obj = ManifestObject::decode_der(black_box(&bytes)).expect("timed decode");
black_box(obj);
}
let elapsed = start.elapsed();
let ns_per_op = (elapsed.as_secs_f64() * 1e9) / (cfg.iterations as f64);
let ops_per_s = (cfg.iterations as f64) / elapsed.as_secs_f64();
round_ns_per_op.push(ns_per_op);
round_ops_per_s.push(ops_per_s);
}
let avg_ns_per_op = round_ns_per_op.iter().sum::<f64>() / (round_ns_per_op.len() as f64);
let avg_ops_per_s = round_ops_per_s.iter().sum::<f64>() / (round_ops_per_s.len() as f64);
let sample_name = cfg.sample.clone().unwrap_or_else(|| {
manifest_path
.file_name()
.map(|s| s.to_string_lossy().to_string())
.unwrap_or_else(|| manifest_path.display().to_string())
});
let sample_name = sample_name
.strip_suffix(".mft")
.unwrap_or(&sample_name)
.to_string();
println!("fixture: {}", manifest_path.display());
println!();
println!("| sample | avg ns/op | ops/s | file count |");
println!("|---|---:|---:|---:|");
println!(
"| {} | {:.2} | {:.2} | {} |",
sample_name, avg_ns_per_op, avg_ops_per_s, file_count
);
}

8
benchmark/routinator_object_bench/Cargo.toml
View File

@ -1,8 +0,0 @@
[package]
name = "routinator-object-bench"
version = "0.1.0"
edition = "2024"
publish = false
[dependencies]
rpki = { version = "=0.19.1", features = ["repository"] }

552
benchmark/routinator_object_bench/src/main.rs
View File

@ -1,552 +0,0 @@
use rpki::repository::cert::Cert;
use rpki::repository::crl::Crl;
use rpki::repository::manifest::Manifest;
use rpki::repository::roa::Roa;
use rpki::repository::aspa::Aspa;
use rpki::repository::resources::{AsResources, IpResources};
use std::hint::black_box;
use std::path::{Path, PathBuf};
use std::time::Instant;
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
enum ObjType {
Cer,
Crl,
Manifest,
Roa,
Aspa,
}
impl ObjType {
fn parse(s: &str) -> Result<Self, String> {
match s {
"cer" => Ok(Self::Cer),
"crl" => Ok(Self::Crl),
"manifest" => Ok(Self::Manifest),
"roa" => Ok(Self::Roa),
"aspa" => Ok(Self::Aspa),
_ => Err("type must be one of: cer, crl, manifest, roa, aspa".into()),
}
}
fn as_str(self) -> &'static str {
match self {
ObjType::Cer => "cer",
ObjType::Crl => "crl",
ObjType::Manifest => "manifest",
ObjType::Roa => "roa",
ObjType::Aspa => "aspa",
}
}
fn ext(self) -> &'static str {
match self {
ObjType::Cer => "cer",
ObjType::Crl => "crl",
ObjType::Manifest => "mft",
ObjType::Roa => "roa",
ObjType::Aspa => "asa",
}
}
}
#[derive(Clone, Debug)]
struct Sample {
obj_type: ObjType,
name: String,
path: PathBuf,
}
#[derive(Clone, Debug)]
struct Config {
dir: PathBuf,
type_filter: Option<ObjType>,
sample_filter: Option<String>,
fixed_iters: Option<u64>,
warmup_iters: u64,
rounds: u64,
min_round_ms: u64,
max_adaptive_iters: u64,
strict: bool,
cert_inspect: bool,
out_csv: Option<PathBuf>,
out_md: Option<PathBuf>,
}
fn usage_and_exit(err: Option<&str>) -> ! {
if let Some(err) = err {
eprintln!("error: {err}");
eprintln!();
}
eprintln!(
"Usage:\n\
cargo run --release --manifest-path rpki/benchmark/routinator_object_bench/Cargo.toml -- [OPTIONS]\n\
\n\
Options:\n\
--dir <PATH> Fixtures root dir (default: ../../tests/benchmark/selected_der_v2)\n\
--type <cer|crl|manifest|roa|aspa> Filter by type\n\
--sample <NAME> Filter by sample name (e.g. p50)\n\
--iters <N> Fixed iterations per round (optional; otherwise adaptive)\n\
--warmup-iters <N> Warmup iterations (default: 50)\n\
--rounds <N> Rounds (default: 5)\n\
--min-round-ms <MS> Adaptive: minimum round time (default: 200)\n\
--max-iters <N> Adaptive: maximum iters (default: 1_000_000)\n\
--strict <true|false> Strict DER where applicable (default: true)\n\
--cert-inspect Also run Cert::inspect_ca/inspect_ee where applicable (default: false)\n\
--out-csv <PATH> Write CSV output\n\
--out-md <PATH> Write Markdown output\n\
"
);
std::process::exit(2);
}
fn parse_bool(s: &str, name: &str) -> bool {
match s {
"1" | "true" | "TRUE" | "yes" | "YES" => true,
"0" | "false" | "FALSE" | "no" | "NO" => false,
_ => usage_and_exit(Some(&format!("{name} must be true/false"))),
}
}
fn parse_u64(s: &str, name: &str) -> u64 {
s.parse::<u64>()
.unwrap_or_else(|_| usage_and_exit(Some(&format!("{name} must be an integer"))))
}
fn default_samples_dir() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/benchmark/selected_der_v2")
}
fn parse_args() -> Config {
let mut dir: PathBuf = default_samples_dir();
let mut type_filter: Option<ObjType> = None;
let mut sample_filter: Option<String> = None;
let mut fixed_iters: Option<u64> = None;
let mut warmup_iters: u64 = 50;
let mut rounds: u64 = 5;
let mut min_round_ms: u64 = 200;
let mut max_adaptive_iters: u64 = 1_000_000;
let mut strict: bool = true;
let mut cert_inspect: bool = false;
let mut out_csv: Option<PathBuf> = None;
let mut out_md: Option<PathBuf> = None;
let mut args = std::env::args().skip(1);
while let Some(arg) = args.next() {
match arg.as_str() {
"--dir" => dir = PathBuf::from(args.next().unwrap_or_else(|| usage_and_exit(None))),
"--type" => {
type_filter = Some(ObjType::parse(
&args.next().unwrap_or_else(|| usage_and_exit(None)),
)
.unwrap_or_else(|e| usage_and_exit(Some(&e))))
}
"--sample" => {
sample_filter = Some(args.next().unwrap_or_else(|| usage_and_exit(None)))
}
"--iters" => {
fixed_iters = Some(parse_u64(
&args.next().unwrap_or_else(|| usage_and_exit(None)),
"--iters",
))
}
"--warmup-iters" => {
warmup_iters = parse_u64(
&args.next().unwrap_or_else(|| usage_and_exit(None)),
"--warmup-iters",
)
}
"--rounds" => {
rounds = parse_u64(&args.next().unwrap_or_else(|| usage_and_exit(None)), "--rounds")
}
"--min-round-ms" => {
min_round_ms = parse_u64(
&args.next().unwrap_or_else(|| usage_and_exit(None)),
"--min-round-ms",
)
}
"--max-iters" => {
max_adaptive_iters = parse_u64(
&args.next().unwrap_or_else(|| usage_and_exit(None)),
"--max-iters",
)
}
"--strict" => {
strict = parse_bool(
&args.next().unwrap_or_else(|| usage_and_exit(None)),
"--strict",
)
}
"--cert-inspect" => cert_inspect = true,
"--out-csv" => out_csv = Some(PathBuf::from(args.next().unwrap_or_else(|| usage_and_exit(None)))),
"--out-md" => out_md = Some(PathBuf::from(args.next().unwrap_or_else(|| usage_and_exit(None)))),
"-h" | "--help" => usage_and_exit(None),
_ => usage_and_exit(Some(&format!("unknown argument: {arg}"))),
}
}
if warmup_iters == 0 {
usage_and_exit(Some("--warmup-iters must be > 0"));
}
if rounds == 0 {
usage_and_exit(Some("--rounds must be > 0"));
}
if min_round_ms == 0 {
usage_and_exit(Some("--min-round-ms must be > 0"));
}
if max_adaptive_iters == 0 {
usage_and_exit(Some("--max-iters must be > 0"));
}
if let Some(n) = fixed_iters {
if n == 0 {
usage_and_exit(Some("--iters must be > 0"));
}
}
Config {
dir,
type_filter,
sample_filter,
fixed_iters,
warmup_iters,
rounds,
min_round_ms,
max_adaptive_iters,
strict,
cert_inspect,
out_csv,
out_md,
}
}
fn read_samples(root: &Path) -> Vec<Sample> {
let mut out = Vec::new();
for obj_type in [
ObjType::Cer,
ObjType::Crl,
ObjType::Manifest,
ObjType::Roa,
ObjType::Aspa,
] {
let dir = root.join(obj_type.as_str());
let rd = match std::fs::read_dir(&dir) {
Ok(rd) => rd,
Err(_) => continue,
};
for ent in rd.flatten() {
let path = ent.path();
if path.extension().and_then(|s| s.to_str()) != Some(obj_type.ext()) {
continue;
}
let name = path
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("unknown")
.to_string();
out.push(Sample { obj_type, name, path });
}
}
out.sort_by(|a, b| a.obj_type.cmp(&b.obj_type).then_with(|| a.name.cmp(&b.name)));
out
}
fn choose_iters_adaptive<F: FnMut()>(mut op: F, min_round_ms: u64, max_iters: u64) -> u64 {
let min_secs = (min_round_ms as f64) / 1e3;
let mut iters: u64 = 1;
loop {
let start = Instant::now();
for _ in 0..iters {
op();
}
let elapsed = start.elapsed().as_secs_f64();
if elapsed >= min_secs {
return iters;
}
if iters >= max_iters {
return iters;
}
iters = (iters.saturating_mul(2)).min(max_iters);
}
}
fn count_ip(res: &IpResources) -> u64 {
if res.is_inherited() {
return 1;
}
let Ok(blocks) = res.to_blocks() else {
return 0;
};
blocks.iter().count() as u64
}
fn count_as(res: &AsResources) -> u64 {
if res.is_inherited() {
return 1;
}
let Ok(blocks) = res.to_blocks() else {
return 0;
};
blocks.iter().count() as u64
}
fn complexity(obj_type: ObjType, bytes: &[u8], strict: bool, cert_inspect: bool) -> u64 {
match obj_type {
ObjType::Cer => {
let cert = Cert::decode(bytes).expect("decode cert");
if cert_inspect {
if cert.is_ca() {
cert.inspect_ca(strict).expect("inspect ca");
} else {
cert.inspect_ee(strict).expect("inspect ee");
}
}
count_ip(cert.v4_resources())
.saturating_add(count_ip(cert.v6_resources()))
.saturating_add(count_as(cert.as_resources()))
}
ObjType::Crl => {
let crl = Crl::decode(bytes).expect("decode crl");
crl.revoked_certs().iter().count() as u64
}
ObjType::Manifest => {
let mft = Manifest::decode(bytes, strict).expect("decode manifest");
if cert_inspect {
mft.cert().inspect_ee(strict).expect("inspect ee");
}
mft.content().len() as u64
}
ObjType::Roa => {
let roa = Roa::decode(bytes, strict).expect("decode roa");
if cert_inspect {
roa.cert().inspect_ee(strict).expect("inspect ee");
}
roa.content().iter().count() as u64
}
ObjType::Aspa => {
let asa = Aspa::decode(bytes, strict).expect("decode aspa");
if cert_inspect {
asa.cert().inspect_ee(strict).expect("inspect ee");
}
asa.content().provider_as_set().len() as u64
}
}
}
fn decode_profile(obj_type: ObjType, bytes: &[u8], strict: bool, cert_inspect: bool) {
match obj_type {
ObjType::Cer => {
let cert = Cert::decode(black_box(bytes)).expect("decode cert");
if cert_inspect {
if cert.is_ca() {
cert.inspect_ca(strict).expect("inspect ca");
} else {
cert.inspect_ee(strict).expect("inspect ee");
}
}
black_box(cert);
}
ObjType::Crl => {
let crl = Crl::decode(black_box(bytes)).expect("decode crl");
black_box(crl);
}
ObjType::Manifest => {
let mft = Manifest::decode(black_box(bytes), strict).expect("decode manifest");
if cert_inspect {
mft.cert().inspect_ee(strict).expect("inspect ee");
}
black_box(mft);
}
ObjType::Roa => {
let roa = Roa::decode(black_box(bytes), strict).expect("decode roa");
if cert_inspect {
roa.cert().inspect_ee(strict).expect("inspect ee");
}
black_box(roa);
}
ObjType::Aspa => {
let asa = Aspa::decode(black_box(bytes), strict).expect("decode aspa");
if cert_inspect {
asa.cert().inspect_ee(strict).expect("inspect ee");
}
black_box(asa);
}
}
}
#[derive(Clone, Debug)]
struct ResultRow {
obj_type: String,
sample: String,
size_bytes: usize,
complexity: u64,
avg_ns_per_op: f64,
ops_per_sec: f64,
}
fn render_markdown(title: &str, rows: &[ResultRow]) -> String {
let mut out = String::new();
out.push_str(&format!("# {title}\n\n"));
out.push_str("| type | sample | size_bytes | complexity | avg ns/op | ops/s |\n");
out.push_str("|---|---|---:|---:|---:|---:|\n");
for r in rows {
out.push_str(&format!(
"| {} | {} | {} | {} | {:.2} | {:.2} |\n",
r.obj_type, r.sample, r.size_bytes, r.complexity, r.avg_ns_per_op, r.ops_per_sec
));
}
out
}
fn render_csv(rows: &[ResultRow]) -> String {
let mut out = String::new();
out.push_str("type,sample,size_bytes,complexity,avg_ns_per_op,ops_per_sec\n");
for r in rows {
let sample = r.sample.replace('"', "\"\"");
out.push_str(&format!(
"{},{},{},{},{:.6},{:.6}\n",
r.obj_type,
format!("\"{}\"", sample),
r.size_bytes,
r.complexity,
r.avg_ns_per_op,
r.ops_per_sec
));
}
out
}
fn create_parent_dirs(path: &Path) {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).unwrap_or_else(|e| {
panic!("create_dir_all {}: {e}", parent.display());
});
}
}
fn write_text_file(path: &Path, content: &str) {
create_parent_dirs(path);
std::fs::write(path, content).unwrap_or_else(|e| panic!("write {}: {e}", path.display()));
}
fn main() {
let cfg = parse_args();
let mut samples = read_samples(&cfg.dir);
if samples.is_empty() {
usage_and_exit(Some(&format!(
"no samples found under: {}",
cfg.dir.display()
)));
}
if let Some(t) = cfg.type_filter {
samples.retain(|s| s.obj_type == t);
if samples.is_empty() {
usage_and_exit(Some(&format!("no sample matched --type {}", t.as_str())));
}
}
if let Some(filter) = cfg.sample_filter.as_deref() {
samples.retain(|s| s.name == filter);
if samples.is_empty() {
usage_and_exit(Some(&format!("no sample matched --sample {filter}")));
}
}
println!("# Routinator baseline (rpki crate) decode benchmark (selected_der_v2)");
println!();
println!("- dir: {}", cfg.dir.display());
println!("- strict: {}", cfg.strict);
println!("- cert_inspect: {}", cfg.cert_inspect);
if let Some(t) = cfg.type_filter {
println!("- type: {}", t.as_str());
}
if let Some(s) = cfg.sample_filter.as_deref() {
println!("- sample: {}", s);
}
if let Some(n) = cfg.fixed_iters {
println!("- iters: {} (fixed)", n);
} else {
println!(
"- warmup: {} iters, rounds: {}, min_round: {}ms (adaptive iters, max {})",
cfg.warmup_iters, cfg.rounds, cfg.min_round_ms, cfg.max_adaptive_iters
);
}
if let Some(p) = cfg.out_csv.as_ref() {
println!("- out_csv: {}", p.display());
}
if let Some(p) = cfg.out_md.as_ref() {
println!("- out_md: {}", p.display());
}
println!();
println!("| type | sample | size_bytes | complexity | avg ns/op | ops/s |");
println!("|---|---|---:|---:|---:|---:|");
let mut rows: Vec<ResultRow> = Vec::with_capacity(samples.len());
for sample in &samples {
let bytes = std::fs::read(&sample.path)
.unwrap_or_else(|e| panic!("read {}: {e}", sample.path.display()));
let size_bytes = bytes.len();
let complexity = complexity(sample.obj_type, bytes.as_slice(), cfg.strict, cfg.cert_inspect);
for _ in 0..cfg.warmup_iters {
decode_profile(sample.obj_type, bytes.as_slice(), cfg.strict, cfg.cert_inspect);
}
let mut per_round_ns_per_op = Vec::with_capacity(cfg.rounds as usize);
for _round in 0..cfg.rounds {
let iters = if let Some(n) = cfg.fixed_iters {
n
} else {
choose_iters_adaptive(
|| decode_profile(sample.obj_type, bytes.as_slice(), cfg.strict, cfg.cert_inspect),
cfg.min_round_ms,
cfg.max_adaptive_iters,
)
};
let start = Instant::now();
for _ in 0..iters {
decode_profile(sample.obj_type, bytes.as_slice(), cfg.strict, cfg.cert_inspect);
}
let elapsed = start.elapsed();
let total_ns = elapsed.as_secs_f64() * 1e9;
per_round_ns_per_op.push(total_ns / (iters as f64));
}
let avg_ns = per_round_ns_per_op.iter().sum::<f64>() / (per_round_ns_per_op.len() as f64);
let ops_per_sec = 1e9_f64 / avg_ns;
println!(
"| {} | {} | {} | {} | {:.2} | {:.2} |",
sample.obj_type.as_str(),
sample.name,
size_bytes,
complexity,
avg_ns,
ops_per_sec
);
rows.push(ResultRow {
obj_type: sample.obj_type.as_str().to_string(),
sample: sample.name.clone(),
size_bytes,
complexity,
avg_ns_per_op: avg_ns,
ops_per_sec,
});
}
if let Some(path) = cfg.out_md.as_ref() {
let md = render_markdown(
"Routinator baseline (rpki crate) decode+inspect (selected_der_v2)",
&rows,
);
write_text_file(path, &md);
eprintln!("Wrote {}", path.display());
}
if let Some(path) = cfg.out_csv.as_ref() {
let csv = render_csv(&rows);
write_text_file(path, &csv);
eprintln!("Wrote {}", path.display());
}
}

123
deploy/docker-installer/.env.example
View File

@ -1,123 +0,0 @@
# ours RP Docker installer configuration
# `build_docker_installer_package.sh` rewrites package-specific placeholders
# when producing amd64/arm64 release packages.
# 中文说明见 docs/README.zh-CN.md。English guide: docs/README.en.md
# Package metadata and architecture guardrails.
PACKAGE_ARCH=__PACKAGE_ARCH__
PACKAGE_PLATFORM=__PACKAGE_PLATFORM__
ALLOW_CROSS_ARCH=0
# Compose project name.
COMPOSE_PROJECT_NAME=ours-rp-__PACKAGE_ARCH__-installer
# Runtime image rebuilt from the current source commit while packaging.
RPKI_IMAGE=__RUNTIME_IMAGE__
RPKI_PLATFORM=__PACKAGE_PLATFORM__
# Metrics image rebuilt from the current source commit while packaging.
METRICS_IMAGE=__METRICS_IMAGE__
METRICS_PLATFORM=__PACKAGE_PLATFORM__
# Restart policy for the soak container. Production default keeps the daemon alive.
# For finite acceptance tests such as MAX_RUNS=3, set SOAK_RESTART_POLICY=no to avoid an extra restarted run.
SOAK_RESTART_POLICY=unless-stopped
# Host-side persistent data directory. All state/runs/logs/monitoring data are bind-mounted here.
HOST_DATA_DIR=__HOST_DATA_DIR__
# RIR list. Options: afrinic,apnic,arin,lacnic,ripe
RIRS=afrinic,apnic,arin,lacnic,ripe
# Negative MAX_RUNS means keep running forever. Default production interval is 10 minutes.
MAX_RUNS=-1
INTERVAL_SECS=600
RETAIN_RUNS=100
# TAL/TA input mode:
# file-with-ta: use packaged fixture TAL + TA only.
# file-live-ta: use packaged fixture TAL; snapshot waits for live TA refresh, delta refreshes TA in background.
# custom-file-with-ta: use one custom TAL + TA from the read-only custom fixture mount.
# url: pass TAL URL to child process.
TAL_INPUT_MODE=file-live-ta
LIVE_TA_REFRESH_BEFORE_SNAPSHOT=1
LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS=15
LIVE_TA_REFRESH_MAX_TIME_SECS=120
# Custom TAL patch mode. Keep TAL_INPUT_MODE=file-live-ta and the five-RIR RIRS
# list above for the original package behavior. For a private fixture, set:
# RIRS=custom
# TAL_INPUT_MODE=custom-file-with-ta
# The paths below are container paths; CUSTOM_FIXTURE_HOST_DIR is a host path
# resolved relative to the compose directory when left at its default.
CUSTOM_FIXTURE_HOST_DIR=../custom-fixtures
CUSTOM_TAL_PATH=/opt/ours-rp/custom-fixtures/tal/custom.tal
CUSTOM_TA_PATH=/opt/ours-rp/custom-fixtures/ta/custom-ta.cer
CUSTOM_TAL_URI=https://host.docker.internal:18443/tal/custom.tal
# Space-separated container paths; paths must not contain spaces. Set this in
# custom mode when the RRDP server uses a private/self-signed CA.
HTTP_ROOT_CERT_PATHS=
# Sync and runtime behavior.
RSYNC_SCOPE=module-root
DISABLE_COMPETING_RPS=0
RUN_ROOT=/var/lib/ours-rp
DB_DIR=/var/lib/ours-rp/state/db
RSYNC_MIRROR_ROOT=/var/lib/ours-rp/state/rsync-mirror
# 每轮完成并复制出正式 run 产物后清理 tmp/daemon-run_*，避免长跑占满磁盘。
CLEAN_TMP_AFTER_RUN=1
OUTPUT_COMPACT_REPORT=1
ALLOW_RSYNC_MIRROR_REUSE=1
FAILURE_SNAPSHOT_RESET=1
# Periodic snapshot reset of active state DB.
# 0: keep existing behavior.
# 1: after one successful snapshot, allow at most N successful delta runs;
# the next run is forced to snapshot and active state/db is rebuilt from empty.
# Lifecycle run state is persisted independently at:
# ${HOST_DATA_DIR}/state/run-lifecycle-state.json
# It is not affected by run retention or state/db reset.
PERIODIC_SNAPSHOT_RESET=0
PERIODIC_SNAPSHOT_MAX_DELTAS=100
DB_STATS_EXACT_EVERY=0
# Resource certificate validation mode passed to the rpki child process.
# Options: validation-update-03 | rfc6487
RESOURCE_VALIDATION_MODE=validation-update-03
# Validation and performance options aligned with current optimized soak defaults.
ENABLE_CHILD_CERTIFICATE_VALIDATION_CACHE=1
RPKI_ANALYZE=1
RPKI_EXTRA_ARGS="--enable-transport-request-prefetch --enable-publication-point-validation-cache --enable-roa-validation-cache --parallel-max-repo-sync-workers-global 4 --parallel-phase2-object-workers 4 --memory-trim-after-validation"
# Progress logs.
RPKI_PROGRESS_LOG=1
RPKI_PROGRESS_SLOW_SECS=20
RPKI_PROGRESS_STAGE_FRESH_SLOW_MS=2000
RPKI_PROGRESS_PP_CONTROL_SLOW_MS=200
RPKI_PROGRESS_PP_CACHE_SLOW_MS=100
RPKI_PROGRESS_CONTROL_LOOP_SLOW_MS=2000
# Metrics sidecar.
METRICS_INSTANCE=__PACKAGE_ARCH__-installer
METRICS_PORT=9556
METRICS_POLL_SECS=10
# Optional external RTR report directory produced by a separately deployed RTR service.
# Default points to an installer-managed empty fallback directory. To enable real RTR
# metrics, set this to the host-side report directory, for example:
# RTR_REPORT_DIR=/root/rpki/report
RTR_REPORT_DIR=__HOST_DATA_DIR__/empty-rtr-report
RTR_REPORT_CONTAINER_DIR=/var/lib/ours-rp/rtr-report
# Prometheus / Grafana.
# Monitor images are packaged as docker-save archives and loaded by install.sh.
MONITOR_PLATFORM=__PACKAGE_PLATFORM__
PROMETHEUS_IMAGE=prom/prometheus:v2.55.1
GRAFANA_IMAGE=grafana/grafana:11.3.1
PROMETHEUS_PORT=9090
PROMETHEUS_RETENTION=7d
GRAFANA_PORT=3000
GRAFANA_ADMIN_USER=admin
GRAFANA_ADMIN_PASSWORD=admin
# First snapshot waiting timeout used by start.sh.
FIRST_RUN_WAIT_TIMEOUT_SECS=7200

100
deploy/docker-installer/compose/docker-compose.yml
View File

@ -1,100 +0,0 @@
services:
ours-rp-soak:
image: ${RPKI_IMAGE:?RPKI_IMAGE is required}
platform: ${RPKI_PLATFORM:?RPKI_PLATFORM is required}
container_name: ${COMPOSE_PROJECT_NAME:-ours-rp-package-installer}-soak
env_file:
- ../.env
environment:
PACKAGE_ROOT: /opt/ours-rp
ENV_FILE: /opt/ours-rp/.env
RUN_ROOT: /var/lib/ours-rp
BIN_DIR: /opt/ours-rp/bin
FIXTURE_DIR: /opt/ours-rp/fixtures
CUSTOM_FIXTURE_DIR: /opt/ours-rp/custom-fixtures
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- ../.env:/opt/ours-rp/.env:ro
# Overlay the patched runner while keeping the original runtime image
# and its exact rpki binaries unchanged.
- ../scripts/soak/run_soak.sh:/opt/ours-rp/run_soak.sh:ro
- ${CUSTOM_FIXTURE_HOST_DIR:-../custom-fixtures}:/opt/ours-rp/custom-fixtures:ro
- ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/state:/var/lib/ours-rp/state
- ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/runs:/var/lib/ours-rp/runs
- ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/logs:/var/lib/ours-rp/logs
- ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/tmp:/var/lib/ours-rp/tmp
restart: ${SOAK_RESTART_POLICY:-unless-stopped}
profiles:
- core
artifact-metrics:
image: ${METRICS_IMAGE:?METRICS_IMAGE is required}
platform: ${METRICS_PLATFORM:?METRICS_PLATFORM is required}
container_name: ${COMPOSE_PROJECT_NAME:-ours-rp-package-installer}-artifact-metrics
env_file:
- ../.env
environment:
RPKI_METRICS_RTR_REPORT_DIR: ${RTR_REPORT_CONTAINER_DIR:-/var/lib/ours-rp/rtr-report}
command:
- /opt/ours-rp/bin/rpki_artifact_metrics
- --run-root
- /var/lib/ours-rp
- --listen
- 0.0.0.0:9556
- --poll-secs
- ${METRICS_POLL_SECS:-10}
- --instance
- ${METRICS_INSTANCE:-package-installer}
ports:
- "${METRICS_PORT:-9556}:9556"
volumes:
- ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/state:/var/lib/ours-rp/state:ro
- ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/runs:/var/lib/ours-rp/runs:ro
- ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/logs:/var/lib/ours-rp/logs:ro
- ${RTR_REPORT_DIR:-/var/lib/ours-rp-package-installer/empty-rtr-report}:${RTR_REPORT_CONTAINER_DIR:-/var/lib/ours-rp/rtr-report}:ro
restart: unless-stopped
profiles:
- sidecar
prometheus:
image: ${PROMETHEUS_IMAGE:-prom/prometheus:v2.55.1}
platform: ${MONITOR_PLATFORM:?MONITOR_PLATFORM is required}
container_name: ${COMPOSE_PROJECT_NAME:-ours-rp-package-installer}-prometheus
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-7d}
- --web.enable-lifecycle
depends_on:
- artifact-metrics
user: "0:0"
ports:
- "${PROMETHEUS_PORT:-9090}:9090"
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/prometheus:/prometheus
restart: unless-stopped
profiles:
- monitor
grafana:
image: ${GRAFANA_IMAGE:-grafana/grafana:11.3.1}
platform: ${MONITOR_PLATFORM:?MONITOR_PLATFORM is required}
container_name: ${COMPOSE_PROJECT_NAME:-ours-rp-package-installer}-grafana
depends_on:
- prometheus
user: "0:0"
ports:
- "${GRAFANA_PORT:-3000}:3000"
environment:
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
GF_USERS_ALLOW_SIGN_UP: "false"
volumes:
- ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/grafana:/var/lib/grafana
- ./grafana/provisioning:/etc/grafana/provisioning:ro
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
restart: unless-stopped
profiles:
- monitor

761
deploy/docker-installer/compose/grafana/dashboards/ours-rp-repo-sync.json
View File

@ -1,761 +0,0 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_publication_points",
"legendFormat": "publication points",
"refId": "A"
}
],
"title": "Publication Points",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 0
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_ok\"}",
"legendFormat": "rrdp ok",
"refId": "A"
}
],
"title": "RRDP OK Points",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 0
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "sum(ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}) or vector(0)",
"legendFormat": "fallback",
"refId": "A"
}
],
"title": "Rsync Fallback Points",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 0
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
"legendFormat": "failed no cache",
"refId": "A"
}
],
"title": "Failed No Cache Points",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 4
},
"id": 5,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"repo_sync_total\"}",
"legendFormat": "repo sync total",
"refId": "A"
},
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"rrdp_download_total\"}",
"legendFormat": "rrdp download",
"refId": "B"
},
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"rsync_download_total\"}",
"legendFormat": "rsync download",
"refId": "C"
}
],
"title": "Repo Sync Download Durations",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 12,
"w": 12,
"h": 8
},
"id": 6,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_repo_sync_phase_count",
"legendFormat": "{{phase}}",
"refId": "A"
}
],
"title": "Repo Sync Phase Counts",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 12,
"w": 12,
"h": 8
},
"id": 7,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}",
"legendFormat": "rrdp failed, rsync ok",
"refId": "A"
},
{
"expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_failed\"}",
"legendFormat": "rrdp failed, rsync failed",
"refId": "B"
},
{
"expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
"legendFormat": "failed no cache",
"refId": "C"
},
{
"expr": "ours_rp_tree_instances{state=\"failed\"}",
"legendFormat": "tree failed",
"refId": "D"
}
],
"title": "Repo Failure / Fallback Counts",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 20,
"w": 12,
"h": 8
},
"id": 8,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_ok\"}",
"legendFormat": "rsync fallback duration",
"refId": "A"
},
{
"expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_failed\"}",
"legendFormat": "failed duration",
"refId": "B"
},
{
"expr": "ours_rp_repo_terminal_state_duration_seconds_total{terminal_state=\"failed_no_cache\"}",
"legendFormat": "failed no cache duration",
"refId": "C"
}
],
"title": "Repo Failure / Fallback Durations",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 29,
"w": 12,
"h": 9
},
"id": 9,
"options": {
"showHeader": true,
"cellHeight": "sm",
"footer": {
"show": false,
"reducer": [
"sum"
],
"countRows": false,
"fields": ""
}
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_rrdp_rsync_failed_repository_duration_seconds",
"format": "table",
"instant": true,
"legendFormat": "",
"refId": "A"
}
],
"title": "RRDP + Rsync Failed Repositories",
"type": "table",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"job": true,
"terminal_state": true,
"rank": true,
"transport": true,
"__name__": true,
"publication_points": true,
"instance": true,
"repo_id": true,
"pp_id": true,
"exported_instance": true,
"rp": true,
"source": true
},
"indexByName": {
"Time": 0,
"host": 1,
"phase": 2,
"uri": 3,
"Value": 4
},
"renameByName": {
"Value": "duration"
}
}
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 29,
"w": 12,
"h": 9
},
"id": 11,
"options": {
"showHeader": true,
"cellHeight": "sm",
"footer": {
"show": false,
"reducer": [
"sum"
],
"countRows": false,
"fields": ""
}
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "topk(20, ours_rp_top_repository_sync_duration_seconds)",
"format": "table",
"instant": true,
"legendFormat": "",
"refId": "A"
}
],
"title": "Top 20 Repositories by Sync Duration",
"type": "table",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"job": true,
"terminal_state": true,
"__name__": true,
"publication_points": true,
"instance": true,
"repo_id": true,
"phase": true,
"pp_id": true,
"exported_instance": true,
"rp": true,
"source": true
},
"indexByName": {
"Time": 0,
"host": 1,
"rank": 2,
"transport": 3,
"uri": 4,
"Value": 5
},
"renameByName": {
"Value": "value"
}
}
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 38,
"w": 24,
"h": 9
},
"id": 10,
"options": {
"showHeader": true,
"cellHeight": "sm",
"footer": {
"show": false,
"reducer": [
"sum"
],
"countRows": false,
"fields": ""
}
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "topk(20, ours_rp_top_publication_point_object_count)",
"format": "table",
"instant": true,
"legendFormat": "",
"refId": "A"
}
],
"title": "Top Publication Points by Objects",
"type": "table",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"job": true,
"__name__": true,
"publication_points": true,
"instance": true,
"repo_id": true,
"phase": true,
"pp_id": true,
"exported_instance": true,
"rp": true,
"source": true
},
"indexByName": {
"Time": 0,
"host": 1,
"rank": 2,
"terminal_state": 3,
"transport": 4,
"uri": 5,
"Value": 6
},
"renameByName": {}
}
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"description": "Per-repository sync success in the latest successful run; 1 means successful, 0 means failed or failed_no_cache.",
"fieldConfig": {
"defaults": {
"unit": "bool"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 47
},
"id": 12,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"expr": "ours_rp_repository_sync_success",
"legendFormat": "{{host}} {{repo_id}}",
"refId": "A"
}
],
"title": "Repository Sync Success by Repo",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"description": "Per-repository total sync duration aggregated from publication point repo_sync_duration_ms.",
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 55
},
"id": 13,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"expr": "ours_rp_repository_sync_duration_seconds{stat=\"sum\"}",
"legendFormat": "{{host}} {{repo_id}}",
"refId": "A"
}
],
"title": "Repository Sync Duration by Repo",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"description": "Per-repository downloaded bytes attributed from report.json downloads events.",
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 63
},
"id": 14,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"expr": "ours_rp_repository_download_bytes",
"legendFormat": "{{host}} {{repo_id}}",
"refId": "A"
}
],
"title": "Repository Download Bytes by Repo",
"type": "timeseries"
}
],
"refresh": "5s",
"schemaVersion": 40,
"tags": [
"ours-rp",
"rpki",
"soak",
"repo-sync"
],
"templating": {
"list": []
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Ours RP Repo Sync",
"uid": "ours-rp-repo-sync",
"version": 3,
"weekStart": ""
}

666
deploy/docker-installer/compose/grafana/dashboards/ours-rp-rtr-overview.json
View File

@ -1,666 +0,0 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"id": 1,
"title": "RTR Metrics Enabled",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 4,
"w": 4,
"x": 0,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"expr": "max(ours_rp_rtr_metrics_enabled)",
"legendFormat": "enabled",
"refId": "A",
"instant": true
}
]
},
{
"id": 2,
"title": "Refresh Success",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 4,
"w": 5,
"x": 4,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"expr": "max(ours_rp_rtr_source_refresh_status{status=\"success\"})",
"legendFormat": "success",
"refId": "A",
"instant": true
}
]
},
{
"id": 3,
"title": "Consecutive Failures",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 4,
"w": 5,
"x": 9,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"expr": "max(ours_rp_rtr_source_refresh_consecutive_failures)",
"legendFormat": "failures",
"refId": "A",
"instant": true
}
]
},
{
"id": 4,
"title": "Last Success Age",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 4,
"w": 5,
"x": 14,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "s",
"decimals": 0,
"min": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"expr": "max(ours_rp_rtr_source_last_success_age_seconds)",
"legendFormat": "age",
"refId": "A",
"instant": true
}
]
},
{
"id": 5,
"title": "RTR RSS",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 4,
"w": 5,
"x": 19,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "bytes",
"decimals": 0,
"min": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"expr": "max(ours_rp_rtr_process_rss_bytes)",
"legendFormat": "rss",
"refId": "A",
"instant": true
}
]
},
{
"id": 6,
"title": "Data Quality Totals",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 4
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_rtr_data_quality_items{stage=~\"ccr_input|before_slurm|after_slurm\",type=\"total\"}",
"legendFormat": "{{stage}} total",
"refId": "A"
},
{
"expr": "ours_rp_rtr_data_quality_items{stage=\"after_slurm\",type=\"vrp\"}",
"legendFormat": "after_slurm vrp",
"refId": "B"
},
{
"expr": "ours_rp_rtr_data_quality_items{stage=\"after_slurm\",type=\"aspa\"}",
"legendFormat": "after_slurm aspa",
"refId": "C"
}
]
},
{
"id": 7,
"title": "SLURM Filters / Assertions",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 4
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_rtr_slurm_filters",
"legendFormat": "filter {{type}}",
"refId": "A"
},
{
"expr": "ours_rp_rtr_slurm_assertions",
"legendFormat": "assert {{type}}",
"refId": "B"
}
]
},
{
"id": 8,
"title": "Cache Ready / Delta Window",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 12
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_rtr_cache_ready",
"legendFormat": "ready",
"refId": "A"
},
{
"expr": "ours_rp_rtr_cache_delta_window_length",
"legendFormat": "v{{version}} length",
"refId": "B"
}
]
},
{
"id": 9,
"title": "Cache Snapshot Items",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 12
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_rtr_cache_snapshot_items",
"legendFormat": "v{{version}} {{type}}",
"refId": "A"
}
]
},
{
"id": 10,
"title": "Latest Delta Items",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 12
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_rtr_cache_delta_items",
"legendFormat": "v{{version}} {{direction}} {{type}}",
"refId": "A"
}
]
},
{
"id": 11,
"title": "Connections",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 20
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_rtr_active_connections",
"legendFormat": "active",
"refId": "A"
},
{
"expr": "ours_rp_rtr_connections",
"legendFormat": "{{transport}}",
"refId": "B"
}
]
},
{
"id": 12,
"title": "Connection Utilization / Max",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 20
},
"fieldConfig": {
"defaults": {
"min": 0
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": ".*utilization.*"
},
"properties": [
{
"id": "unit",
"value": "percentunit"
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": ".*max.*"
},
"properties": [
{
"id": "unit",
"value": "short"
}
]
}
]
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_rtr_connection_utilization",
"legendFormat": "utilization",
"refId": "A"
},
{
"expr": "ours_rp_rtr_max_connections",
"legendFormat": "max",
"refId": "B"
}
]
},
{
"id": 13,
"title": "Report Age",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 20
},
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_rtr_source_report_age_seconds",
"legendFormat": "source",
"refId": "A"
},
{
"expr": "ours_rp_rtr_runtime_report_age_seconds",
"legendFormat": "runtime",
"refId": "B"
},
{
"expr": "ours_rp_rtr_clients_report_age_seconds",
"legendFormat": "clients",
"refId": "C"
}
]
}
],
"refresh": "10s",
"schemaVersion": 40,
"tags": [
"rpki",
"inter-rp",
"routinator"
],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "browser",
"title": "RTR Service Overview",
"uid": "ours-rp-rtr-overview",
"version": 1
}

875
deploy/docker-installer/compose/grafana/dashboards/ours-rp-soak-overview.json
View File

@ -1,875 +0,0 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 0,
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 0,
"w": 6,
"h": 4
},
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "max(ours_rp_cir_trust_anchors and on (exported_instance) topk(1, ours_rp_run_sequence))",
"legendFormat": "RIRs",
"refId": "A"
}
],
"title": "Current Run RIRs",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"x": 6,
"y": 0,
"w": 6,
"h": 4
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "max(ours_rp_run_duration_seconds and on (exported_instance) topk(1, ours_rp_run_sequence))",
"legendFormat": "wall",
"refId": "A"
}
],
"title": "Latest Wall Time",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 0,
"w": 6,
"h": 4
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "max(ours_rp_run_max_rss_bytes and on (exported_instance) topk(1, ours_rp_run_sequence))",
"legendFormat": "rss",
"refId": "A"
}
],
"title": "Latest Max RSS",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 0,
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 18,
"y": 0,
"w": 6,
"h": 4
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "max(ours_rp_publication_points and on (exported_instance) topk(1, ours_rp_run_sequence))",
"legendFormat": "publication points",
"refId": "A"
}
],
"title": "Publication Points",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 0,
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 4,
"w": 6,
"h": 4
},
"id": 9,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "max(ours_rp_run_sequence)",
"legendFormat": "seq",
"refId": "A"
}
],
"title": "Latest Run Sequence",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 2,
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "orange",
"value": 90
},
{
"color": "green",
"value": 98
}
]
}
},
"overrides": []
},
"gridPos": {
"x": 6,
"y": 4,
"w": 6,
"h": 4
},
"id": 10,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "max((100 * sum by (exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"publication_point_cache\"}) / sum by (exported_instance) (ours_rp_publication_points)) and on (exported_instance) topk(1, ours_rp_run_sequence))",
"legendFormat": "PP cache hit ratio",
"refId": "A"
}
],
"title": "Latest PP Cache Hit Ratio",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none",
"decimals": 0
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 4,
"w": 6,
"h": 4
},
"id": 11,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "max(ours_rp_vrps{kind=\"total\"} and on (exported_instance) topk(1, ours_rp_run_sequence))",
"legendFormat": "VRPs raw",
"refId": "A"
}
],
"title": "VRPs",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none",
"decimals": 0
},
"overrides": []
},
"gridPos": {
"x": 18,
"y": 4,
"w": 6,
"h": 4
},
"id": 12,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "max(ours_rp_vaps and on (exported_instance) topk(1, ours_rp_run_sequence))",
"legendFormat": "VAPs",
"refId": "A"
}
],
"title": "VAPs",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"id": 5,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_run_duration_seconds",
"legendFormat": "wall",
"refId": "A"
},
{
"expr": "ours_rp_stage_duration_seconds{stage=\"validation\"}",
"legendFormat": "validation",
"refId": "B"
}
],
"title": "Run / Validation Duration",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 0,
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"id": 6,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_vrps{kind=\"total\"}",
"legendFormat": "VRPs raw",
"refId": "A"
},
{
"expr": "ours_rp_vrps{kind=\"unique\"}",
"legendFormat": "VRPs unique",
"refId": "D"
},
{
"expr": "ours_rp_vaps",
"legendFormat": "VAPs",
"refId": "B"
},
{
"expr": "ours_rp_cir_objects",
"legendFormat": "CIR objects",
"refId": "C"
}
],
"title": "Output and Input Sizes",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 0,
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"id": 8,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_large_publication_points",
"legendFormat": "> {{object_count_gt}} objects",
"refId": "A"
}
],
"title": "Large Publication Points by Object Count",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"id": 13,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"validation\"}",
"legendFormat": "validation",
"refId": "A"
},
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"report_write\"}",
"legendFormat": "report write",
"refId": "E"
},
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"ccr_write\"}",
"legendFormat": "ccr write",
"refId": "F"
},
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"cir_write\"}",
"legendFormat": "cir write",
"refId": "G"
}
],
"title": "Output Stage Durations",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "bytes",
"decimals": 2
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"id": 14,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_run_max_rss_bytes",
"legendFormat": "Max RSS",
"refId": "A"
}
],
"title": "Max RSS Over Time",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "percent",
"decimals": 2,
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "orange",
"value": 90
},
{
"color": "green",
"value": 98
}
]
}
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"id": 17,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"min",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "100 * sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"publication_point_cache\"}) / sum by (job, instance, exported_instance) (ours_rp_publication_points)",
"legendFormat": "PP cache hit ratio",
"refId": "A"
}
],
"title": "PP Cache Hit Ratio",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "bytes",
"decimals": 2
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 32,
"w": 24,
"h": 8
},
"id": 15,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_state_db_size_bytes",
"legendFormat": "{{db}}",
"refId": "A"
}
],
"title": "State DB Size Over Time",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none",
"decimals": 0
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 40,
"w": 24,
"h": 8
},
"id": 16,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_state_db_files",
"legendFormat": "{{db}}",
"refId": "A"
}
],
"title": "State DB File Count Over Time",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none",
"decimals": 0,
"min": 0
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 48,
"w": 24,
"h": 8
},
"id": 18,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"fresh\"})",
"legendFormat": "fresh pp",
"refId": "A"
},
{
"expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"roa\"})",
"legendFormat": "fresh roa",
"refId": "B"
},
{
"expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"manifest\"})",
"legendFormat": "fresh mft",
"refId": "C"
},
{
"expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"certificate\"})",
"legendFormat": "fresh crt",
"refId": "D"
},
{
"expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"crl\"})",
"legendFormat": "fresh crl",
"refId": "E"
}
],
"title": "Fresh PP / Object Counts by Run",
"type": "timeseries"
}
],
"refresh": "5s",
"schemaVersion": 40,
"tags": [
"ours-rp",
"rpki",
"soak"
],
"templating": {
"list": []
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Ours RP Soak Overview",
"uid": "ours-rp-soak-overview",
"version": 4,
"weekStart": ""
}

12
deploy/docker-installer/compose/grafana/provisioning/dashboards/dashboard.yml
View File

@ -1,12 +0,0 @@
apiVersion: 1
providers:
- name: ours-rp-docker-installer
orgId: 1
folder: Ours RP Docker Installer
type: file
disableDeletion: false
updateIntervalSeconds: 10
allowUiUpdates: true
options:
path: /var/lib/grafana/dashboards

10
deploy/docker-installer/compose/grafana/provisioning/datasources/prometheus.yml
View File

@ -1,10 +0,0 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: Prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: true

13
deploy/docker-installer/compose/prometheus.yml
View File

@ -1,13 +0,0 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
scrape_configs:
- job_name: ours-rp-artifact-metrics
metrics_path: /metrics
static_configs:
- targets:
- artifact-metrics:9556
labels:
rp: ours-rp
source: docker-installer-artifact-sidecar

14
deploy/docker-installer/custom-fixtures/README.md
View File

@ -1,14 +0,0 @@
# Custom TAL fixture mount
This directory is intentionally shipped as a mount point, not as a repository
of private keys. Run the patch tool `tools/generate_custom_fixture.py` to create
the test-only TAL, TA, RPKI objects, HTTPS CA/server certificates, RRDP files,
and rsync module data below it.
The generated fixture contains `baseline-v1`, `sync-hash-mismatch` and
`baseline-v2` for snapshot / hash / delta tests, plus
`validation-expired`, `validation-max-length`,
`validation-max-length-invalid`, `validation-roa-prefix-outside`,
`validation-out-of-resource`, `validation-ca-over-resource`, and
`validation-nonstandard` for validation cases. Generated keys are test-only
and must not be used in a production RPKI hierarchy.

30
deploy/docker-installer/custom-tal-patch/README.zh-CN.md
View File

@ -1,30 +0,0 @@
# 自建 TAL/TA 补丁
本目录描述并生成一个叠加到客户 Arm64 Ours RP 组件包上的测试补丁。补丁基线是源码 commit `9f4f4cf069e9fa8f6f97ed0f1065adac38abe82f`，不会替换客户包中的 runtime image 或 `bin/rpki`、`bin/rpki_daemon` 二进制。
补丁做三件事：
1. 在 runner 中增加 `RIRS=custom`、`TAL_INPUT_MODE=custom-file-with-ta`、自定义 TAL/TA 路径和 `HTTP_ROOT_CERT_PATHS`；
2. 通过 Compose 只读挂载 patched runner、`custom-fixtures/` 和 `host.docker.internal`，连接固定 HTTPS RRDP 与 rsync 服务；
3. 提供固定证书链、RPKI 对象生成器和服务启动脚本，不依赖 Barry、Rapport 或公网 RIR。
先停止 Ours RP，再执行：
```bash
./apply_custom_tal_patch.sh \
--component-root /opt/ours-rp-rtr-stack/components/ours-rp \
--enable-custom
```
`--enable-custom` 会修改组件 `.env` 的 custom 模式入口；不加该选项只安装 overlay，不改变当前 `.env`。回滚：
```bash
./rollback_custom_tal_patch.sh \
--component-root /opt/ours-rp-rtr-stack/components/ours-rp
```
补丁应用脚本会校验 `PACKAGE-MANIFEST.env` 的源码 commit 和 `arm64` 架构，并留下 `CUSTOM-TAL-PATCH-MANIFEST.env` 与 `.custom-tal-patch-state/` 备份。升级组件后需要重新检查 patch marker；不要把该 overlay 误当成客户原始安装包。
生成器默认把容器访问地址写成 `host.docker.internal`，固定服务端口为 HTTPS RRDP `18443`、rsync `1873`，rsync module 为 `custom`。可用 `--host localhost` 生成给宿主机直接运行 `bin/rpki` 的本地变体。
用例包括：`baseline-v1`、`sync-hash-mismatch`、`baseline-v2`、`validation-expired`、`validation-max-length`、`validation-max-length-invalid`、`validation-roa-prefix-outside`、`validation-out-of-resource`、`validation-ca-over-resource`、`validation-nonstandard`。其中 `validation-ca-over-resource` 在 `RESOURCE_VALIDATION_MODE=rfc6487` 下验证严格拒绝；`validation-update-03` 是原包默认的兼容处理模式。

187
deploy/docker-installer/custom-tal-patch/apply_custom_tal_patch.sh
View File

@ -1,187 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PATCH_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
PAYLOAD_ROOT="$SCRIPT_DIR/payload"
STATE_DIR_NAME=".custom-tal-patch-state"
EXPECTED_SOURCE_COMMIT="9f4f4cf069e9fa8f6f97ed0f1065adac38abe82f"
usage() {
cat <<'USAGE'
Usage:
./apply_custom_tal_patch.sh --component-root <installed-ours-rp-root>
./apply_custom_tal_patch.sh --component-root <root> --enable-custom
The default operation installs the overlay but does not change the existing
.env. --enable-custom writes the custom TAL/TA/root-cert example values into
the component .env so the next run uses the fixed local fixture.
Stop the ours RP service before applying or rolling back the overlay.
USAGE
}
die() {
echo "error: $*" >&2
exit 2
}
COMPONENT_ROOT=""
ENABLE_CUSTOM=0
while [[ $# -gt 0 ]]; do
case "$1" in
--component-root)
COMPONENT_ROOT="${2:-}"
shift 2
;;
--enable-custom)
ENABLE_CUSTOM=1
shift
;;
-h|--help)
usage
exit 0
;;
*)
die "unknown option: $1"
;;
esac
done
[[ -n "$COMPONENT_ROOT" ]] || die "--component-root is required"
COMPONENT_ROOT="$(cd "$COMPONENT_ROOT" 2>/dev/null && pwd)" \
|| die "component root does not exist: $COMPONENT_ROOT"
[[ -f "$COMPONENT_ROOT/PACKAGE-MANIFEST.env" ]] \
|| die "missing component PACKAGE-MANIFEST.env: $COMPONENT_ROOT"
[[ -d "$PAYLOAD_ROOT" ]] || die "missing patch payload: $PAYLOAD_ROOT"
# shellcheck disable=SC1091
source "$COMPONENT_ROOT/PACKAGE-MANIFEST.env"
[[ "${source_commit:-}" == "$EXPECTED_SOURCE_COMMIT" ]] \
|| die "base source commit mismatch: ${source_commit:-missing} != $EXPECTED_SOURCE_COMMIT"
[[ "${package_arch:-${PACKAGE_ARCH:-}}" == "arm64" ]] \
|| die "this patch is for the customer Arm64 component, got ${package_arch:-${PACKAGE_ARCH:-missing}}"
STATE_ROOT="$COMPONENT_ROOT/$STATE_DIR_NAME"
BACKUP_ROOT="$STATE_ROOT/original"
marker="$COMPONENT_ROOT/CUSTOM-TAL-PATCH-MANIFEST.env"
set_env_value() {
local env_path="$1"
local key="$2"
local value="$3"
local tmp_path="${env_path}.custom-tal-patch.tmp"
[[ -f "$env_path" ]] || die "missing component .env: $env_path"
awk -v key="$key" -v value="$value" '
BEGIN { found = 0 }
$0 ~ "^" key "=" { print key "=" value; found = 1; next }
{ print }
END { if (!found) print key "=" value }
' "$env_path" > "$tmp_path"
mv "$tmp_path" "$env_path"
}
copy_payload_file() {
local source_path="$1"
local target_path="$2"
mkdir -p "$(dirname "$target_path")"
cp "$source_path" "$target_path"
chmod --reference="$source_path" "$target_path" 2>/dev/null || true
}
append_custom_env_example() {
local target_path="$1"
if ! grep -Eq '^CUSTOM_TAL_URI=' "$target_path"; then
{
printf '\n# #142 custom TAL patch settings; original package values above are preserved.\n'
cat "$SCRIPT_DIR/custom-tal.env.example"
} >> "$target_path"
fi
}
if [[ -e "$STATE_ROOT" ]]; then
die "patch is already applied or an incomplete state exists: $STATE_ROOT"
fi
mkdir -p "$BACKUP_ROOT"
for relative_path in compose/docker-compose.yml .env.example; do
[[ -f "$COMPONENT_ROOT/$relative_path" ]] \
|| die "missing base file: $COMPONENT_ROOT/$relative_path"
mkdir -p "$BACKUP_ROOT/$(dirname "$relative_path")"
cp -a "$COMPONENT_ROOT/$relative_path" "$BACKUP_ROOT/$relative_path"
done
runner_path="$COMPONENT_ROOT/scripts/soak/run_soak.sh"
if [[ -e "$runner_path" ]]; then
mkdir -p "$BACKUP_ROOT/scripts/soak"
cp -a "$runner_path" "$BACKUP_ROOT/scripts/soak/run_soak.sh"
printf 'runner_was_present=1\n' > "$STATE_ROOT/state.env"
else
printf 'runner_was_present=0\n' > "$STATE_ROOT/state.env"
fi
if [[ -d "$COMPONENT_ROOT/custom-fixtures" ]]; then
cp -a "$COMPONENT_ROOT/custom-fixtures" "$BACKUP_ROOT/custom-fixtures"
printf 'custom_fixtures_were_present=1\n' >> "$STATE_ROOT/state.env"
else
printf 'custom_fixtures_were_present=0\n' >> "$STATE_ROOT/state.env"
fi
if [[ -d "$COMPONENT_ROOT/custom-tal-patch" ]]; then
cp -a "$COMPONENT_ROOT/custom-tal-patch" "$BACKUP_ROOT/custom-tal-patch"
printf 'patch_tools_were_present=1\n' >> "$STATE_ROOT/state.env"
else
printf 'patch_tools_were_present=0\n' >> "$STATE_ROOT/state.env"
fi
if [[ -f "$COMPONENT_ROOT/custom-tal.env.example" ]]; then
cp -a "$COMPONENT_ROOT/custom-tal.env.example" "$BACKUP_ROOT/custom-tal.env.example"
printf 'custom_env_example_was_present=1\n' >> "$STATE_ROOT/state.env"
else
printf 'custom_env_example_was_present=0\n' >> "$STATE_ROOT/state.env"
fi
if [[ "$ENABLE_CUSTOM" == "1" ]]; then
[[ -f "$COMPONENT_ROOT/.env" ]] || die "missing component .env: $COMPONENT_ROOT/.env"
cp -a "$COMPONENT_ROOT/.env" "$BACKUP_ROOT/.env"
fi
copy_payload_file "$PAYLOAD_ROOT/compose/docker-compose.yml" "$COMPONENT_ROOT/compose/docker-compose.yml"
append_custom_env_example "$COMPONENT_ROOT/.env.example"
copy_payload_file "$PAYLOAD_ROOT/scripts/soak/run_soak.sh" "$runner_path"
mkdir -p "$COMPONENT_ROOT/custom-fixtures/tal" \
"$COMPONENT_ROOT/custom-fixtures/ta" \
"$COMPONENT_ROOT/custom-fixtures/certs" \
"$COMPONENT_ROOT/custom-fixtures/services"
if [[ -d "$PAYLOAD_ROOT/custom-fixtures" ]]; then
cp -a "$PAYLOAD_ROOT/custom-fixtures/." "$COMPONENT_ROOT/custom-fixtures/"
fi
mkdir -p "$COMPONENT_ROOT/custom-tal-patch/tools"
cp -a "$PAYLOAD_ROOT/tools/." "$COMPONENT_ROOT/custom-tal-patch/tools/"
copy_payload_file "$SCRIPT_DIR/custom-tal.env.example" \
"$COMPONENT_ROOT/custom-tal.env.example"
if [[ "$ENABLE_CUSTOM" == "1" ]]; then
set_env_value "$COMPONENT_ROOT/.env" RIRS custom
set_env_value "$COMPONENT_ROOT/.env" TAL_INPUT_MODE custom-file-with-ta
set_env_value "$COMPONENT_ROOT/.env" CUSTOM_FIXTURE_HOST_DIR ../custom-fixtures
set_env_value "$COMPONENT_ROOT/.env" CUSTOM_TAL_PATH /opt/ours-rp/custom-fixtures/tal/custom.tal
set_env_value "$COMPONENT_ROOT/.env" CUSTOM_TA_PATH /opt/ours-rp/custom-fixtures/ta/custom-ta.cer
set_env_value "$COMPONENT_ROOT/.env" CUSTOM_TAL_URI https://host.docker.internal:18443/tal/custom.tal
set_env_value "$COMPONENT_ROOT/.env" HTTP_ROOT_CERT_PATHS /opt/ours-rp/custom-fixtures/certs/rrdp-ca.pem
printf 'env_was_enabled=1\n' >> "$STATE_ROOT/state.env"
else
printf 'env_was_enabled=0\n' >> "$STATE_ROOT/state.env"
fi
[[ -f "$SCRIPT_DIR/PATCH-MANIFEST.env" ]] \
|| die "missing PATCH-MANIFEST.env"
cp "$SCRIPT_DIR/PATCH-MANIFEST.env" "$marker"
printf 'patch_root=%s\n' "$PATCH_ROOT" >> "$STATE_ROOT/state.env"
printf 'applied_at_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$STATE_ROOT/state.env"
echo "custom TAL patch applied to $COMPONENT_ROOT"
if [[ "$ENABLE_CUSTOM" == "1" ]]; then
echo "custom mode enabled in $COMPONENT_ROOT/.env"
else
echo "custom mode remains disabled; review custom-tal.env.example before enabling it"
fi

15
deploy/docker-installer/custom-tal-patch/custom-tal.env.example
View File

@ -1,15 +0,0 @@
# Copy these values into the component .env after applying the patch.
# The patch's --enable-custom option writes the same values automatically.
RIRS=custom
TAL_INPUT_MODE=custom-file-with-ta
CUSTOM_FIXTURE_HOST_DIR=../custom-fixtures
CUSTOM_TAL_PATH=/opt/ours-rp/custom-fixtures/tal/custom.tal
CUSTOM_TA_PATH=/opt/ours-rp/custom-fixtures/ta/custom-ta.cer
CUSTOM_TAL_URI=https://host.docker.internal:18443/tal/custom.tal
HTTP_ROOT_CERT_PATHS=/opt/ours-rp/custom-fixtures/certs/rrdp-ca.pem
# Recommended for a finite acceptance run; keep the customer default for a
# long-running test only after the fixture service is supervised.
MAX_RUNS=1
INTERVAL_SECS=0
SOAK_RESTART_POLICY=no

80
deploy/docker-installer/custom-tal-patch/rollback_custom_tal_patch.sh
View File

@ -1,80 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'USAGE'
Usage:
./rollback_custom_tal_patch.sh --component-root <installed-ours-rp-root>
Stop the ours RP service before rolling back the overlay.
USAGE
}
die() {
echo "error: $*" >&2
exit 2
}
COMPONENT_ROOT=""
while [[ $# -gt 0 ]]; do
case "$1" in
--component-root)
COMPONENT_ROOT="${2:-}"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
die "unknown option: $1"
;;
esac
done
[[ -n "$COMPONENT_ROOT" ]] || die "--component-root is required"
COMPONENT_ROOT="$(cd "$COMPONENT_ROOT" 2>/dev/null && pwd)" \
|| die "component root does not exist: $COMPONENT_ROOT"
STATE_ROOT="$COMPONENT_ROOT/.custom-tal-patch-state"
BACKUP_ROOT="$STATE_ROOT/original"
[[ -d "$STATE_ROOT" ]] || die "patch state not found: $STATE_ROOT"
[[ -f "$BACKUP_ROOT/compose/docker-compose.yml" ]] \
|| die "patch backup is incomplete: $BACKUP_ROOT"
cp -a "$BACKUP_ROOT/compose/docker-compose.yml" "$COMPONENT_ROOT/compose/docker-compose.yml"
cp -a "$BACKUP_ROOT/.env.example" "$COMPONENT_ROOT/.env.example"
if [[ -f "$BACKUP_ROOT/scripts/soak/run_soak.sh" ]]; then
mkdir -p "$COMPONENT_ROOT/scripts/soak"
cp -a "$BACKUP_ROOT/scripts/soak/run_soak.sh" "$COMPONENT_ROOT/scripts/soak/run_soak.sh"
else
rm -f "$COMPONENT_ROOT/scripts/soak/run_soak.sh"
rmdir "$COMPONENT_ROOT/scripts/soak" 2>/dev/null || true
fi
if [[ -d "$BACKUP_ROOT/custom-fixtures" ]]; then
rm -rf "$COMPONENT_ROOT/custom-fixtures"
cp -a "$BACKUP_ROOT/custom-fixtures" "$COMPONENT_ROOT/custom-fixtures"
else
rm -rf "$COMPONENT_ROOT/custom-fixtures"
fi
if [[ -d "$BACKUP_ROOT/custom-tal-patch" ]]; then
rm -rf "$COMPONENT_ROOT/custom-tal-patch"
cp -a "$BACKUP_ROOT/custom-tal-patch" "$COMPONENT_ROOT/custom-tal-patch"
else
rm -rf "$COMPONENT_ROOT/custom-tal-patch"
fi
if [[ -f "$BACKUP_ROOT/custom-tal.env.example" ]]; then
cp -a "$BACKUP_ROOT/custom-tal.env.example" "$COMPONENT_ROOT/custom-tal.env.example"
else
rm -f "$COMPONENT_ROOT/custom-tal.env.example"
fi
if [[ -f "$BACKUP_ROOT/.env" ]]; then
cp -a "$BACKUP_ROOT/.env" "$COMPONENT_ROOT/.env"
fi
rm -f "$COMPONENT_ROOT/CUSTOM-TAL-PATCH-MANIFEST.env"
rm -rf "$STATE_ROOT"
echo "custom TAL patch rolled back from $COMPONENT_ROOT"

636
deploy/docker-installer/custom-tal-patch/tools/generate_custom_fixture.py
View File

@ -1,636 +0,0 @@
#!/usr/bin/env python3
"""Generate a small self-contained RPKI repository for installer tests.
The generator uses Python cryptography and the OpenSSL CLI for CMS signing.
It does not invoke Barry, Rapport, or a public RIR service.
"""
from __future__ import annotations
import argparse
import base64
import hashlib
import ipaddress
import shutil
import subprocess
import tempfile
from datetime import datetime, timedelta, timezone
from pathlib import Path
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat, PublicFormat
from cryptography.x509 import AccessDescription, AuthorityInformationAccess, DistributionPoint, UniformResourceIdentifier
from cryptography.x509.oid import AuthorityInformationAccessOID, NameOID, ObjectIdentifier
RRDP_HOST = "host.docker.internal"
RRDP_PORT = 18443
RSYNC_PORT = 1873
TAL_URI = f"rsync://{RRDP_HOST}:{RSYNC_PORT}/custom/ta.cer"
TAL_HTTPS_URI = f"https://{RRDP_HOST}:{RRDP_PORT}/ta/custom-ta.cer"
RRDP_URI = f"https://{RRDP_HOST}:{RRDP_PORT}/rrdp/notification.xml"
ROOT_REPO_URI = f"rsync://{RRDP_HOST}:{RSYNC_PORT}/custom/root/"
ROOT_MFT_URI = f"{ROOT_REPO_URI}root.mft"
CHILD_REPO_URI = f"rsync://{RRDP_HOST}:{RSYNC_PORT}/custom/child/"
CHILD_MFT_URI = f"{CHILD_REPO_URI}child.mft"
OID_SIA = ObjectIdentifier("1.3.6.1.5.5.7.1.11")
OID_RESOURCES_IP = ObjectIdentifier("1.3.6.1.5.5.7.1.7")
OID_RESOURCES_AS = ObjectIdentifier("1.3.6.1.5.5.7.1.8")
OID_POLICY_IP_AS = ObjectIdentifier("1.3.6.1.5.5.7.14.2")
OID_ROA_ECONTENT = "1.2.840.113549.1.9.16.1.24"
OID_MANIFEST_ECONTENT = "1.2.840.113549.1.9.16.1.26"
OID_SHA256 = "2.16.840.1.101.3.4.2.1"
SIA_CA_REPOSITORY = "1.3.6.1.5.5.7.48.5"
SIA_RPKI_MANIFEST = "1.3.6.1.5.5.7.48.10"
SIA_SIGNED_OBJECT = "1.3.6.1.5.5.7.48.11"
SIA_RPKI_NOTIFY = "1.3.6.1.5.5.7.48.13"
UTC = timezone.utc
VALID_FROM = datetime(2026, 1, 1, tzinfo=UTC)
VALID_TO = datetime(2027, 12, 31, 23, 59, 59, tzinfo=UTC)
CHILD_PREFIX = ipaddress.ip_network("203.0.113.0/24")
SECOND_PREFIX = ipaddress.ip_network("203.0.113.128/25")
OUTSIDE_PREFIX = ipaddress.ip_network("203.0.114.0/24")
TEST_ASN = 64496
SESSION_ID = "11111111-2222-4333-8444-555555555555"
def tlv(tag: int, value: bytes) -> bytes:
length = len(value)
if length < 128:
encoded_length = bytes([length])
else:
raw = length.to_bytes((length.bit_length() + 7) // 8, "big")
encoded_length = bytes([0x80 | len(raw)]) + raw
return bytes([tag]) + encoded_length + value
def seq(*values: bytes) -> bytes:
return tlv(0x30, b"".join(values))
def integer(value: int) -> bytes:
raw = value.to_bytes(max(1, (value.bit_length() + 7) // 8), "big")
if raw[0] & 0x80:
raw = b"\x00" + raw
return tlv(0x02, raw)
def oid(oid_value: str) -> bytes:
parts = [int(part) for part in oid_value.split(".")]
encoded = bytearray([40 * parts[0] + parts[1]])
for part in parts[2:]:
chunks = [part & 0x7F]
part >>= 7
while part:
chunks.append(0x80 | (part & 0x7F))
part >>= 7
encoded.extend(reversed(chunks))
return tlv(0x06, bytes(encoded))
def octet(value: bytes) -> bytes:
return tlv(0x04, value)
def uri(value: str) -> bytes:
return tlv(0x86, value.encode("ascii"))
def bit_string(value: bytes, unused: int = 0) -> bytes:
return tlv(0x03, bytes([unused]) + value)
def generalized_time(value: datetime) -> bytes:
return tlv(0x18, value.astimezone(UTC).strftime("%Y%m%d%H%M%SZ").encode("ascii"))
def context_zero(value: bytes) -> bytes:
return tlv(0xA0, value)
def ip_choice(networks: list[ipaddress._BaseNetwork]) -> bytes:
entries = []
for network in networks:
length = (network.prefixlen + 7) // 8
raw = network.network_address.packed[:length]
entries.append(bit_string(raw, length * 8 - network.prefixlen))
return seq(*entries)
def ip_resources(networks: list[ipaddress._BaseNetwork]) -> bytes:
families: dict[int, list[ipaddress._BaseNetwork]] = {1: [], 2: []}
for network in networks:
families[1 if network.version == 4 else 2].append(network)
return seq(
*(
seq(octet(afi.to_bytes(2, "big")), ip_choice(items))
for afi, items in families.items()
if items
)
)
def ip_inherit() -> bytes:
return seq(
seq(octet(b"\x00\x01"), tlv(0x05, b"")),
seq(octet(b"\x00\x02"), tlv(0x05, b"")),
)
def as_resources(asns: list[int] | None) -> bytes:
if asns is None:
return seq(context_zero(tlv(0x05, b"")))
return seq(context_zero(seq(*(integer(asn) for asn in asns))))
def sia(entries: list[tuple[str, str]]) -> bytes:
return seq(*(seq(oid(method), uri(location)) for method, location in entries))
def write(path: Path, data: bytes, mode: int | None = None) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
if mode is not None:
path.chmod(mode)
def pem_cert(cert: x509.Certificate) -> bytes:
return cert.public_bytes(Encoding.PEM)
def pem_key(key: rsa.RSAPrivateKey) -> bytes:
return key.private_bytes(Encoding.PEM, PrivateFormat.TraditionalOpenSSL, NoEncryption())
def write_key(path: Path, key: rsa.RSAPrivateKey) -> None:
write(path, pem_key(key), 0o600)
def sign_cert(
subject: x509.Name,
issuer: x509.Name,
public_key,
signing_key: rsa.RSAPrivateKey,
serial: int,
is_ca: bool,
ski_key: rsa.RSAPrivateKey,
aki_key: rsa.RSAPrivateKey | None = None,
ip_ext: bytes | None = None,
as_ext: bytes | None = None,
ca_repository: str | None = None,
manifest_uri: str | None = None,
notify_uri: str | None = None,
signed_object_uri: str | None = None,
crl_uri: str | None = None,
ca_issuer_uri: str | None = None,
not_before: datetime = VALID_FROM,
not_after: datetime = VALID_TO,
) -> x509.Certificate:
builder = (
x509.CertificateBuilder()
.subject_name(subject)
.issuer_name(issuer)
.public_key(public_key)
.serial_number(serial)
.not_valid_before(not_before)
.not_valid_after(not_after)
)
if is_ca:
builder = builder.add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True)
if is_ca:
usage = x509.KeyUsage(False, False, False, False, False, True, True, False, False)
else:
usage = x509.KeyUsage(True, False, False, False, False, False, False, False, False)
builder = builder.add_extension(usage, critical=True)
builder = builder.add_extension(x509.SubjectKeyIdentifier.from_public_key(ski_key.public_key()), critical=False)
if aki_key is not None:
builder = builder.add_extension(x509.AuthorityKeyIdentifier.from_issuer_public_key(aki_key.public_key()), critical=False)
builder = builder.add_extension(
x509.CertificatePolicies([x509.PolicyInformation(OID_POLICY_IP_AS, None)]),
critical=True,
)
if crl_uri is not None:
builder = builder.add_extension(
x509.CRLDistributionPoints(
[DistributionPoint([UniformResourceIdentifier(crl_uri)], None, None, None)]
),
critical=False,
)
if ca_issuer_uri is not None:
builder = builder.add_extension(
AuthorityInformationAccess(
[AccessDescription(AuthorityInformationAccessOID.CA_ISSUERS, UniformResourceIdentifier(ca_issuer_uri))]
),
critical=False,
)
sia_entries: list[tuple[str, str]] = []
if ca_repository is not None:
sia_entries.append((SIA_CA_REPOSITORY, ca_repository))
if manifest_uri is not None:
sia_entries.append((SIA_RPKI_MANIFEST, manifest_uri))
if notify_uri is not None:
sia_entries.append((SIA_RPKI_NOTIFY, notify_uri))
if signed_object_uri is not None:
sia_entries.append((SIA_SIGNED_OBJECT, signed_object_uri))
if sia_entries:
builder = builder.add_extension(x509.UnrecognizedExtension(OID_SIA, sia(sia_entries)), critical=False)
if ip_ext is not None:
builder = builder.add_extension(x509.UnrecognizedExtension(OID_RESOURCES_IP, ip_ext), critical=True)
if as_ext is not None:
builder = builder.add_extension(x509.UnrecognizedExtension(OID_RESOURCES_AS, as_ext), critical=True)
return builder.sign(signing_key, hashes.SHA256())
def make_crl(subject: x509.Name, key: rsa.RSAPrivateKey, number: int) -> x509.CertificateRevocationList:
return (
x509.CertificateRevocationListBuilder()
.issuer_name(subject)
.last_update(VALID_FROM + timedelta(days=30))
.next_update(VALID_TO - timedelta(days=30))
.add_extension(x509.AuthorityKeyIdentifier.from_issuer_public_key(key.public_key()), critical=False)
.add_extension(x509.CRLNumber(number), critical=False)
.sign(key, hashes.SHA256())
)
def roa_content(prefix: ipaddress._BaseNetwork, max_length: int) -> bytes:
length = (prefix.prefixlen + 7) // 8
raw = prefix.network_address.packed[:length]
address = seq(bit_string(raw, length * 8 - prefix.prefixlen), integer(max_length))
family = seq(octet((1 if prefix.version == 4 else 2).to_bytes(2, "big")), seq(address))
return seq(integer(TEST_ASN), seq(family))
def manifest_content(number: int, files: list[tuple[str, bytes]]) -> bytes:
entries = [
seq(tlv(0x16, name.encode("ascii")), bit_string(hashlib.sha256(content).digest()))
for name, content in files
]
return seq(
integer(number),
generalized_time(VALID_FROM + timedelta(days=30)),
generalized_time(VALID_TO - timedelta(days=30)),
oid(OID_SHA256),
seq(*entries),
)
def write_cms(content: bytes, content_type: str, signer_cert: x509.Certificate, signer_key: rsa.RSAPrivateKey, output: Path, work: Path) -> None:
content_path = work / f"{output.name}.content.der"
signer_path = work / f"{output.name}.signer.pem"
key_path = work / f"{output.name}.key.pem"
write(content_path, content)
write(signer_path, pem_cert(signer_cert))
write_key(key_path, signer_key)
subprocess.run(
[
"openssl", "cms", "-sign", "-binary", "-in", str(content_path),
"-signer", str(signer_path), "-inkey", str(key_path),
"-outform", "DER", "-nodetach", "-nosmimecap", "-keyid", "-md", "sha256",
"-econtent_type", content_type, "-out", str(output),
],
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
)
def make_roa(
name: str,
prefix,
max_length: int,
not_before: datetime,
not_after: datetime,
child_name,
child_key,
common: Path,
work: Path,
certificate_prefix=None,
) -> None:
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
cert = sign_cert(
x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, f"#142 EE {name}")]),
child_name,
key.public_key(),
child_key,
x509.random_serial_number(),
False,
key,
child_key,
ip_ext=ip_resources([certificate_prefix or prefix]),
signed_object_uri=f"{CHILD_REPO_URI}{name}",
crl_uri=f"{CHILD_REPO_URI}child.crl",
ca_issuer_uri=f"{CHILD_REPO_URI}child.cer",
not_before=not_before,
not_after=not_after,
)
write_cms(roa_content(prefix, max_length), OID_ROA_ECONTENT, cert, key, common / name, work)
def make_manifest(name: str, number: int, files: list[tuple[str, bytes]], child_name, child_key, common: Path, work: Path) -> bytes:
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
cert = sign_cert(
x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, f"#142 Manifest EE {name}")]),
child_name,
key.public_key(),
child_key,
x509.random_serial_number(),
False,
key,
child_key,
ip_ext=ip_inherit(),
as_ext=as_resources(None),
signed_object_uri=CHILD_MFT_URI if name != "root" else ROOT_MFT_URI,
crl_uri=f"{CHILD_REPO_URI}child.crl" if name != "root" else f"{ROOT_REPO_URI}root.crl",
ca_issuer_uri=f"{CHILD_REPO_URI}child.cer" if name != "root" else f"{ROOT_REPO_URI}ta.cer",
)
output = common / f"{name}.mft"
write_cms(manifest_content(number, files), OID_MANIFEST_ECONTENT, cert, key, output, work)
return output.read_bytes()
def publish(uri: str, path: Path) -> str:
encoded = base64.b64encode(path.read_bytes()).decode("ascii")
return f' <publish uri="{uri}">{encoded}</publish>\n'
def snapshot(case_root: Path, names: list[str], serial: int, extra_root_names: list[str] | None = None) -> bytes:
repository = case_root / "repository"
objects = [
(f"{ROOT_REPO_URI}root.crl", repository / "root/root.crl"),
(f"{ROOT_REPO_URI}child.cer", repository / "root/child.cer"),
*[
(f"{ROOT_REPO_URI}{name}", repository / f"root/{name}")
for name in extra_root_names or []
],
(f"{CHILD_REPO_URI}child.crl", repository / "child/child.crl"),
*[(f"{CHILD_REPO_URI}{name}", repository / f"child/{name}") for name in names],
(ROOT_MFT_URI, repository / "root/root.mft"),
(CHILD_MFT_URI, repository / "child/child.mft"),
]
body = "".join(publish(uri, path) for uri, path in objects)
return (
f'<snapshot xmlns="http://www.ripe.net/rpki/rrdp" session_id="{SESSION_ID}" serial="{serial}">\n'
f"{body}</snapshot>\n"
).encode()
def notification(serial: int, snap: bytes, delta: bytes | None) -> bytes:
lines = [
f'<notification xmlns="http://www.ripe.net/rpki/rrdp" version="1" session_id="{SESSION_ID}" serial="{serial}">',
f' <snapshot uri="https://{RRDP_HOST}:{RRDP_PORT}/rrdp/snapshot.xml" hash="{hashlib.sha256(snap).hexdigest()}"/>',
]
if delta is not None:
lines.append(
f' <delta serial="{serial}" uri="https://{RRDP_HOST}:{RRDP_PORT}/rrdp/delta-{serial}.xml" hash="{hashlib.sha256(delta).hexdigest()}"/>'
)
return ("\n".join(lines) + "\n</notification>\n").encode()
def corrupt_first_hash(xml: bytes) -> bytes:
marker = b'hash="'
start = xml.index(marker) + len(marker)
end = xml.index(b'"', start)
value = bytearray(xml[start:end])
value[0] = ord("0") if value[0] != ord("0") else ord("1")
return xml[:start] + bytes(value) + xml[end:]
def delta(add_name: str, add_path: Path, new_mft: Path) -> bytes:
body = publish(CHILD_MFT_URI, new_mft)
body += publish(f"{CHILD_REPO_URI}{add_name}", add_path)
return f'<delta xmlns="http://www.ripe.net/rpki/rrdp" session_id="{SESSION_ID}" serial="2">\n{body}</delta>\n'.encode()
def build_case(
output: Path,
common: Path,
tal: bytes,
ta: bytes,
name: str,
number: int,
names: list[str],
child_mft: bytes,
change: bytes | None = None,
root_mft: bytes | None = None,
extra_root_objects: list[tuple[str, bytes]] | None = None,
notification_hash_mismatch: bool = False,
) -> None:
case_root = output / "cases" / name
repo = case_root / "repository"
(repo / "root").mkdir(parents=True, exist_ok=True)
(repo / "child").mkdir(parents=True, exist_ok=True)
shutil.copy2(common / "ta.cer", repo / "ta.cer")
for item in ("root.crl", "child.cer"):
shutil.copy2(common / item, repo / "root" / item)
(repo / "root/root.mft").write_bytes(root_mft if root_mft is not None else (common / "root.mft").read_bytes())
for item, content in extra_root_objects or []:
(repo / "root" / item).write_bytes(content)
for item in (
"child.crl", "valid.roa", "valid2.roa", "expired.roa", "max-length.roa",
"max-length-invalid.roa", "roa-prefix-outside.roa", "out-of-resource.roa", "bad-format.roa",
):
if (common / item).exists():
shutil.copy2(common / item, repo / "child" / item)
(repo / "child/child.mft").write_bytes(child_mft)
snap = snapshot(case_root, names, number, [item for item, _ in extra_root_objects or []])
(case_root / "http/rrdp").mkdir(parents=True, exist_ok=True)
(case_root / "http/tal").mkdir(parents=True, exist_ok=True)
(case_root / "http/ta").mkdir(parents=True, exist_ok=True)
(case_root / "http/rrdp/snapshot.xml").write_bytes(snap)
notification_xml = notification(number, snap, change)
if notification_hash_mismatch:
notification_xml = corrupt_first_hash(notification_xml)
(case_root / "http/rrdp/notification.xml").write_bytes(notification_xml)
if change is not None:
(case_root / "http/rrdp/delta-2.xml").write_bytes(change)
(case_root / "http/tal/custom.tal").write_bytes(tal)
(case_root / "http/ta/custom-ta.cer").write_bytes(ta)
(case_root / "CASE-MANIFEST.txt").write_text(
f"case={name}\nserial={number}\nobjects={','.join(names)}\nsnapshot_sha256={hashlib.sha256(snap).hexdigest()}\n",
encoding="utf-8",
)
def make_https_certificates(output: Path) -> None:
ca_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
ca_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "#142 fixed RRDP test CA")])
ca_cert = (
x509.CertificateBuilder()
.subject_name(ca_name).issuer_name(ca_name).public_key(ca_key.public_key())
.serial_number(x509.random_serial_number()).not_valid_before(VALID_FROM).not_valid_after(VALID_TO)
.add_extension(x509.BasicConstraints(ca=True, path_length=1), critical=True)
.add_extension(x509.KeyUsage(False, False, False, False, False, True, True, False, False), critical=True)
.sign(ca_key, hashes.SHA256())
)
server_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
server_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, RRDP_HOST)])
try:
server_host = x509.IPAddress(ipaddress.ip_address(RRDP_HOST))
except ValueError:
server_host = x509.DNSName(RRDP_HOST)
san_entries = [server_host]
if RRDP_HOST != "localhost":
san_entries.append(x509.DNSName("localhost"))
server_cert = (
x509.CertificateBuilder()
.subject_name(server_name).issuer_name(ca_cert.subject).public_key(server_key.public_key())
.serial_number(x509.random_serial_number()).not_valid_before(VALID_FROM).not_valid_after(VALID_TO)
.add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True)
.add_extension(x509.KeyUsage(True, False, False, False, False, False, False, False, False), critical=True)
.add_extension(x509.SubjectAlternativeName(san_entries), critical=False)
.sign(ca_key, hashes.SHA256())
)
write(output / "certs/rrdp-ca.pem", pem_cert(ca_cert))
write_key(output / "certs/rrdp-ca.key", ca_key)
write(output / "certs/rrdp-server.pem", pem_cert(server_cert))
write_key(output / "certs/rrdp-server.key", server_key)
def main() -> None:
global RRDP_HOST, TAL_URI, TAL_HTTPS_URI, RRDP_URI
global ROOT_REPO_URI, ROOT_MFT_URI, CHILD_REPO_URI, CHILD_MFT_URI
parser = argparse.ArgumentParser()
parser.add_argument("--output", type=Path, required=True)
parser.add_argument(
"--host",
default=RRDP_HOST,
help="host embedded in TAL, certificate SIA and RRDP data (default: host.docker.internal)",
)
args = parser.parse_args()
RRDP_HOST = args.host
TAL_URI = f"rsync://{RRDP_HOST}:{RSYNC_PORT}/custom/ta.cer"
TAL_HTTPS_URI = f"https://{RRDP_HOST}:{RRDP_PORT}/ta/custom-ta.cer"
RRDP_URI = f"https://{RRDP_HOST}:{RRDP_PORT}/rrdp/notification.xml"
ROOT_REPO_URI = f"rsync://{RRDP_HOST}:{RSYNC_PORT}/custom/root/"
ROOT_MFT_URI = f"{ROOT_REPO_URI}root.mft"
CHILD_REPO_URI = f"rsync://{RRDP_HOST}:{RSYNC_PORT}/custom/child/"
CHILD_MFT_URI = f"{CHILD_REPO_URI}child.mft"
output = args.output.resolve()
if output.exists():
shutil.rmtree(output)
for directory in ("tal", "ta", "certs", "services", "keys"):
(output / directory).mkdir(parents=True)
root_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
child_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
root_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "#142 Custom RPKI Root")])
child_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "#142 Custom RPKI Child")])
root_cert = sign_cert(
root_name, root_name, root_key.public_key(), root_key, 0x14200001, True, root_key,
ip_ext=ip_resources([CHILD_PREFIX]),
as_ext=as_resources([TEST_ASN]), ca_repository=ROOT_REPO_URI, manifest_uri=ROOT_MFT_URI,
notify_uri=RRDP_URI,
)
child_cert = sign_cert(
child_name, root_name, child_key.public_key(), root_key, 0x14200002, True, child_key, root_key,
ip_ext=ip_resources([CHILD_PREFIX]), as_ext=as_resources([TEST_ASN]),
ca_repository=CHILD_REPO_URI, manifest_uri=CHILD_MFT_URI, notify_uri=RRDP_URI,
crl_uri=f"{ROOT_REPO_URI}root.crl", ca_issuer_uri=f"{ROOT_REPO_URI}child.cer",
)
bad_child_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "#142 Custom RPKI Overclaiming Child")])
bad_child_repo_uri = f"rsync://{RRDP_HOST}:{RSYNC_PORT}/custom/bad-child/"
bad_child_cert = sign_cert(
bad_child_name, root_name, child_key.public_key(), root_key, 0x14200003, True, child_key, root_key,
ip_ext=ip_resources([OUTSIDE_PREFIX]), as_ext=as_resources([TEST_ASN]),
ca_repository=bad_child_repo_uri, manifest_uri=f"{bad_child_repo_uri}bad-child.mft", notify_uri=RRDP_URI,
crl_uri=f"{ROOT_REPO_URI}root.crl", ca_issuer_uri=f"{ROOT_REPO_URI}child.cer",
)
root_crl = make_crl(root_name, root_key, 1)
child_crl = make_crl(child_name, child_key, 1)
tal = (
f"{TAL_URI}\n{TAL_HTTPS_URI}\n\n"
+ base64.b64encode(root_key.public_key().public_bytes(Encoding.DER, PublicFormat.SubjectPublicKeyInfo)).decode()
+ "\n"
).encode()
write(output / "tal/custom.tal", tal)
write(output / "ta/custom-ta.cer", root_cert.public_bytes(Encoding.DER))
with tempfile.TemporaryDirectory(prefix="custom-rpki-cms-") as temp:
work = Path(temp)
common = output / ".common"
common.mkdir()
write(common / "ta.cer", root_cert.public_bytes(Encoding.DER))
write(common / "child.cer", child_cert.public_bytes(Encoding.DER))
write(common / "bad-child.cer", bad_child_cert.public_bytes(Encoding.DER))
write(common / "root.crl", root_crl.public_bytes(Encoding.DER))
write(common / "child.crl", child_crl.public_bytes(Encoding.DER))
for name, prefix, max_length, before, after, certificate_prefix in (
("valid.roa", CHILD_PREFIX, 24, VALID_FROM + timedelta(days=31), VALID_TO - timedelta(days=31), None),
("valid2.roa", SECOND_PREFIX, 25, VALID_FROM + timedelta(days=31), VALID_TO - timedelta(days=31), None),
("expired.roa", CHILD_PREFIX, 24, VALID_FROM - timedelta(days=30), VALID_FROM - timedelta(days=1), None),
("max-length.roa", CHILD_PREFIX, 25, VALID_FROM + timedelta(days=31), VALID_TO - timedelta(days=31), None),
("max-length-invalid.roa", CHILD_PREFIX, 33, VALID_FROM + timedelta(days=31), VALID_TO - timedelta(days=31), None),
("roa-prefix-outside.roa", OUTSIDE_PREFIX, 24, VALID_FROM + timedelta(days=31), VALID_TO - timedelta(days=31), CHILD_PREFIX),
("out-of-resource.roa", OUTSIDE_PREFIX, 24, VALID_FROM + timedelta(days=31), VALID_TO - timedelta(days=31), None),
):
make_roa(
name, prefix, max_length, before, after, child_name, child_key, common, work,
certificate_prefix=certificate_prefix,
)
bad = bytearray((common / "valid.roa").read_bytes())
bad[-1] ^= 1
write(common / "bad-format.roa", bytes(bad))
root_mft = make_manifest(
"root", 1, [
("root.crl", (common / "root.crl").read_bytes()),
("child.cer", (common / "child.cer").read_bytes()),
], root_name, root_key, common, work,
)
write(common / "root.mft", root_mft)
bad_root_mft = make_manifest(
"root", 1, [
("root.crl", (common / "root.crl").read_bytes()),
("child.cer", (common / "child.cer").read_bytes()),
("bad-child.cer", (common / "bad-child.cer").read_bytes()),
], root_name, root_key, common, work,
)
write(common / "root.mft", root_mft)
case_specs = (
("baseline-v1", 1, ["child.crl", "valid.roa"]),
("sync-hash-mismatch", 1, ["child.crl", "valid.roa"]),
("baseline-v2", 2, ["child.crl", "valid.roa", "valid2.roa"]),
("validation-expired", 1, ["child.crl", "expired.roa"]),
("validation-max-length", 1, ["child.crl", "max-length.roa"]),
("validation-max-length-invalid", 1, ["child.crl", "max-length-invalid.roa"]),
("validation-roa-prefix-outside", 1, ["child.crl", "roa-prefix-outside.roa"]),
("validation-out-of-resource", 1, ["child.crl", "out-of-resource.roa"]),
("validation-ca-over-resource", 1, ["child.crl", "valid.roa"]),
("validation-nonstandard", 1, ["child.crl", "bad-format.roa"]),
)
child_mfts = {}
for name, number, names in case_specs:
child_mfts[name] = make_manifest(
name, number, [(item, (common / item).read_bytes()) for item in names],
child_name, child_key, common, work,
)
for name, number, names in case_specs:
change = None
if name == "baseline-v2":
change = delta("valid2.roa", common / "valid2.roa", common / "baseline-v2.mft")
build_case(
output, common, tal, root_cert.public_bytes(Encoding.DER), name, number, names,
child_mfts[name], change,
root_mft=bad_root_mft if name == "validation-ca-over-resource" else None,
extra_root_objects=[("bad-child.cer", (common / "bad-child.cer").read_bytes())]
if name == "validation-ca-over-resource" else None,
notification_hash_mismatch=name == "sync-hash-mismatch",
)
write_key(output / "keys/root.key", root_key)
write_key(output / "keys/child.key", child_key)
make_https_certificates(output)
shutil.rmtree(output / ".common")
(output / "FIXTURE-MANIFEST.txt").write_text(
"fixture_schema_version=1\nfixture_kind=custom-tal-fixed-local-rpki\n"
f"tal_uri={TAL_URI}\nrrdp_uri={RRDP_URI}\nrsync_module=custom\n"
f"rrdp_port={RRDP_PORT}\nrsync_port={RSYNC_PORT}\n"
"cases=baseline-v1,sync-hash-mismatch,baseline-v2,validation-expired,validation-max-length,validation-max-length-invalid,"
"validation-roa-prefix-outside,validation-out-of-resource,validation-ca-over-resource,validation-nonstandard\n",
encoding="utf-8",
)
print(f"generated custom fixture: {output}")
if __name__ == "__main__":
main()

36
deploy/docker-installer/custom-tal-patch/tools/serve_https.py
View File

@ -1,36 +0,0 @@
#!/usr/bin/env python3
"""Serve a fixture directory over HTTPS with request logging."""
from __future__ import annotations
import argparse
import http.server
import ssl
from pathlib import Path
class RequestHandler(http.server.SimpleHTTPRequestHandler):
def log_message(self, format: str, *args) -> None:
print("%s %s" % (self.log_date_time_string(), format % args), flush=True)
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--directory", type=Path, required=True)
parser.add_argument("--certfile", type=Path, required=True)
parser.add_argument("--keyfile", type=Path, required=True)
parser.add_argument("--host", default="0.0.0.0")
parser.add_argument("--port", type=int, default=18443)
args = parser.parse_args()
handler = lambda *handler_args, directory=str(args.directory): RequestHandler(
*handler_args, directory=directory
)
server = http.server.ThreadingHTTPServer((args.host, args.port), handler)
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
context.load_cert_chain(certfile=args.certfile, keyfile=args.keyfile)
server.socket = context.wrap_socket(server.socket, server_side=True)
print(f"https fixture server listening on {args.host}:{args.port} root={args.directory}", flush=True)
server.serve_forever()
if __name__ == "__main__":
main()

87
deploy/docker-installer/custom-tal-patch/tools/start_fixed_services.sh
View File

@ -1,87 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$BASH_SOURCE")" && pwd)"
FIXTURE_ROOT=""
CASE_NAME="baseline-v1"
RRDP_PORT="$(printenv RRDP_PORT 2>/dev/null || true)"
RSYNC_PORT="$(printenv RSYNC_PORT 2>/dev/null || true)"
PID_ROOT=""
[[ -n "$RRDP_PORT" ]] || RRDP_PORT=18443
[[ -n "$RSYNC_PORT" ]] || RSYNC_PORT=1873
usage() {
cat <<'USAGE'
Usage:
./start_fixed_services.sh --fixture-root <generated-fixtures> [--case NAME]
Cases: baseline-v1, sync-hash-mismatch, baseline-v2, validation-expired,
validation-max-length, validation-max-length-invalid,
validation-roa-prefix-outside, validation-out-of-resource,
validation-ca-over-resource, validation-nonstandard
Environment: RRDP_PORT (default 18443), RSYNC_PORT (default 1873)
USAGE
}
die() {
echo "error: $*" >&2
exit 2
}
while [[ $# -gt 0 ]]; do
case "$1" in
--fixture-root) FIXTURE_ROOT="$2"; shift 2 ;;
--case) CASE_NAME="$2"; shift 2 ;;
--pid-root) PID_ROOT="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) die "unknown option: $1" ;;
esac
done
[[ -n "$FIXTURE_ROOT" ]] || die "--fixture-root is required"
FIXTURE_ROOT="$(cd "$FIXTURE_ROOT" 2>/dev/null && pwd)" || die "fixture root not found"
CASE_ROOT="$FIXTURE_ROOT/cases/$CASE_NAME"
[[ -d "$CASE_ROOT/repository" && -d "$CASE_ROOT/http" ]] || die "fixture case not found: $CASE_NAME"
if [[ -z "$PID_ROOT" ]]; then
PID_ROOT="$FIXTURE_ROOT/services/$CASE_NAME"
fi
mkdir -p "$PID_ROOT"
[[ ! -f "$PID_ROOT/pids.env" ]] || die "services already appear to be running: $PID_ROOT/pids.env"
rm -f "$PID_ROOT/rsyncd.pid" "$PID_ROOT/rsyncd.lock"
RSYNCD_CONF="$PID_ROOT/rsyncd.conf"
RSYNC_UID="$(id -un)"
RSYNC_GID="$(id -gn)"
cat > "$RSYNCD_CONF" <<EOF
uid = $RSYNC_UID
gid = $RSYNC_GID
use chroot = no
read only = yes
port = $RSYNC_PORT
pid file = $PID_ROOT/rsyncd.pid
lock file = $PID_ROOT/rsyncd.lock
log file = $PID_ROOT/rsyncd.log
[custom]
path = $CASE_ROOT/repository
comment = #142 fixed custom RPKI repository
read only = yes
EOF
rsync --daemon --no-detach --config="$RSYNCD_CONF" >"$PID_ROOT/rsyncd.stdout.log" 2>&1 &
RSYNC_PID=$!
python3 "$SCRIPT_DIR/serve_https.py" \
--directory "$CASE_ROOT/http" \
--certfile "$FIXTURE_ROOT/certs/rrdp-server.pem" \
--keyfile "$FIXTURE_ROOT/certs/rrdp-server.key" \
--port "$RRDP_PORT" \
>"$PID_ROOT/https.log" 2>&1 &
HTTPS_PID=$!
printf 'case=%s\nrrdp_pid=%s\nrsync_pid=%s\nrrdp_port=%s\nrsync_port=%s\n' \
"$CASE_NAME" "$HTTPS_PID" "$RSYNC_PID" "$RRDP_PORT" "$RSYNC_PORT" > "$PID_ROOT/pids.env"
sleep 1
kill -0 "$HTTPS_PID" 2>/dev/null || die "HTTPS fixture server exited; see $PID_ROOT/https.log"
kill -0 "$RSYNC_PID" 2>/dev/null || die "rsync fixture server exited; see $PID_ROOT/rsyncd.stdout.log"
echo "fixed RRDP/rsync services started case=$CASE_NAME rrdp=$RRDP_PORT rsync=$RSYNC_PORT pid_root=$PID_ROOT"

23
deploy/docker-installer/custom-tal-patch/tools/stop_fixed_services.sh
View File

@ -1,23 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
PID_ROOT="$1"
[[ -n "$PID_ROOT" ]] || { echo "Usage: $0 <pid-root>" >&2; exit 2; }
[[ -f "$PID_ROOT/pids.env" ]] || { echo "services are not running: $PID_ROOT" >&2; exit 2; }
# shellcheck disable=SC1090
source "$PID_ROOT/pids.env"
service_pids=("${rrdp_pid:-}" "${rsync_pid:-}")
for pid in "${service_pids[@]}"; do
if [[ "$pid" =~ ^[0-9]+$ ]] && kill -0 "$pid" 2>/dev/null; then
kill "$pid" 2>/dev/null || true
fi
done
for pid in "${service_pids[@]}"; do
[[ "$pid" =~ ^[0-9]+$ ]] || continue
for _ in $(seq 1 50); do
kill -0 "$pid" 2>/dev/null || break
sleep 0.1
done
done
rm -f "$PID_ROOT/pids.env"
echo "fixed RRDP/rsync services stopped: $PID_ROOT"

194
deploy/docker-installer/docs/README.en.md
View File

@ -1,194 +0,0 @@
# ours RP Docker Installer Guide (`__PACKAGE_ARCH__` / `__PACKAGE_PLATFORM__`)
## Goal
This package deploys ours RP on Linux servers whose host architecture matches the packaged installer metadata, using Docker Compose and continuous all-five RIR validation.
The package includes four architecture-matched images: ours RP runtime, artifact metrics, Prometheus, and Grafana. Deployment does not need to pull application images on the target host. Runtime state, run artifacts, logs, Prometheus data and Grafana data are persisted through host bind mounts.
Every installer build rebuilds the ours RP runtime and artifact metrics images. The deliverable name is `ours-rp-installer-{arch}-{git8}-{YYYYMMDDTHHMMSSZ}.tar.gz`, and both ours RP image tags use the current source commit. `PACKAGE-MANIFEST.env` records the full commit, UTC build time, dirty state, and image archive hashes; `./scripts/status.sh --brief` displays that provenance after installation.
By default the host architecture must match `PACKAGE_ARCH` from `PACKAGE-MANIFEST.env`. If you intentionally want to run this package on a different host architecture through QEMU/binfmt, set:
```bash
ALLOW_CROSS_ARCH=1
```
## Compose Template Source
`compose/` is the single production Docker Compose template source. The Docker installer build script copies this directory directly. Do not maintain a second copy of the Compose, Prometheus, or Grafana dashboard files elsewhere.
The legacy direct Compose deployment entry point has been removed. For either amd64 or arm64, build the matching installer package with `scripts/docker/build_docker_installer_package.sh`, then use `scripts/install.sh`, `scripts/start.sh`, and `scripts/status.sh` as described here.
## Quick Start
```bash
tar -xzf ours-rp-installer-__PACKAGE_ARCH__-*.tar.gz
cd ours-rp-installer-__PACKAGE_ARCH__-*
./scripts/install.sh
cp .env.example .env # install.sh creates .env automatically if missing
vim .env
./scripts/start.sh
./scripts/status.sh
```
Defaults:
- `PACKAGE_ARCH=__PACKAGE_ARCH__`
- `PACKAGE_PLATFORM=__PACKAGE_PLATFORM__`
- `RIRS=afrinic,apnic,arin,lacnic,ripe`
- `MAX_RUNS=-1`
- `INTERVAL_SECS=600`
- `TAL_INPUT_MODE=file-live-ta`
- `RESOURCE_VALIDATION_MODE=validation-update-03`
- `LIVE_TA_REFRESH_BEFORE_SNAPSHOT=1`
- `PERIODIC_SNAPSHOT_RESET=0`
- `PERIODIC_SNAPSHOT_MAX_DELTAS=100`
- `HOST_DATA_DIR=__HOST_DATA_DIR__`
- `SOAK_RESTART_POLICY=unless-stopped`
- `RPKI_IMAGE=__RUNTIME_IMAGE__`
- `METRICS_IMAGE=__METRICS_IMAGE__`
- `METRICS_PLATFORM=__PACKAGE_PLATFORM__`
- `MONITOR_PLATFORM=__PACKAGE_PLATFORM__`
- `ALLOW_CROSS_ARCH=0`
- `RTR_REPORT_DIR=__HOST_DATA_DIR__/empty-rtr-report`, which mounts an empty fallback directory by default; after deploying a separate RTR service, register its same-host report directory through `scripts/register_rtr_monitor.sh`
## First Start Semantics
If there is no successful run under `HOST_DATA_DIR/runs`, `start.sh` starts the core `ours-rp-soak` service first and waits for the first snapshot to succeed before starting metrics, Prometheus and Grafana.
Container/image mapping:
- `ours-rp-soak` uses `RPKI_IMAGE`
- `artifact-metrics` uses `METRICS_IMAGE`
- `prometheus` / `grafana` use their monitor images
The first snapshot refreshes live TA certificates before starting the RP process.
## Resource Validation Mode
New knob:
```bash
RESOURCE_VALIDATION_MODE=validation-update-03
```
Supported values:
- `validation-update-03`: default behavior, using the current validation-update draft semantics;
- `rfc6487`: switch back to the original RFC 6487 resource containment behavior.
The installer / soak runner passes `--resource-validation-mode` to every `rpki` child start.
When an upgraded deployment reuses an older `.env` that does not contain this key, the installer appends the new default `validation-update-03` automatically.
## Architecture Guardrails
The key scripts read:
- `PACKAGE_ARCH` / `PACKAGE_PLATFORM` from `.env`
- `PACKAGE-MANIFEST.env`
- the current host `uname -m`
Default behavior:
1. matching host and package architectures: run natively;
2. mismatched host and package architectures: fail with an explicit error;
3. only when `ALLOW_CROSS_ARCH=1` is set do the scripts attempt to enable matching `binfmt/qemu`.
That means an `arm64` package on an `x86_64` host does not silently switch to emulation by default.
## Ports
Default ports:
- metrics: `http://<host>:9556/metrics`
- Prometheus: `http://<host>:9090`
- Grafana: `http://<host>:3000`
Grafana credentials come from `.env`:
```bash
GRAFANA_ADMIN_USER=admin
GRAFANA_ADMIN_PASSWORD=admin
```
Change the password and restrict public access for production deployments.
## Optional RTR report metrics input
If a separately deployed RTR service on the same host continuously writes `rtr-source-*`, `rtr-runtime-*`, and `rtr-clients-*` JSON reports, run:
```bash
./scripts/register_rtr_monitor.sh --report-dir /root/rpki/report
```
The command accepts only an absolute path on the same host, validates the candidate Compose mount, and atomically updates `.env`. It recreates only `artifact-metrics`, Prometheus, and Grafana; it does not restart or recreate `ours-rp-soak`. The `artifact-metrics` container mounts the directory read-only and reads it through `RPKI_METRICS_RTR_REPORT_DIR`, producing `ours_rp_rtr_*` metrics. By default, `RTR_REPORT_DIR` points at an empty fallback directory under the installer data root, so the metrics container also starts cleanly without an RTR service.
## Data Directory
Default host directory:
```text
__HOST_DATA_DIR__/
state/
runs/
logs/
tmp/
prometheus/
grafana/
```
Each `runs/run_XXXX/` directory contains `report.json`, `result.ccr`, `input.cir`, `vrps.csv`, `vaps.csv`, `stage-timing.json`, logs and metadata.
## Periodic Snapshot Reset
New knobs:
```bash
PERIODIC_SNAPSHOT_RESET=0
PERIODIC_SNAPSHOT_MAX_DELTAS=100
```
Semantics:
- disabled by default, keeping previous behavior unchanged;
- when enabled, one successful snapshot is followed by at most `N` successful delta runs;
- after the threshold is reached, the next run is forced to snapshot;
- before that forced snapshot, only the active `state/db` is reset, while `runs/`, `logs/`, `state/rsync-mirror`, `.env`, and Prometheus/Grafana data are preserved;
- the counter is persisted independently in `HOST_DATA_DIR/state/run-lifecycle-state.json`, so it does not depend on retained `runs/` history;
- if that lifecycle file is corrupt, it is backed up as `run-lifecycle-state.json.corrupt.<timestamp>.<pid>` before best-effort bootstrap from retained runs;
- after a successful forced snapshot, the old DB staging is deleted so disk usage does not keep growing elsewhere.
Check the latest `run-meta.json` for:
- `sync_mode`
- `snapshot_reason`
- `periodic_snapshot_delta_count`
- `periodic_snapshot_forced`
- `reset_db_cleanup_status`
## Common Commands
```bash
./scripts/status.sh
./scripts/logs.sh ours-rp-soak --tail 200
./scripts/restart.sh
./scripts/stop.sh
./scripts/cleanup.sh --keep-runs 100 --execute
./scripts/uninstall.sh
```
For finite acceptance tests, for example `MAX_RUNS=3`, also set:
```bash
SOAK_RESTART_POLICY=no
```
Otherwise Compose `unless-stopped` will restart the container after it exits successfully.
`uninstall.sh` keeps data by default. Use the following only when you really want to delete `HOST_DATA_DIR`:
```bash
./scripts/uninstall.sh --purge-data
```

196
deploy/docker-installer/docs/README.zh-CN.md
View File

@ -1,196 +0,0 @@
# ours RP Docker 安装包使用说明（`__PACKAGE_ARCH__` / `__PACKAGE_PLATFORM__`）
## 目标
本安装包用于在目标架构与包元数据匹配的 Linux 服务器上，通过 Docker Compose 部署 ours RP，并持续运行 all5 RIR 同步验证任务。
安装包内置与包架构一致的四类镜像：ours RP runtime、artifact metrics、Prometheus、Grafana。部署时不需要现场拉取应用镜像。运行产物、状态数据库、日志、Prometheus 和 Grafana 数据均通过宿主机目录挂载保存。
每次构建安装包都会重新构建 ours RP runtime 与 artifact metrics 镜像。交付包命名为 `ours-rp-installer-{arch}-{git8}-{YYYYMMDDTHHMMSSZ}.tar.gz`，两类 ours RP 镜像分别使用当前源码 commit 的 tag。完整 commit、UTC 构建时间、dirty 状态和镜像归档 hash 保存在 `PACKAGE-MANIFEST.env`，可通过 `./scripts/status.sh --brief` 查看。
默认情况下，宿主架构必须与 `PACKAGE-MANIFEST.env` 中的 `PACKAGE_ARCH` 匹配；若确需在异构主机上通过 QEMU/binfmt 运行，必须显式设置：
```bash
ALLOW_CROSS_ARCH=1
```
## Compose 模板来源
`compose/` 是唯一的生产 Docker Compose 模板来源，Docker 安装包构建脚本会直接复制该目录。不要在其他目录维护第二份 compose、Prometheus 或 Grafana dashboard 文件。
旧的直接 Compose 部署入口已经移除。需要部署 amd64 或 arm64 时，统一先用 `scripts/docker/build_docker_installer_package.sh` 构建对应架构安装包，再按本说明执行 `scripts/install.sh`、`scripts/start.sh` 和 `scripts/status.sh`。
## 快速开始
```bash
tar -xzf ours-rp-installer-__PACKAGE_ARCH__-*.tar.gz
cd ours-rp-installer-__PACKAGE_ARCH__-*
./scripts/install.sh
cp .env.example .env # 如 install.sh 已自动创建，可直接编辑现有 .env
vim .env
./scripts/start.sh
./scripts/status.sh
```
默认配置：
- `PACKAGE_ARCH=__PACKAGE_ARCH__`
- `PACKAGE_PLATFORM=__PACKAGE_PLATFORM__`
- `RIRS=afrinic,apnic,arin,lacnic,ripe`
- `MAX_RUNS=-1`
- `INTERVAL_SECS=600`
- `TAL_INPUT_MODE=file-live-ta`
- `RESOURCE_VALIDATION_MODE=validation-update-03`
- `LIVE_TA_REFRESH_BEFORE_SNAPSHOT=1`
- `PERIODIC_SNAPSHOT_RESET=0`
- `PERIODIC_SNAPSHOT_MAX_DELTAS=100`
- `HOST_DATA_DIR=__HOST_DATA_DIR__`
- `SOAK_RESTART_POLICY=unless-stopped`
- `RPKI_IMAGE=__RUNTIME_IMAGE__`
- `METRICS_IMAGE=__METRICS_IMAGE__`
- `METRICS_PLATFORM=__PACKAGE_PLATFORM__`
- `MONITOR_PLATFORM=__PACKAGE_PLATFORM__`
- `ALLOW_CROSS_ARCH=0`
- `RTR_REPORT_DIR=__HOST_DATA_DIR__/empty-rtr-report`，默认挂载空目录；独立 RTR 服务部署完成后，使用 `scripts/register_rtr_monitor.sh` 注册同机 report 目录
## 首次启动语义
如果 `HOST_DATA_DIR/runs` 下没有成功 run，`start.sh` 会先启动核心 `ours-rp-soak`，等待第一轮 snapshot 成功后再启动 metrics、Prometheus 和 Grafana。
容器镜像分工：
- `ours-rp-soak` 使用 `RPKI_IMAGE`
- `artifact-metrics` 使用 `METRICS_IMAGE`
- `prometheus` / `grafana` 使用各自 monitor image
第一轮 snapshot 会先拉取 live TA，避免 clean state 使用旧 fixture TA。
## 资源验证模式
新增配置：
```bash
RESOURCE_VALIDATION_MODE=validation-update-03
```
可选值：
- `validation-update-03`：默认，按当前 validation update draft 语义运行；
- `rfc6487`：切换为 RFC 6487 原始资源包含判定语义。
installer / soak runner 每次启动 `rpki` 子进程时都会显式传入 `--resource-validation-mode`。
如果升级时复用旧 `.env` 且缺少该变量，installer 会自动补入默认值 `validation-update-03`，不会破坏旧 `.env` 复用流程。
## 架构检查
关键脚本会读取：
- `.env` 中的 `PACKAGE_ARCH` / `PACKAGE_PLATFORM`
- `PACKAGE-MANIFEST.env`
- 当前宿主 `uname -m`
默认行为：
1. 宿主与包架构匹配：直接运行；
2. 宿主与包架构不匹配：明确报错并停止；
3. 仅当 `ALLOW_CROSS_ARCH=1` 时，脚本才会尝试启用对应架构的 `binfmt/qemu`。
因此在 `x86_64` 主机上运行 `arm64` 包，不会默认静默进入模拟执行。
## 访问端口
默认端口：
- metrics: `http://<host>:9556/metrics`
- Prometheus: `http://<host>:9090`
- Grafana: `http://<host>:3000`
Grafana 默认账号密码来自 `.env`：
```bash
GRAFANA_ADMIN_USER=admin
GRAFANA_ADMIN_PASSWORD=admin
```
生产部署时应修改密码并限制外部访问。
## 可选 RTR report 监控接入
如果同一台机器上已经有独立部署的 RTR 服务，并且它持续输出 `rtr-source-*`、`rtr-runtime-*`、`rtr-clients-*` JSON report，执行：
```bash
./scripts/register_rtr_monitor.sh --report-dir /root/rpki/report
```
该命令只接受本机绝对路径，会先校验候选 Compose 挂载，再原子更新 `.env`。随后只重建 `artifact-metrics`、Prometheus 和 Grafana，不重启或重建 `ours-rp-soak`。`artifact-metrics` 以只读方式挂载该目录，并通过 `RPKI_METRICS_RTR_REPORT_DIR` 读取 report，输出 `ours_rp_rtr_*` 指标。默认 `RTR_REPORT_DIR` 指向安装包数据目录下的空 fallback 目录，因此未接入 RTR 服务时 metrics 容器也能稳定启动。
## 数据目录
默认宿主机目录：
```text
__HOST_DATA_DIR__/
state/
runs/
logs/
tmp/
prometheus/
grafana/
```
`runs/run_XXXX/` 中包含每轮 `report.json`、`result.ccr`、`input.cir`、`vrps.csv`、`vaps.csv`、`stage-timing.json`、日志和元数据。
## 定期 snapshot reset
新增配置：
```bash
PERIODIC_SNAPSHOT_RESET=0
PERIODIC_SNAPSHOT_MAX_DELTAS=100
```
语义：
- 默认关闭，行为与旧版本一致；
- 开启后，一次成功 snapshot 后最多连续执行 `N` 个成功 delta；
- 达到阈值后，下一轮强制跑 snapshot；
- 强制 snapshot 前只重置 active `state/db`，保留 `runs/`、`logs/`、`state/rsync-mirror`、`.env`、Prometheus/Grafana 数据；
- 周期计数保存在独立 lifecycle 文件 `HOST_DATA_DIR/state/run-lifecycle-state.json`，不依赖 `runs/` 保留窗口；
- lifecycle 文件损坏时会先备份为 `run-lifecycle-state.json.corrupt.<timestamp>.<pid>`，再从当前保留 run 尽力 bootstrap；
- 强制 snapshot 成功后旧 DB staging 会被删除，避免磁盘只是换目录继续增长。
可通过最新 `run-meta.json` 中的以下字段确认：
- `sync_mode`
- `snapshot_reason`
- `periodic_snapshot_delta_count`
- `periodic_snapshot_forced`
- `reset_db_cleanup_status`
## 常用命令
```bash
./scripts/status.sh
./scripts/logs.sh ours-rp-soak --tail 200
./scripts/restart.sh
./scripts/stop.sh
./scripts/cleanup.sh --keep-runs 100 --execute
./scripts/uninstall.sh
```
如果做有限轮次验收，例如 `MAX_RUNS=3`，建议同时设置：
```bash
SOAK_RESTART_POLICY=no
```
否则 Compose 的 `unless-stopped` 策略会在容器正常退出后再次拉起下一轮。
`uninstall.sh` 默认不删除数据。只有显式执行：
```bash
./scripts/uninstall.sh --purge-data
```
才会删除 `HOST_DATA_DIR`。

137
deploy/docker-installer/docs/operations.en.md
View File

@ -1,137 +0,0 @@
# Operations Guide (`__PACKAGE_ARCH__`)
## Install
```bash
./scripts/install.sh
```
The installer is idempotent:
- existing `.env` is kept;
- existing Docker/Compose installation is reused;
- repeated loading of packaged runtime, metrics, Prometheus and Grafana images for the package architecture is safe;
- existing data directory is reused;
- matching host/package architecture is required by default.
## Start
```bash
./scripts/start.sh
```
Start without waiting for the first snapshot:
```bash
./scripts/start.sh --no-wait-first-run
```
If you intentionally want to run this package on a different host architecture through QEMU/binfmt, first set:
```bash
ALLOW_CROSS_ARCH=1
```
Then rerun `./scripts/install.sh` or `./scripts/self-check.sh`, and only then will the scripts attempt to enable the matching binfmt handler.
## Stop and Restart
```bash
./scripts/stop.sh
./scripts/restart.sh
```
## Status Checks
```bash
./scripts/status.sh
./scripts/self-check.sh
```
Important checks:
- Docker/Compose availability;
- runtime, metrics, Prometheus and Grafana images exist;
- `HOST_DATA_DIR` is writable;
- Compose config is valid;
- latest run status;
- metrics, Prometheus and Grafana endpoints;
- whether `host_arch`, `package_arch`, and `arch_mode` match expectations.
`status.sh` also prints:
- `package_arch`
- `package_platform`
- `runtime_image`
- `metrics_image`
- `allow_cross_arch`
- `periodic_snapshot_reset`
- `periodic_snapshot_max_deltas`
- `rtr_report_dir`
- `rtr_report_container_dir`
To verify RTR report ingestion:
```bash
./scripts/register_rtr_monitor.sh --report-dir /root/rpki/report --dry-run
./scripts/register_rtr_monitor.sh --report-dir /root/rpki/report
./scripts/status.sh
curl -s "http://127.0.0.1:${METRICS_PORT:-9556}/metrics" | grep '^ours_rp_rtr_' | head
```
Registration accepts only an absolute directory on the same host. The command retains a timestamped `.env` backup and updates only the metrics, Prometheus, and Grafana sidecars; the `ours-rp-soak` container ID must remain unchanged.
## Upgrade
Extract the new package into a new directory and explicitly reuse the existing `.env` through the upgrade script:
```bash
./scripts/upgrade.sh --reuse-env-from /path/to/old-installer/.env
```
If the new package directory already has a `.env`, the upgrade script keeps it.
If an older `.env` does not contain `METRICS_IMAGE` or `METRICS_PLATFORM`, the install/upgrade flow backfills package-matched defaults automatically.
Upgrade does not delete:
- `runs/`
- `logs/`
- `state/rsync-mirror`
- runtime configuration referenced by `.env`
- Prometheus / Grafana data
To validate periodic forced snapshot behavior, temporarily set:
```bash
PERIODIC_SNAPSHOT_RESET=1
PERIODIC_SNAPSHOT_MAX_DELTAS=2
```
Then confirm the latest `run-meta.json` contains:
```bash
snapshot_reason=periodic_snapshot_delta_limit
```
And inspect the independent lifecycle state:
```bash
jq '{last_run,last_success_snapshot,successful_deltas_since_snapshot,state_health}' \
"${HOST_DATA_DIR}/state/run-lifecycle-state.json"
```
After validation, restore:
```bash
PERIODIC_SNAPSHOT_MAX_DELTAS=100
```
## Cleanup
```bash
./scripts/cleanup.sh --keep-runs 100
./scripts/cleanup.sh --keep-runs 100 --execute
```
Cleanup is dry-run by default. Add `--execute` after reviewing the output.

137
deploy/docker-installer/docs/operations.zh-CN.md
View File

@ -1,137 +0,0 @@
# 运维手册（`__PACKAGE_ARCH__`）
## 安装
```bash
./scripts/install.sh
```
安装脚本是幂等的：
- 已有 `.env` 不覆盖；
- 已安装 Docker/Compose 则跳过；
- 包内 runtime、metrics、Prometheus、Grafana 对应架构镜像重复加载是安全的；
- 数据目录已存在则复用；
- 默认要求宿主与包架构匹配。
## 启动
```bash
./scripts/start.sh
```
如需后台启动后不等待首轮 snapshot：
```bash
./scripts/start.sh --no-wait-first-run
```
如需显式允许异构主机通过 QEMU/binfmt 跑包，先在 `.env` 中设置：
```bash
ALLOW_CROSS_ARCH=1
```
然后重新执行 `./scripts/install.sh` 或 `./scripts/self-check.sh`，脚本才会尝试启用对应 binfmt。
## 停止和重启
```bash
./scripts/stop.sh
./scripts/restart.sh
```
## 状态检查
```bash
./scripts/status.sh
./scripts/self-check.sh
```
重点检查项：
- Docker/Compose 可用；
- runtime、metrics、Prometheus、Grafana 镜像存在；
- `HOST_DATA_DIR` 可写；
- Compose 配置合法；
- 最新 run 状态；
- metrics、Prometheus、Grafana endpoint；
- `host_arch`、`package_arch`、`arch_mode` 是否符合预期。
`status.sh` 还会显示：
- `package_arch`
- `package_platform`
- `runtime_image`
- `metrics_image`
- `allow_cross_arch`
- `periodic_snapshot_reset`
- `periodic_snapshot_max_deltas`
- `rtr_report_dir`
- `rtr_report_container_dir`
如需确认 RTR report 已接入：
```bash
./scripts/register_rtr_monitor.sh --report-dir /root/rpki/report --dry-run
./scripts/register_rtr_monitor.sh --report-dir /root/rpki/report
./scripts/status.sh
curl -s "http://127.0.0.1:${METRICS_PORT:-9556}/metrics" | grep '^ours_rp_rtr_' | head
```
注册只支持同机的绝对目录。注册命令会保留一份带时间戳的 `.env` 备份，并且只更新 metrics、Prometheus、Grafana 三个旁路容器；`ours-rp-soak` 的容器 ID 应保持不变。
## 升级
把新安装包解压到新目录后，推荐通过升级脚本显式复用旧 `.env`：
```bash
./scripts/upgrade.sh --reuse-env-from /path/to/old-installer/.env
```
如果新目录已经存在 `.env`，升级脚本会保留它，不覆盖。
如果旧 `.env` 缺失 `METRICS_IMAGE` 或 `METRICS_PLATFORM`，升级/安装链路会按当前包架构自动补齐默认值。
升级不会删除以下数据：
- `runs/`
- `logs/`
- `state/rsync-mirror`
- `.env` 对应的运行配置
- Prometheus / Grafana 数据
验证定期 forced snapshot 时，可临时设置：
```bash
PERIODIC_SNAPSHOT_RESET=1
PERIODIC_SNAPSHOT_MAX_DELTAS=2
```
然后检查最新 `run-meta.json` 应出现：
```bash
snapshot_reason=periodic_snapshot_delta_limit
```
并检查独立 lifecycle 状态：
```bash
jq '{last_run,last_success_snapshot,successful_deltas_since_snapshot,state_health}' \
"${HOST_DATA_DIR}/state/run-lifecycle-state.json"
```
验证完成后恢复：
```bash
PERIODIC_SNAPSHOT_MAX_DELTAS=100
```
## 清理
```bash
./scripts/cleanup.sh --keep-runs 100
./scripts/cleanup.sh --keep-runs 100 --execute
```
默认 dry-run，确认后加 `--execute`。

142
deploy/docker-installer/docs/troubleshooting.en.md
View File

@ -1,142 +0,0 @@
# Troubleshooting (`__PACKAGE_ARCH__` / `__PACKAGE_PLATFORM__`)
## Docker or Compose Is Unavailable
Run:
```bash
docker version
docker compose version
```
If missing, run:
```bash
./scripts/install.sh
```
## Package Architecture Mismatch
If `status.sh`, `install.sh`, or `self-check.sh` reports:
```text
package architecture mismatch
```
the host `uname -m` does not match the packaged `PACKAGE_ARCH=__PACKAGE_ARCH__`. The default behavior is to fail clearly rather than silently switching to emulation.
Only if you intentionally accept QEMU/binfmt cross-architecture execution should you set:
```bash
ALLOW_CROSS_ARCH=1
```
Then rerun install or self-check.
## `__PACKAGE_ARCH__` Image Cannot Run
Running `__PACKAGE_PLATFORM__` images on a different host architecture requires binfmt/qemu. The installer only attempts this automatically when `ALLOW_CROSS_ARCH=1`; you can also enable it manually:
```bash
docker run --rm --privileged tonistiigi/binfmt --install __PACKAGE_ARCH__
docker run --rm --platform __PACKAGE_PLATFORM__ debian:bookworm-slim uname -m
```
For `arm64` packages the expected output is `aarch64`; for `amd64` packages the expected output is `x86_64`.
To confirm the metrics image is also loaded correctly:
```bash
docker image inspect "$(grep '^METRICS_IMAGE=' .env | cut -d= -f2-)"
docker run --rm --platform __PACKAGE_PLATFORM__ \
"$(grep '^METRICS_IMAGE=' .env | cut -d= -f2-)" \
/opt/ours-rp/bin/rpki_artifact_metrics --help
```
## RTR report metrics are empty
First check registration and the report directory:
```bash
./scripts/status.sh
./scripts/register_rtr_monitor.sh --report-dir /root/rpki/report --dry-run
```
Check the metrics container mount:
```bash
docker inspect "$(grep '^COMPOSE_PROJECT_NAME=' .env | cut -d= -f2-)-artifact-metrics" \
| jq '.[0].Mounts[] | select(.Destination=="/var/lib/ours-rp/rtr-report")'
```
## First Snapshot Times Out
All-five snapshot can be slow, especially under QEMU. Increase timeout:
```bash
./scripts/start.sh --timeout-secs 14400
```
## Output Counts Are Too Low
Check:
```bash
grep LIVE_TA_REFRESH_BEFORE_SNAPSHOT .env
ls -l __HOST_DATA_DIR__/state/live-ta
tail -100 __HOST_DATA_DIR__/logs/live-ta-refresh-*.log
```
In `file-live-ta` mode, snapshot should wait until live TA refresh succeeds.
## Grafana Login Fails
Check `.env`:
```bash
GRAFANA_ADMIN_USER=admin
GRAFANA_ADMIN_PASSWORD=admin
```
If Grafana has already started, changing `.env` may not reset the existing Grafana database. Stop services and back up/clean `${HOST_DATA_DIR}/grafana` if needed.
## A Finite Acceptance Test Starts an Extra Run
If `.env` sets a finite `MAX_RUNS=3` while `SOAK_RESTART_POLICY=unless-stopped`, Docker Compose restarts the soak container after it exits successfully.
For finite tests, set:
```bash
SOAK_RESTART_POLICY=no
```
## How to Confirm a Periodic Forced Snapshot
Check the latest run metadata:
```bash
latest="$(find ${HOST_DATA_DIR}/runs -maxdepth 1 -type d -name 'run_*' | sort | tail -1)"
jq '{run_id,sync_mode,snapshot_reason,periodic_snapshot_delta_count,periodic_snapshot_forced,reset_db_cleanup_status}' "$latest/run-meta.json"
```
For a threshold-triggered reset you should see:
- `sync_mode: "snapshot"`
- `snapshot_reason: "periodic_snapshot_delta_limit"`
- `periodic_snapshot_forced: true`
To confirm delta counting still advances after retained runs are pruned, also inspect:
```bash
jq '{last_success_snapshot,successful_deltas_since_snapshot,recent_runs_count:(.recent_runs|length)}' \
"${HOST_DATA_DIR}/state/run-lifecycle-state.json"
```
## Lifecycle State File Is Corrupt
The script backs up the corrupt file before best-effort bootstrap from retained runs:
```bash
ls -1 "${HOST_DATA_DIR}/state"/run-lifecycle-state.json.corrupt.*
jq . "${HOST_DATA_DIR}/state/run-lifecycle-state.json"
```

142
deploy/docker-installer/docs/troubleshooting.zh-CN.md
View File

@ -1,142 +0,0 @@
# 故障排查（`__PACKAGE_ARCH__` / `__PACKAGE_PLATFORM__`）
## Docker 或 Compose 不可用
执行：
```bash
docker version
docker compose version
```
如果缺失，重新执行：
```bash
./scripts/install.sh
```
## 架构不匹配报错
当 `status.sh` / `install.sh` / `self-check.sh` 报出：
```text
package architecture mismatch
```
说明宿主 `uname -m` 与包的 `PACKAGE_ARCH=__PACKAGE_ARCH__` 不一致。默认行为就是失败而不是静默模拟运行。
只有在你明确接受 QEMU/binfmt 跨架构运行时，才设置：
```bash
ALLOW_CROSS_ARCH=1
```
然后重新执行安装或自检。
## `__PACKAGE_ARCH__` 镜像无法运行
在异构主机上运行 `__PACKAGE_PLATFORM__` 镜像需要 binfmt/qemu。安装脚本只有在 `ALLOW_CROSS_ARCH=1` 时才会自动尝试启用；也可以手动执行：
```bash
docker run --rm --privileged tonistiigi/binfmt --install __PACKAGE_ARCH__
docker run --rm --platform __PACKAGE_PLATFORM__ debian:bookworm-slim uname -m
```
对于 `arm64` 包，预期输出 `aarch64`；对于 `amd64` 包，预期输出 `x86_64`。
如需进一步确认 metrics 镜像也已正确加载：
```bash
docker image inspect "$(grep '^METRICS_IMAGE=' .env | cut -d= -f2-)"
docker run --rm --platform __PACKAGE_PLATFORM__ \
"$(grep '^METRICS_IMAGE=' .env | cut -d= -f2-)" \
/opt/ours-rp/bin/rpki_artifact_metrics --help
```
## RTR report 指标为空
先检查注册状态和 report 目录：
```bash
./scripts/status.sh
./scripts/register_rtr_monitor.sh --report-dir /root/rpki/report --dry-run
```
检查 metrics 容器是否挂载该目录：
```bash
docker inspect "$(grep '^COMPOSE_PROJECT_NAME=' .env | cut -d= -f2-)-artifact-metrics" \
| jq '.[0].Mounts[] | select(.Destination=="/var/lib/ours-rp/rtr-report")'
```
## 首轮 snapshot 超时
all5 snapshot 可能很慢，尤其在 QEMU 环境。可以提高超时：
```bash
./scripts/start.sh --timeout-secs 14400
```
## 产物数量异常偏低
检查：
```bash
grep LIVE_TA_REFRESH_BEFORE_SNAPSHOT .env
ls -l __HOST_DATA_DIR__/state/live-ta
tail -100 __HOST_DATA_DIR__/logs/live-ta-refresh-*.log
```
`file-live-ta` 模式下，snapshot 应等待 live TA 成功刷新。
## Grafana 无法登录
确认 `.env` 中：
```bash
GRAFANA_ADMIN_USER=admin
GRAFANA_ADMIN_PASSWORD=admin
```
如果曾经启动过 Grafana，修改 `.env` 不一定重置已有 Grafana 数据库账号。可以停止服务后按需备份并清理 `${HOST_DATA_DIR}/grafana`。
## 有限轮次验收后又多跑了一轮
如果 `.env` 中设置了 `MAX_RUNS=3` 这类有限轮次，同时 `SOAK_RESTART_POLICY=unless-stopped`，Docker Compose 会在 soak 容器正常退出后重新启动容器。
有限验收建议设置：
```bash
SOAK_RESTART_POLICY=no
```
## 如何确认触发了定期 forced snapshot
检查最新 run metadata：
```bash
latest="$(find ${HOST_DATA_DIR}/runs -maxdepth 1 -type d -name 'run_*' | sort | tail -1)"
jq '{run_id,sync_mode,snapshot_reason,periodic_snapshot_delta_count,periodic_snapshot_forced,reset_db_cleanup_status}' "$latest/run-meta.json"
```
阈值触发时应看到：
- `sync_mode: "snapshot"`
- `snapshot_reason: "periodic_snapshot_delta_limit"`
- `periodic_snapshot_forced: true`
如需确认 retain 裁剪后仍在累计 delta，可继续检查：
```bash
jq '{last_success_snapshot,successful_deltas_since_snapshot,recent_runs_count:(.recent_runs|length)}' \
"${HOST_DATA_DIR}/state/run-lifecycle-state.json"
```
## Lifecycle 状态文件损坏
脚本会先备份损坏文件，再从当前保留的 `runs/` 尽力 bootstrap：
```bash
ls -1 "${HOST_DATA_DIR}/state"/run-lifecycle-state.json.corrupt.*
jq . "${HOST_DATA_DIR}/state/run-lifecycle-state.json"
```

56
deploy/docker-installer/scripts/cleanup.sh
View File

@ -1,56 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=common.sh
source "$SCRIPT_DIR/common.sh"
DRY_RUN=1
KEEP_RUNS=""
usage() {
cat <<'USAGE'
Usage: ./scripts/cleanup.sh [--execute] [--keep-runs N]
By default this is a dry-run. It removes old run_* directories beyond KEEP_RUNS
and clears tmp contents.
USAGE
}
while [[ $# -gt 0 ]]; do
case "$1" in
--execute)
DRY_RUN=0
shift
;;
--keep-runs)
KEEP_RUNS="$2"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
die "unknown option: $1"
;;
esac
done
load_env
keep="${KEEP_RUNS:-${RETAIN_RUNS:-100}}"
mapfile -t runs < <(find "$HOST_DATA_DIR/runs" -maxdepth 1 -type d -name 'run_*' 2>/dev/null | sort)
delete_count=$(( ${#runs[@]} - keep ))
if (( delete_count > 0 )); then
for ((i=0; i<delete_count; i++)); do
if [[ "$DRY_RUN" == "1" ]]; then
echo "DRY-RUN rm -rf ${runs[$i]}"
else
rm -rf "${runs[$i]}"
fi
done
fi
if [[ "$DRY_RUN" == "1" ]]; then
echo "DRY-RUN rm -rf $HOST_DATA_DIR/tmp/*"
else
find "$HOST_DATA_DIR/tmp" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
fi
df -h "$HOST_DATA_DIR" 2>/dev/null || true

597
deploy/docker-installer/scripts/common.sh
View File

@ -1,597 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
INSTALLER_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
ENV_FILE="${ENV_FILE:-$INSTALLER_ROOT/.env}"
ENV_EXAMPLE="$INSTALLER_ROOT/.env.example"
COMPOSE_FILE="$INSTALLER_ROOT/compose/docker-compose.yml"
MANIFEST_FILE="${MANIFEST_FILE:-$INSTALLER_ROOT/PACKAGE-MANIFEST.env}"
log() {
printf '[ours-rp-installer] %s\n' "$*"
}
warn() {
printf '[ours-rp-installer][WARN] %s\n' "$*" >&2
}
die() {
printf '[ours-rp-installer][ERROR] %s\n' "$*" >&2
exit 1
}
require_cmd() {
command -v "$1" >/dev/null 2>&1 || die "missing command: $1"
}
normalize_arch() {
case "$1" in
amd64|x86_64|linux/amd64)
printf 'amd64\n'
;;
arm64|aarch64|linux/arm64)
printf 'arm64\n'
;;
*)
return 1
;;
esac
}
platform_for_arch() {
printf 'linux/%s\n' "$1"
}
env_file_has_key() {
local env_path="$1"
local key="$2"
[[ -f "$env_path" ]] && grep -Eq "^${key}=" "$env_path"
}
persist_env_default() {
local env_path="$1"
local key="$2"
local value="$3"
local tmp_env
if env_file_has_key "$env_path" "$key"; then
return 0
fi
tmp_env="$(mktemp)"
cat "$env_path" > "$tmp_env"
printf '%s=%s\n' "$key" "$value" >> "$tmp_env"
mv "$tmp_env" "$env_path"
}
env_flag_enabled() {
case "${1:-0}" in
1|true|TRUE|yes|YES|on|ON)
return 0
;;
*)
return 1
;;
esac
}
detect_host_arch() {
local raw_arch
raw_arch="$(uname -m)"
normalize_arch "$raw_arch" || die "unsupported host architecture: $raw_arch"
}
load_manifest() {
MANIFEST_PACKAGE_ARCH_RAW=""
MANIFEST_PACKAGE_PLATFORM_RAW=""
if [[ -f "$MANIFEST_FILE" ]]; then
set -a
# shellcheck disable=SC1090
source "$MANIFEST_FILE"
set +a
MANIFEST_PACKAGE_ARCH_RAW="${PACKAGE_ARCH:-${package_arch:-}}"
MANIFEST_PACKAGE_PLATFORM_RAW="${PACKAGE_PLATFORM:-${package_platform:-}}"
fi
}
effective_package_arch() {
if [[ -n "${PACKAGE_ARCH:-}" ]]; then
normalize_arch "$PACKAGE_ARCH" || die "unsupported PACKAGE_ARCH=${PACKAGE_ARCH}"
return 0
fi
if [[ -n "${PACKAGE_PLATFORM:-}" ]]; then
normalize_arch "$PACKAGE_PLATFORM" || die "unsupported PACKAGE_PLATFORM=${PACKAGE_PLATFORM}"
return 0
fi
if [[ -n "${RPKI_PLATFORM:-}" ]]; then
normalize_arch "$RPKI_PLATFORM" || die "unsupported RPKI_PLATFORM=${RPKI_PLATFORM}"
return 0
fi
if [[ -n "${METRICS_PLATFORM:-}" ]]; then
normalize_arch "$METRICS_PLATFORM" || die "unsupported METRICS_PLATFORM=${METRICS_PLATFORM}"
return 0
fi
if [[ -n "${MONITOR_PLATFORM:-}" ]]; then
normalize_arch "$MONITOR_PLATFORM" || die "unsupported MONITOR_PLATFORM=${MONITOR_PLATFORM}"
return 0
fi
die "unable to determine package architecture from manifest or env"
}
arch_mode() {
local host_arch package_arch
host_arch="$(detect_host_arch)"
package_arch="$(effective_package_arch)"
if [[ "$host_arch" == "$package_arch" ]]; then
printf 'native\n'
return 0
fi
if env_flag_enabled "${ALLOW_CROSS_ARCH:-0}"; then
printf 'cross-arch-enabled\n'
return 0
fi
printf 'mismatch-blocked\n'
}
assert_arch_compatibility() {
local host_arch package_arch package_platform mode
host_arch="$(detect_host_arch)"
package_arch="$(effective_package_arch)"
package_platform="${PACKAGE_PLATFORM:-${RPKI_PLATFORM:-$(platform_for_arch "$package_arch")}}"
mode="$(arch_mode)"
case "$mode" in
native)
return 0
;;
cross-arch-enabled)
log "cross-arch execution explicitly enabled: host_arch=$host_arch package_arch=$package_arch package_platform=$package_platform"
return 0
;;
mismatch-blocked)
die "package architecture mismatch: host_arch=$host_arch package_arch=$package_arch package_platform=$package_platform. Set ALLOW_CROSS_ARCH=1 in $ENV_FILE to allow explicit QEMU/binfmt emulation."
;;
*)
die "unknown arch compatibility mode: $mode"
;;
esac
}
load_env() {
load_manifest
if [[ ! -f "$ENV_FILE" ]]; then
[[ -f "$ENV_EXAMPLE" ]] || die "missing $ENV_EXAMPLE"
cp "$ENV_EXAMPLE" "$ENV_FILE"
log "created .env from .env.example"
fi
set -a
# shellcheck disable=SC1090
source "$ENV_FILE"
set +a
if [[ -n "${MANIFEST_PACKAGE_ARCH_RAW:-}" ]]; then
manifest_package_arch="$(normalize_arch "$MANIFEST_PACKAGE_ARCH_RAW")" || die "unsupported manifest package arch: $MANIFEST_PACKAGE_ARCH_RAW"
if [[ -n "${PACKAGE_ARCH:-}" ]]; then
env_package_arch="$(normalize_arch "$PACKAGE_ARCH")" || die "unsupported env package arch: $PACKAGE_ARCH"
[[ "$env_package_arch" == "$manifest_package_arch" ]] || die "env PACKAGE_ARCH=$env_package_arch mismatches manifest PACKAGE_ARCH=$manifest_package_arch"
fi
PACKAGE_ARCH="$manifest_package_arch"
else
if [[ -z "${PACKAGE_ARCH:-}" ]]; then
PACKAGE_ARCH="$(effective_package_arch)"
else
PACKAGE_ARCH="$(normalize_arch "$PACKAGE_ARCH")"
fi
fi
if [[ -n "${MANIFEST_PACKAGE_PLATFORM_RAW:-}" ]]; then
manifest_package_platform="$MANIFEST_PACKAGE_PLATFORM_RAW"
if [[ -n "${PACKAGE_PLATFORM:-}" && "$PACKAGE_PLATFORM" != "$manifest_package_platform" ]]; then
die "env PACKAGE_PLATFORM=$PACKAGE_PLATFORM mismatches manifest PACKAGE_PLATFORM=$manifest_package_platform"
fi
PACKAGE_PLATFORM="$manifest_package_platform"
else
PACKAGE_PLATFORM="${PACKAGE_PLATFORM:-$(platform_for_arch "$PACKAGE_ARCH")}"
fi
HOST_DATA_DIR="${HOST_DATA_DIR:-/var/lib/ours-rp-${PACKAGE_ARCH}-installer}"
COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-ours-rp-${PACKAGE_ARCH}-installer}"
RPKI_IMAGE="${RPKI_IMAGE:-ours-rp-runtime-${PACKAGE_ARCH}:${source_commit_short:-unknown}}"
RPKI_PLATFORM="${RPKI_PLATFORM:-$PACKAGE_PLATFORM}"
METRICS_IMAGE="${METRICS_IMAGE:-ours-rp-metrics-${PACKAGE_ARCH}:${source_commit_short:-unknown}}"
METRICS_PLATFORM="${METRICS_PLATFORM:-$PACKAGE_PLATFORM}"
RTR_REPORT_DIR="${RTR_REPORT_DIR:-$HOST_DATA_DIR/empty-rtr-report}"
RTR_REPORT_CONTAINER_DIR="${RTR_REPORT_CONTAINER_DIR:-/var/lib/ours-rp/rtr-report}"
MONITOR_PLATFORM="${MONITOR_PLATFORM:-$PACKAGE_PLATFORM}"
PROMETHEUS_IMAGE="${PROMETHEUS_IMAGE:-prom/prometheus:v2.55.1}"
GRAFANA_IMAGE="${GRAFANA_IMAGE:-grafana/grafana:11.3.1}"
METRICS_INSTANCE="${METRICS_INSTANCE:-${PACKAGE_ARCH}-installer}"
ALLOW_CROSS_ARCH="${ALLOW_CROSS_ARCH:-0}"
RESOURCE_VALIDATION_MODE="${RESOURCE_VALIDATION_MODE:-validation-update-03}"
FIRST_RUN_WAIT_TIMEOUT_SECS="${FIRST_RUN_WAIT_TIMEOUT_SECS:-7200}"
persist_env_default "$ENV_FILE" "METRICS_IMAGE" "$METRICS_IMAGE"
persist_env_default "$ENV_FILE" "METRICS_PLATFORM" "$METRICS_PLATFORM"
persist_env_default "$ENV_FILE" "RTR_REPORT_DIR" "$RTR_REPORT_DIR"
persist_env_default "$ENV_FILE" "RTR_REPORT_CONTAINER_DIR" "$RTR_REPORT_CONTAINER_DIR"
persist_env_default "$ENV_FILE" "RESOURCE_VALIDATION_MODE" "$RESOURCE_VALIDATION_MODE"
}
compose_cmd() {
docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" -p "$COMPOSE_PROJECT_NAME" "$@"
}
validate_rtr_report_dir() {
load_env
[[ -n "${RTR_REPORT_DIR:-}" ]] || return 0
[[ "$RTR_REPORT_DIR" = /* ]] || die "RTR_REPORT_DIR must be an absolute host path: $RTR_REPORT_DIR"
[[ -d "$RTR_REPORT_DIR" ]] || die "RTR_REPORT_DIR does not exist or is not a directory: $RTR_REPORT_DIR"
}
validate_rtr_registration_dir() {
local report_dir="$1"
[[ -n "$report_dir" ]] || die "--report-dir is required"
[[ "$report_dir" = /* ]] || die "--report-dir must be an absolute host path: $report_dir"
[[ "$report_dir" =~ ^/[A-Za-z0-9._/@:+,-]+$ ]] || die "--report-dir contains unsupported characters: $report_dir"
[[ -d "$report_dir" ]] || die "--report-dir does not exist: $report_dir"
[[ -r "$report_dir" ]] || die "--report-dir is not readable: $report_dir"
}
rtr_report_file_count() {
local report_dir="$1"
{
find "$report_dir" -maxdepth 1 -type f \( \
-name 'rtr-source-*.json' -o \
-name 'rtr-runtime-*.json' -o \
-name 'rtr-clients-*.json' \
\) -print 2>/dev/null || true
} | wc -l | tr -d '[:space:]'
}
latest_rtr_report_file() {
local report_dir="$1"
{
find "$report_dir" -maxdepth 1 -type f \( \
-name 'rtr-source-*.json' -o \
-name 'rtr-runtime-*.json' -o \
-name 'rtr-clients-*.json' \
\) -printf '%T@ %p\n' 2>/dev/null || true
} | sort -nr | awk 'NR == 1 { sub(/^[^ ]+ /, ""); print }'
}
replace_env_key_file() {
local env_path="$1"
local key="$2"
local value="$3"
local temp_path
[[ "$key" =~ ^[A-Z][A-Z0-9_]*$ ]] || die "invalid environment key: $key"
[[ "$value" != *$'\n'* && "$value" != *$'\r'* ]] || die "environment value must be single-line"
temp_path="$(mktemp "${env_path}.tmp.XXXXXX")"
awk -v key="$key" -v value="$value" '
BEGIN { replaced = 0 }
$0 ~ "^" key "=" {
print key "=" value
replaced = 1
next
}
{ print }
END {
if (!replaced) {
print key "=" value
}
}
' "$env_path" > "$temp_path"
chmod --reference="$env_path" "$temp_path" 2>/dev/null || true
mv "$temp_path" "$env_path"
}
wait_for_endpoint() {
local url="$1"
local timeout_secs="$2"
local elapsed=0
while (( elapsed < timeout_secs )); do
if endpoint_ok "$url"; then
return 0
fi
sleep 2
elapsed=$((elapsed + 2))
done
return 1
}
create_data_dirs() {
load_env
mkdir -p \
"$HOST_DATA_DIR/state" \
"$HOST_DATA_DIR/runs" \
"$HOST_DATA_DIR/logs" \
"$HOST_DATA_DIR/tmp" \
"$HOST_DATA_DIR/empty-rtr-report" \
"$HOST_DATA_DIR/prometheus" \
"$HOST_DATA_DIR/grafana"
validate_rtr_report_dir
chmod 755 "$HOST_DATA_DIR" "$HOST_DATA_DIR/state" "$HOST_DATA_DIR/runs" "$HOST_DATA_DIR/logs" "$HOST_DATA_DIR/tmp" || true
chmod 777 "$HOST_DATA_DIR/prometheus" "$HOST_DATA_DIR/grafana" || true
}
latest_run_dir() {
load_env
find "$HOST_DATA_DIR/runs" -maxdepth 1 -mindepth 1 -type d -name 'run_*' 2>/dev/null | sort | tail -1
}
latest_success_run_dir() {
load_env
find "$HOST_DATA_DIR/runs" -maxdepth 2 -type f -path '*/run-summary.json' 2>/dev/null \
| while read -r summary; do
if jq -e '.status == "success"' "$summary" >/dev/null 2>&1; then
dirname "$summary"
fi
done | sort | tail -1
}
has_success_run() {
[[ -n "$(latest_success_run_dir)" ]]
}
print_run_summary() {
local run_dir="$1"
local summary="$run_dir/run-summary.json"
local meta="$run_dir/run-meta.json"
local timing="$run_dir/stage-timing.json"
local process_time="$run_dir/process-time.txt"
local vrps_file="$run_dir/vrps.csv"
local vaps_file="$run_dir/vaps.csv"
local status="unknown"
local sync_mode="unknown"
local wall_ms="null"
local validation_ms="null"
local repo_sync_ms="null"
local max_rss_kb="null"
local publication_points="null"
local vrps="null"
local vaps="null"
local warnings="null"
[[ -f "$summary" ]] || {
warn "missing run-summary.json in $run_dir"
return 1
}
status="$(jq -r '.status // "unknown"' "$summary" 2>/dev/null || echo unknown)"
wall_ms="$(jq -r '.wallMs // .wall_ms // "null"' "$summary" 2>/dev/null || echo null)"
warnings="$(jq -r '.warningCount // .warnings // "null"' "$summary" 2>/dev/null || echo null)"
if [[ -f "$meta" ]]; then
sync_mode="$(jq -r '.sync_mode // .syncMode // "unknown"' "$meta" 2>/dev/null || echo unknown)"
status="$(jq -r --arg fallback "$status" '.status // $fallback' "$meta" 2>/dev/null || echo "$status")"
fi
if [[ -f "$timing" ]]; then
validation_ms="$(jq -r '.validation_ms // "null"' "$timing" 2>/dev/null || echo null)"
repo_sync_ms="$(jq -r '.repo_sync_ms_total // "null"' "$timing" 2>/dev/null || echo null)"
publication_points="$(jq -r '.publication_points // "null"' "$timing" 2>/dev/null || echo null)"
fi
if [[ -f "$process_time" ]]; then
max_rss_kb="$(awk -F': ' '/Maximum resident set size/ {print $2; found=1} END {if (!found) print "null"}' "$process_time")"
fi
if [[ -f "$vrps_file" ]]; then
vrps="$(( $(wc -l < "$vrps_file") > 0 ? $(wc -l < "$vrps_file") - 1 : 0 ))"
fi
if [[ -f "$vaps_file" ]]; then
vaps="$(( $(wc -l < "$vaps_file") > 0 ? $(wc -l < "$vaps_file") - 1 : 0 ))"
fi
jq -n \
--arg run "$(basename "$run_dir")" \
--arg status "$status" \
--arg syncMode "$sync_mode" \
--argjson wallMs "$wall_ms" \
--argjson validationMs "$validation_ms" \
--argjson repoSyncMs "$repo_sync_ms" \
--argjson maxRssKb "$max_rss_kb" \
--argjson vrps "$vrps" \
--argjson vaps "$vaps" \
--argjson publicationPoints "$publication_points" \
--argjson warnings "$warnings" \
'{run:$run,status:$status,syncMode:$syncMode,wallMs:$wallMs,validationMs:$validationMs,repoSyncMs:$repoSyncMs,maxRssKb:$maxRssKb,vrps:$vrps,vaps:$vaps,publicationPoints:$publicationPoints,warnings:$warnings}'
}
wait_for_new_success_run() {
local before_latest="$1"
local timeout_secs="$2"
local start_epoch now run_dir summary meta status meta_status
start_epoch="$(date +%s)"
while true; do
run_dir="$(latest_run_dir || true)"
if [[ -n "$run_dir" && "$run_dir" != "$before_latest" ]]; then
summary="$run_dir/run-summary.json"
meta="$run_dir/run-meta.json"
if [[ -f "$summary" ]]; then
status="$(jq -r '.status // "unknown"' "$summary" 2>/dev/null || echo unknown)"
if [[ "$status" == "success" ]]; then
meta_status="unknown"
if [[ -f "$meta" ]]; then
meta_status="$(jq -r '.status // "unknown"' "$meta" 2>/dev/null || echo unknown)"
fi
if [[ "$meta_status" == "success" ]]; then
print_run_summary "$run_dir" || true
return 0
fi
fi
if [[ "$status" == "failed" || "$status" == "error" ]]; then
print_run_summary "$run_dir" || true
die "run failed: $run_dir"
fi
fi
fi
now="$(date +%s)"
if (( now - start_epoch > timeout_secs )); then
die "timed out waiting for first successful run after ${timeout_secs}s"
fi
sleep 10
done
}
docker_compose_available() {
docker compose version >/dev/null 2>&1
}
install_docker_if_missing() {
if command -v docker >/dev/null 2>&1 && docker_compose_available && command -v jq >/dev/null 2>&1 && command -v rsync >/dev/null 2>&1 && command -v curl >/dev/null 2>&1; then
log "docker and docker compose are already installed"
return 0
fi
if [[ "${SKIP_DEP_INSTALL:-0}" == "1" ]]; then
die "docker/docker compose missing and SKIP_DEP_INSTALL=1"
fi
if ! command -v apt-get >/dev/null 2>&1; then
die "docker/docker compose missing; automatic install currently supports apt-get only"
fi
log "installing missing runtime packages via apt"
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y ca-certificates curl jq rsync gzip tar docker.io
if ! docker_compose_available; then
if apt-cache show docker-compose-v2 >/dev/null 2>&1; then
DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose-v2
elif apt-cache show docker-compose-plugin >/dev/null 2>&1; then
DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose-plugin
elif apt-cache show docker-compose >/dev/null 2>&1; then
DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose
fi
fi
systemctl enable --now docker >/dev/null 2>&1 || true
docker_compose_available || die "docker compose is still unavailable after install"
}
load_installer_images() {
load_env
require_cmd docker
shopt -s nullglob
local image load_output desired_tag
local found=0
for image in "$INSTALLER_ROOT"/images/*.tar "$INSTALLER_ROOT"/images/*.tar.gz; do
found=1
log "loading docker image: $image"
if [[ "$image" == *.gz ]]; then
load_output="$(gzip -dc "$image" | docker load)"
else
load_output="$(docker load -i "$image")"
fi
printf '%s\n' "$load_output"
desired_tag=""
case "$(basename "$image")" in
"${image_tar:-}")
desired_tag="${image_tag:-$RPKI_IMAGE}"
;;
"${metrics_image_tar:-}")
desired_tag="${metrics_image:-$METRICS_IMAGE}"
;;
"${prometheus_image_tar:-}")
desired_tag="${prometheus_image:-$PROMETHEUS_IMAGE}"
;;
"${grafana_image_tar:-}")
desired_tag="${grafana_image:-$GRAFANA_IMAGE}"
;;
esac
if [[ -n "$desired_tag" ]]; then
ensure_loaded_image_tag "$desired_tag" "$load_output" "$image"
fi
done
shopt -u nullglob
(( found == 1 )) || warn "no image tar found under $INSTALLER_ROOT/images"
}
ensure_loaded_image_tag() {
local desired_tag="$1"
local load_output="$2"
local image_path="$3"
local loaded_ref=""
local loaded_id=""
local desired_id=""
loaded_ref="$(printf '%s\n' "$load_output" | awk -F'Loaded image: ' '/Loaded image: / {print $2; exit}')"
if [[ -z "$loaded_ref" ]]; then
loaded_ref="$(printf '%s\n' "$load_output" | awk -F'Loaded image ID: ' '/Loaded image ID: / {print $2; exit}')"
fi
[[ -n "$loaded_ref" ]] || die "unable to determine loaded image reference for $image_path"
loaded_id="$(docker image inspect --format '{{.Id}}' "$loaded_ref" 2>/dev/null || true)"
desired_id="$(docker image inspect --format '{{.Id}}' "$desired_tag" 2>/dev/null || true)"
if [[ -n "$loaded_id" && -n "$desired_id" && "$loaded_id" == "$desired_id" ]]; then
return 0
fi
log "tagging loaded image for package use: source=$loaded_ref target=$desired_tag"
docker tag "$loaded_ref" "$desired_tag"
}
ensure_binfmt_if_needed() {
require_cmd docker
load_env
local host_arch package_arch
host_arch="$(detect_host_arch)"
package_arch="$(effective_package_arch)"
if [[ "$host_arch" == "$package_arch" ]]; then
return 0
fi
assert_arch_compatibility
log "host arch is $host_arch; enabling binfmt/qemu for package arch $package_arch"
docker run --rm --privileged tonistiigi/binfmt --install "$package_arch"
}
verify_runtime_image() {
load_env
require_cmd docker
log "verifying runtime image $RPKI_IMAGE on $RPKI_PLATFORM"
verify_image_platform "$RPKI_IMAGE" "$RPKI_PLATFORM" "runtime"
verify_image_usage_output "$RPKI_IMAGE" "$RPKI_PLATFORM" "runtime" /opt/ours-rp/bin/rpki --help
}
verify_metrics_image() {
load_env
require_cmd docker
log "verifying metrics image $METRICS_IMAGE on $METRICS_PLATFORM"
verify_image_platform "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics"
verify_image_usage_output "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics" /opt/ours-rp/bin/rpki_artifact_metrics --help
}
verify_image_platform() {
local image="$1"
local expected_platform="$2"
local role="$3"
local actual_platform
docker image inspect "$image" >/dev/null
actual_platform="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image" 2>/dev/null || echo unknown)"
[[ "$actual_platform" == "$expected_platform" ]] || die "$role image platform mismatch: image=$image expected=$expected_platform actual=$actual_platform"
}
verify_image_usage_output() {
local image="$1"
local expected_platform="$2"
local role="$3"
shift 3
local help_path
local status
help_path="$(mktemp "${TMPDIR:-/tmp}/ours-rp-image-help.XXXXXX")"
set +e
docker run --rm --platform "$expected_platform" "$image" "$@" >"$help_path" 2>&1
status=$?
set -e
if [[ "$status" != "0" && "$status" != "1" ]]; then
cat "$help_path" >&2 || true
rm -f "$help_path"
die "$role image command failed: image=$image platform=$expected_platform exit=$status"
fi
grep -q '^Usage' "$help_path" || {
cat "$help_path" >&2 || true
rm -f "$help_path"
die "$role image did not emit usage output: image=$image platform=$expected_platform exit=$status"
}
head -5 "$help_path" || true
rm -f "$help_path"
}
verify_monitor_images() {
load_env
require_cmd docker
verify_image_platform "$PROMETHEUS_IMAGE" "$MONITOR_PLATFORM" "prometheus"
verify_image_platform "$GRAFANA_IMAGE" "$MONITOR_PLATFORM" "grafana"
}
endpoint_ok() {
local url="$1"
curl -fsS --max-time 5 "$url" >/dev/null 2>&1
}

49
deploy/docker-installer/scripts/install.sh
View File

@ -1,49 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=common.sh
source "$SCRIPT_DIR/common.sh"
usage() {
cat <<'USAGE'
Usage: ./scripts/install.sh [--skip-dep-install]
Install or update the ours RP Docker installer package idempotently.
USAGE
}
while [[ $# -gt 0 ]]; do
case "$1" in
--skip-dep-install)
export SKIP_DEP_INSTALL=1
shift
;;
-h|--help)
usage
exit 0
;;
*)
die "unknown option: $1"
;;
esac
done
load_env
assert_arch_compatibility
install_docker_if_missing
require_cmd curl
require_cmd jq
require_cmd rsync
require_cmd gzip
require_cmd tar
create_data_dirs
load_installer_images
ensure_binfmt_if_needed
verify_runtime_image
verify_metrics_image
verify_monitor_images
compose_config_path="$(mktemp "${TMPDIR:-/tmp}/ours-rp-compose-config.XXXXXX.yml")"
compose_cmd --profile core --profile sidecar --profile monitor config >"$compose_config_path"
"$SCRIPT_DIR/self-check.sh" --quick
rm -f "$compose_config_path"
log "install complete"

7
deploy/docker-installer/scripts/logs.sh
View File

@ -1,7 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=common.sh
source "$SCRIPT_DIR/common.sh"
load_env
compose_cmd --profile core --profile sidecar --profile monitor logs "$@"

149
deploy/docker-installer/scripts/register_rtr_monitor.sh
View File

@ -1,149 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/common.sh"
REPORT_DIR=""
DRY_RUN=0
CANDIDATE_ENV=""
CONFIG_FILE=""
ORIGINAL_ENV_FILE=""
ENV_BACKUP=""
REGISTRATION_APPLIED=0
usage() {
cat <<'EOF'
Usage: ./scripts/register_rtr_monitor.sh --report-dir <absolute-host-path> [--dry-run]
Registers one local RTR report directory for the artifact metrics sidecar.
The directory must be on the same host as this installer and may contain
rtr-source-*.json, rtr-runtime-*.json, and rtr-clients-*.json report files.
This command recreates only artifact-metrics, Prometheus, and Grafana. It does
not restart or recreate the ours-rp-soak container.
EOF
}
cleanup() {
[[ -n "$CANDIDATE_ENV" && -f "$CANDIDATE_ENV" ]] && rm -f "$CANDIDATE_ENV"
[[ -n "$CONFIG_FILE" && -f "$CONFIG_FILE" ]] && rm -f "$CONFIG_FILE"
return 0
}
trap cleanup EXIT
restore_failed_registration() {
if (( REGISTRATION_APPLIED == 0 )) || [[ -z "$ENV_BACKUP" || ! -f "$ENV_BACKUP" ]]; then
return 0
fi
warn "registration failed after .env replacement; restoring previous metrics configuration"
cp -p "$ENV_BACKUP" "$ORIGINAL_ENV_FILE"
ENV_FILE="$ORIGINAL_ENV_FILE"
load_env || warn "unable to reload restored .env"
compose_cmd --profile sidecar up -d --force-recreate artifact-metrics \
|| warn "unable to recreate artifact-metrics with restored configuration"
}
fail_after_apply() {
restore_failed_registration
die "$*"
}
while (( $# > 0 )); do
case "$1" in
--report-dir)
[[ $# -ge 2 ]] || die "--report-dir requires a value"
REPORT_DIR="$2"
shift 2
;;
--dry-run)
DRY_RUN=1
shift
;;
-h|--help)
usage
exit 0
;;
*)
die "unknown argument: $1"
;;
esac
done
require_cmd docker
require_cmd curl
require_cmd grep
require_cmd jq
validate_rtr_registration_dir "$REPORT_DIR"
load_env
assert_arch_compatibility
ORIGINAL_ENV_FILE="$ENV_FILE"
REPORT_FILE_COUNT="$(rtr_report_file_count "$REPORT_DIR")"
LATEST_REPORT_FILE="$(latest_rtr_report_file "$REPORT_DIR")"
if (( REPORT_FILE_COUNT == 0 )); then
warn "no recognized RTR report JSON files are currently present in $REPORT_DIR"
fi
CANDIDATE_ENV="$(mktemp "$INSTALLER_ROOT/.env.rtr-register.XXXXXX")"
cp -p "$ORIGINAL_ENV_FILE" "$CANDIDATE_ENV"
replace_env_key_file "$CANDIDATE_ENV" "RTR_REPORT_DIR" "$REPORT_DIR"
ENV_FILE="$CANDIDATE_ENV"
load_env
CONFIG_FILE="$(mktemp "$INSTALLER_ROOT/.rtr-compose-config.XXXXXX")"
compose_cmd --profile sidecar --profile monitor config --format json > "$CONFIG_FILE"
jq -e --arg source "$REPORT_DIR" --arg target "$RTR_REPORT_CONTAINER_DIR" '
.services["artifact-metrics"].volumes
| any(.[]; .type == "bind" and .source == $source and .target == $target and .read_only == true)
' "$CONFIG_FILE" >/dev/null || die "candidate compose configuration is missing expected read-only RTR bind mount"
if (( DRY_RUN == 1 )); then
printf 'dry_run=ok\n'
printf 'report_dir=%s\n' "$REPORT_DIR"
printf 'recognized_report_files=%s\n' "$REPORT_FILE_COUNT"
printf 'latest_report=%s\n' "${LATEST_REPORT_FILE:--}"
printf 'expected_mount_source=%s\n' "$REPORT_DIR"
printf 'expected_mount_target=%s\n' "$RTR_REPORT_CONTAINER_DIR"
exit 0
fi
ENV_FILE="$ORIGINAL_ENV_FILE"
load_env
soak_container_before="$(compose_cmd ps -q ours-rp-soak || true)"
ENV_BACKUP="${ORIGINAL_ENV_FILE}.rtr-register-$(date -u +%Y%m%dT%H%M%SZ).bak"
cp -p "$ORIGINAL_ENV_FILE" "$ENV_BACKUP"
mv "$CANDIDATE_ENV" "$ORIGINAL_ENV_FILE"
CANDIDATE_ENV=""
REGISTRATION_APPLIED=1
load_env
compose_cmd --profile sidecar up -d --force-recreate artifact-metrics \
|| fail_after_apply "unable to recreate artifact-metrics"
compose_cmd --profile sidecar --profile monitor up -d prometheus grafana \
|| fail_after_apply "unable to start Prometheus and Grafana"
soak_container_after="$(compose_cmd ps -q ours-rp-soak || true)"
if [[ -n "$soak_container_before" && "$soak_container_before" != "$soak_container_after" ]]; then
fail_after_apply "ours-rp-soak container changed during RTR registration; investigate before continuing"
fi
wait_for_endpoint "http://127.0.0.1:${METRICS_PORT:-9556}/metrics" 60 \
|| fail_after_apply "artifact-metrics did not become ready"
wait_for_endpoint "http://127.0.0.1:${PROMETHEUS_PORT:-9090}/-/ready" 60 \
|| fail_after_apply "Prometheus did not become ready"
wait_for_endpoint "http://127.0.0.1:${GRAFANA_PORT:-3000}/api/health" 60 \
|| fail_after_apply "Grafana did not become ready"
if (( REPORT_FILE_COUNT > 0 )); then
curl -fsS "http://127.0.0.1:${METRICS_PORT:-9556}/metrics" | grep -q '^ours_rp_rtr_' \
|| fail_after_apply "artifact-metrics exposes no RTR metrics after registration"
fi
printf 'registration=ok\n'
printf 'report_dir=%s\n' "$REPORT_DIR"
printf 'recognized_report_files=%s\n' "$REPORT_FILE_COUNT"
printf 'latest_report=%s\n' "${LATEST_REPORT_FILE:--}"
printf 'env_backup=%s\n' "$ENV_BACKUP"
printf 'ours_rp_soak_container_unchanged=%s\n' "${soak_container_before:-not-running}"

5
deploy/docker-installer/scripts/restart.sh
View File

@ -1,5 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
"$SCRIPT_DIR/stop.sh" || true
"$SCRIPT_DIR/start.sh" "$@"

41
deploy/docker-installer/scripts/self-check.sh
View File

@ -1,41 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=common.sh
source "$SCRIPT_DIR/common.sh"
QUICK=0
while [[ $# -gt 0 ]]; do
case "$1" in
--quick)
QUICK=1
shift
;;
-h|--help)
echo "Usage: ./scripts/self-check.sh [--quick]"
exit 0
;;
*)
die "unknown option: $1"
;;
esac
done
load_env
assert_arch_compatibility
require_cmd docker
require_cmd jq
docker compose version >/dev/null
[[ -f "$COMPOSE_FILE" ]] || die "missing compose file"
[[ -f "$ENV_FILE" ]] || die "missing .env"
create_data_dirs
[[ -w "$HOST_DATA_DIR" ]] || die "data dir is not writable: $HOST_DATA_DIR"
compose_cmd --profile core --profile sidecar --profile monitor config >/dev/null
verify_image_platform "$RPKI_IMAGE" "$RPKI_PLATFORM" "runtime"
verify_image_platform "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics"
verify_monitor_images
if [[ "$QUICK" == "0" ]]; then
verify_runtime_image
verify_metrics_image
fi
log "self-check ok"

59
deploy/docker-installer/scripts/start.sh
View File

@ -1,59 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=common.sh
source "$SCRIPT_DIR/common.sh"
WAIT_FIRST_RUN=1
TIMEOUT_SECS=""
usage() {
cat <<'USAGE'
Usage: ./scripts/start.sh [--no-wait-first-run] [--timeout-secs N]
Start ours RP. If no successful run exists, wait for the first snapshot to succeed
before starting metrics, Prometheus and Grafana.
USAGE
}
while [[ $# -gt 0 ]]; do
case "$1" in
--no-wait-first-run)
WAIT_FIRST_RUN=0
shift
;;
--timeout-secs)
TIMEOUT_SECS="$2"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
die "unknown option: $1"
;;
esac
done
load_env
assert_arch_compatibility
create_data_dirs
timeout_secs="${TIMEOUT_SECS:-$FIRST_RUN_WAIT_TIMEOUT_SECS}"
before_latest="$(latest_run_dir || true)"
had_success=0
if has_success_run; then
had_success=1
fi
log "starting core soak service"
compose_cmd --profile core up -d ours-rp-soak
if [[ "$had_success" == "0" && "$WAIT_FIRST_RUN" == "1" ]]; then
log "no previous successful run found; waiting for first run timeout=${timeout_secs}s"
wait_for_new_success_run "$before_latest" "$timeout_secs"
fi
log "starting metrics and monitor services"
compose_cmd --profile sidecar --profile monitor up -d artifact-metrics prometheus grafana
"$SCRIPT_DIR/status.sh" --brief || true

87
deploy/docker-installer/scripts/status.sh
View File

@ -1,87 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=common.sh
source "$SCRIPT_DIR/common.sh"
BRIEF=0
while [[ $# -gt 0 ]]; do
case "$1" in
--brief)
BRIEF=1
shift
;;
-h|--help)
echo "Usage: ./scripts/status.sh [--brief]"
exit 0
;;
*)
die "unknown option: $1"
;;
esac
done
load_env
host_arch="$(detect_host_arch)"
mode="$(arch_mode)"
echo "installer_root=$INSTALLER_ROOT"
echo "manifest_file=$MANIFEST_FILE"
echo "package_name=${package_name:-}"
echo "source_commit=${source_commit:-${git_commit:-}}"
echo "source_commit_short=${source_commit_short:-}"
echo "source_dirty=${source_dirty:-}"
echo "build_timestamp_utc=${build_timestamp_utc:-${created_at_utc:-}}"
echo "host_data_dir=$HOST_DATA_DIR"
echo "host_arch=$host_arch"
echo "package_arch=$PACKAGE_ARCH"
echo "package_platform=$PACKAGE_PLATFORM"
echo "allow_cross_arch=$ALLOW_CROSS_ARCH"
echo "arch_mode=$mode"
echo "runtime_image=$RPKI_IMAGE"
echo "runtime_platform=$RPKI_PLATFORM"
echo "runtime_image_revision=${runtime_image_revision:-}"
echo "runtime_image_dirty=${runtime_image_dirty:-}"
echo "metrics_image=$METRICS_IMAGE"
echo "metrics_platform=$METRICS_PLATFORM"
echo "metrics_image_revision=${metrics_image_revision:-}"
echo "metrics_image_dirty=${metrics_image_dirty:-}"
echo "rtr_report_dir=$RTR_REPORT_DIR"
echo "rtr_report_container_dir=$RTR_REPORT_CONTAINER_DIR"
echo "monitor_platform=$MONITOR_PLATFORM"
echo "rirs=${RIRS:-}"
echo "max_runs=${MAX_RUNS:-}"
echo "interval_secs=${INTERVAL_SECS:-}"
echo "periodic_snapshot_reset=${PERIODIC_SNAPSHOT_RESET:-0}"
echo "periodic_snapshot_max_deltas=${PERIODIC_SNAPSHOT_MAX_DELTAS:-100}"
echo
if command -v docker >/dev/null 2>&1; then
docker version --format 'docker={{.Server.Version}}' 2>/dev/null || echo "docker=unavailable"
docker compose version 2>/dev/null || true
compose_cmd --profile core --profile sidecar --profile monitor ps || true
else
echo "docker=missing"
fi
echo
df -h "$HOST_DATA_DIR" 2>/dev/null || true
echo
latest="$(latest_run_dir || true)"
if [[ -n "$latest" ]]; then
echo "latest_run=$latest"
print_run_summary "$latest" || true
else
echo "latest_run=none"
fi
if [[ "$BRIEF" == "0" ]]; then
echo
endpoint_ok "http://127.0.0.1:${METRICS_PORT:-9556}/metrics" && echo "metrics=ok" || echo "metrics=unavailable"
endpoint_ok "http://127.0.0.1:${PROMETHEUS_PORT:-9090}/-/ready" && echo "prometheus=ok" || echo "prometheus=unavailable"
endpoint_ok "http://127.0.0.1:${GRAFANA_PORT:-3000}/api/health" && echo "grafana=ok" || echo "grafana=unavailable"
echo
if [[ "$RTR_REPORT_DIR" == "$HOST_DATA_DIR/empty-rtr-report" ]]; then
echo "rtr_registration=fallback-empty-directory"
else
echo "rtr_registration=registered"
fi
echo "rtr_report_files=$(rtr_report_file_count "$RTR_REPORT_DIR")"
echo "rtr_latest_report=$(latest_rtr_report_file "$RTR_REPORT_DIR" || true)"
fi

7
deploy/docker-installer/scripts/stop.sh
View File

@ -1,7 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=common.sh
source "$SCRIPT_DIR/common.sh"
load_env
compose_cmd --profile core --profile sidecar --profile monitor stop "$@"

32
deploy/docker-installer/scripts/uninstall.sh
View File

@ -1,32 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=common.sh
source "$SCRIPT_DIR/common.sh"
PURGE_DATA=0
while [[ $# -gt 0 ]]; do
case "$1" in
--purge-data)
PURGE_DATA=1
shift
;;
-h|--help)
echo "Usage: ./scripts/uninstall.sh [--purge-data]"
exit 0
;;
*)
die "unknown option: $1"
;;
esac
done
load_env
compose_cmd --profile core --profile sidecar --profile monitor down --remove-orphans || true
if [[ "$PURGE_DATA" == "1" ]]; then
[[ "$HOST_DATA_DIR" == "/" || -z "$HOST_DATA_DIR" ]] && die "refuse to purge unsafe HOST_DATA_DIR=$HOST_DATA_DIR"
rm -rf "$HOST_DATA_DIR"
log "purged data dir $HOST_DATA_DIR"
else
log "containers removed; data kept at $HOST_DATA_DIR"
fi

110
deploy/docker-installer/scripts/upgrade.sh
View File

@ -1,110 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=common.sh
source "$SCRIPT_DIR/common.sh"
REUSE_ENV_FROM=""
UPDATE_PACKAGE_IMAGE=1
usage() {
cat <<'USAGE'
Usage: ./scripts/upgrade.sh [--reuse-env-from /path/to/.env] [--keep-reused-image]
By default, --reuse-env-from creates the new .env from this package's
.env.example first, then overlays existing user settings from the old .env.
Image tags are intentionally kept from the new package so the upgraded service
actually runs the new packaged runtime and monitor images. Use
--keep-reused-image only when you intentionally want to keep previous image tags.
USAGE
}
while [[ $# -gt 0 ]]; do
case "$1" in
--reuse-env-from)
REUSE_ENV_FROM="$2"
shift 2
;;
--keep-reused-image)
UPDATE_PACKAGE_IMAGE=0
shift
;;
-h|--help)
usage
exit 0
;;
*)
die "unknown option: $1"
;;
esac
done
env_get_key() {
local env_path="$1"
local key="$2"
awk -F= -v key="$key" '$1 == key {sub(/^[^=]*=/, ""); print; exit}' "$env_path"
}
env_set_key() {
local env_path="$1"
local key="$2"
local value="$3"
local tmp_env
if grep -q "^${key}=" "$env_path"; then
tmp_env="$(mktemp)"
awk -v key="$key" -v value="$value" '
BEGIN { done=0 }
$0 ~ "^" key "=" { print key "=" value; done=1; next }
{ print }
END { if (!done) print key "=" value }
' "$env_path" > "$tmp_env"
mv "$tmp_env" "$env_path"
else
printf '%s=%s\n' "$key" "$value" >> "$env_path"
fi
}
overlay_reused_env() {
local source_env="$1"
local target_env="$2"
local key
local value
while IFS='=' read -r key _; do
[[ -n "$key" ]] || continue
[[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue
case "$key" in
PACKAGE_ARCH|PACKAGE_PLATFORM|RPKI_PLATFORM|METRICS_PLATFORM|MONITOR_PLATFORM)
continue
;;
RPKI_IMAGE|METRICS_IMAGE|PROMETHEUS_IMAGE|GRAFANA_IMAGE)
[[ "$UPDATE_PACKAGE_IMAGE" == "0" ]] || continue
;;
esac
value="$(env_get_key "$source_env" "$key")"
env_set_key "$target_env" "$key" "$value"
done < <(grep -E '^[A-Za-z_][A-Za-z0-9_]*=' "$source_env" || true)
}
if [[ -n "$REUSE_ENV_FROM" ]]; then
[[ -f "$REUSE_ENV_FROM" ]] || die "missing reuse env file: $REUSE_ENV_FROM"
if [[ ! -f "$ENV_FILE" ]]; then
[[ -f "$ENV_EXAMPLE" ]] || die "missing $ENV_EXAMPLE"
cp "$ENV_EXAMPLE" "$ENV_FILE"
overlay_reused_env "$REUSE_ENV_FROM" "$ENV_FILE"
log "created new package env from .env.example and overlaid user settings from $REUSE_ENV_FROM"
else
log "keeping existing env at $ENV_FILE; reuse source ignored: $REUSE_ENV_FROM"
fi
fi
load_env
assert_arch_compatibility
create_data_dirs
install_docker_if_missing
load_installer_images
ensure_binfmt_if_needed
verify_runtime_image
verify_metrics_image
verify_monitor_images
compose_cmd --profile core --profile sidecar --profile monitor up -d --force-recreate
"$SCRIPT_DIR/status.sh" --brief || true

117
docker/ours-rp-metrics.Dockerfile
View File

@ -1,117 +0,0 @@
ARG BUILDER_IMAGE=rust:1-bookworm
ARG RUNTIME_IMAGE=debian:bookworm-slim
FROM --platform=$BUILDPLATFORM ${BUILDER_IMAGE} AS builder
ARG BUILDARCH
ARG TARGETARCH
ARG TARGETPLATFORM
WORKDIR /src
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
build-essential \
ca-certificates \
clang \
cmake \
git \
libclang-dev \
make \
perl \
pkg-config \
python3 \
&& if [ "$TARGETARCH" != "$BUILDARCH" ]; then \
case "$TARGETARCH" in \
arm64) \
apt-get install -y --no-install-recommends \
g++-aarch64-linux-gnu \
gcc-aarch64-linux-gnu \
libc6-dev-arm64-cross \
;; \
amd64) \
apt-get install -y --no-install-recommends \
g++-x86-64-linux-gnu \
gcc-x86-64-linux-gnu \
libc6-dev-amd64-cross \
;; \
*) \
echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 \
;; \
esac; \
fi \
&& rm -rf /var/lib/apt/lists/* \
&& case "$TARGETARCH" in \
amd64|arm64) ;; \
*) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 ;; \
esac
COPY . .
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
--mount=type=cache,target=/src/target \
set -eux; \
case "$TARGETARCH" in \
amd64) target_triple=x86_64-unknown-linux-gnu ;; \
arm64) target_triple=aarch64-unknown-linux-gnu ;; \
*) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 ;; \
esac; \
rustup target add "$target_triple"; \
if [ "$TARGETARCH" != "$BUILDARCH" ]; then \
case "$TARGETARCH" in \
arm64) \
export \
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \
CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc \
CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ \
AR_aarch64_unknown_linux_gnu=aarch64-linux-gnu-ar \
PKG_CONFIG_ALLOW_CROSS=1 \
;; \
amd64) \
export \
CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc \
CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc \
CXX_x86_64_unknown_linux_gnu=x86_64-linux-gnu-g++ \
AR_x86_64_unknown_linux_gnu=x86_64-linux-gnu-ar \
PKG_CONFIG_ALLOW_CROSS=1 \
;; \
*) \
echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 \
;; \
esac; \
fi; \
cargo build --release --target "$target_triple" --bin rpki_artifact_metrics; \
mkdir -p /build-out/bin; \
cp "target/$target_triple/release/rpki_artifact_metrics" /build-out/bin/
FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS runtime
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
tzdata \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /opt/ours-rp
COPY --from=builder /build-out/bin/ /opt/ours-rp/bin/
ARG SOURCE_COMMIT=unknown
ARG SOURCE_DIRTY=unknown
ARG BUILD_TIMESTAMP_UTC=unknown
LABEL org.opencontainers.image.title="ours-rp-metrics" \
org.opencontainers.image.description="Ours RP artifact metrics image for multi-arch Docker Compose deployment" \
org.opencontainers.image.revision="${SOURCE_COMMIT}" \
org.opencontainers.image.created="${BUILD_TIMESTAMP_UTC}" \
org.opencontainers.image.source-dirty="${SOURCE_DIRTY}"
RUN chmod +x /opt/ours-rp/bin/* \
&& mkdir -p /var/lib/ours-rp/state /var/lib/ours-rp/runs /var/lib/ours-rp/logs
ENV RUN_ROOT=/var/lib/ours-rp \
BIN_DIR=/opt/ours-rp/bin \
RUST_BACKTRACE=1
VOLUME ["/var/lib/ours-rp"]

145
docker/ours-rp-runtime.Dockerfile
View File

@ -1,145 +0,0 @@
ARG BUILDER_IMAGE=rust:1-bookworm
ARG RUNTIME_IMAGE=debian:bookworm-slim
FROM --platform=$BUILDPLATFORM ${BUILDER_IMAGE} AS builder
ARG BUILDARCH
ARG TARGETARCH
ARG TARGETPLATFORM
WORKDIR /src
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
build-essential \
ca-certificates \
clang \
cmake \
git \
libclang-dev \
make \
perl \
pkg-config \
python3 \
&& if [ "$TARGETARCH" != "$BUILDARCH" ]; then \
case "$TARGETARCH" in \
arm64) \
apt-get install -y --no-install-recommends \
g++-aarch64-linux-gnu \
gcc-aarch64-linux-gnu \
libc6-dev-arm64-cross \
;; \
amd64) \
apt-get install -y --no-install-recommends \
g++-x86-64-linux-gnu \
gcc-x86-64-linux-gnu \
libc6-dev-amd64-cross \
;; \
*) \
echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 \
;; \
esac; \
fi \
&& rm -rf /var/lib/apt/lists/* \
&& case "$TARGETARCH" in \
amd64|arm64) ;; \
*) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 ;; \
esac
COPY . .
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
--mount=type=cache,target=/src/target \
set -eux; \
case "$TARGETARCH" in \
amd64) target_triple=x86_64-unknown-linux-gnu ;; \
arm64) target_triple=aarch64-unknown-linux-gnu ;; \
*) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 ;; \
esac; \
rustup target add "$target_triple"; \
if [ "$TARGETARCH" != "$BUILDARCH" ]; then \
case "$TARGETARCH" in \
arm64) \
export \
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \
CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc \
CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ \
AR_aarch64_unknown_linux_gnu=aarch64-linux-gnu-ar \
PKG_CONFIG_ALLOW_CROSS=1 \
;; \
amd64) \
export \
CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc \
CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc \
CXX_x86_64_unknown_linux_gnu=x86_64-linux-gnu-g++ \
AR_x86_64_unknown_linux_gnu=x86_64-linux-gnu-ar \
PKG_CONFIG_ALLOW_CROSS=1 \
;; \
*) \
echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 \
;; \
esac; \
fi; \
cargo build --release --target "$target_triple" \
--bin rpki \
--bin rpki_daemon \
--bin db_stats; \
mkdir -p /build-out/bin; \
cp \
"target/$target_triple/release/rpki" \
"target/$target_triple/release/rpki_daemon" \
"target/$target_triple/release/db_stats" \
/build-out/bin/
FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS runtime
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
bash \
ca-certificates \
coreutils \
curl \
findutils \
iputils-ping \
jq \
procps \
python3 \
rsync \
time \
tzdata \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /opt/ours-rp
COPY --from=builder /build-out/bin/ /opt/ours-rp/bin/
COPY scripts/soak/run_soak.sh /opt/ours-rp/run_soak.sh
COPY scripts/soak/portable-soak.env.example /opt/ours-rp/portable-soak.env.example
COPY tests/fixtures/tal/ /opt/ours-rp/fixtures/tal/
COPY tests/fixtures/ta/ /opt/ours-rp/fixtures/ta/
COPY fixtures/live_20260619/tal/ /opt/ours-rp/fixtures/live_20260619/tal/
COPY fixtures/live_20260619/ta/ /opt/ours-rp/fixtures/live_20260619/ta/
ARG SOURCE_COMMIT=unknown
ARG SOURCE_DIRTY=unknown
ARG BUILD_TIMESTAMP_UTC=unknown
LABEL org.opencontainers.image.title="ours-rp-runtime" \
org.opencontainers.image.description="Ours RP runtime image for multi-arch Docker Compose deployment" \
org.opencontainers.image.revision="${SOURCE_COMMIT}" \
org.opencontainers.image.created="${BUILD_TIMESTAMP_UTC}" \
org.opencontainers.image.source-dirty="${SOURCE_DIRTY}"
RUN chmod +x /opt/ours-rp/run_soak.sh /opt/ours-rp/bin/* \
&& mkdir -p /var/lib/ours-rp/state /var/lib/ours-rp/runs /var/lib/ours-rp/logs /var/lib/ours-rp/tmp
ENV PACKAGE_ROOT=/opt/ours-rp \
ENV_FILE=/opt/ours-rp/.env \
RUN_ROOT=/var/lib/ours-rp \
BIN_DIR=/opt/ours-rp/bin \
FIXTURE_DIR=/opt/ours-rp/fixtures \
RUST_BACKTRACE=1
VOLUME ["/var/lib/ours-rp"]
CMD ["/opt/ours-rp/run_soak.sh"]

80
fixtures/live_20260619/manifest.json
View File

@ -1,80 +0,0 @@
{
"created_at_utc": "2026-06-19T08:08:03Z",
"items": [
{
"rir": "afrinic",
"ta_bytes": 1216,
"ta_download": "200 1216 1.351147",
"ta_elapsed_s": 1.227,
"ta_path": "rpki_2/rpki/fixtures/live_20260619/ta/afrinic-ta.cer",
"ta_sha256": "43a26fd28bafb9398e5b2ab19e036b450bd04f4973a7f5ad151cebdee0edac36",
"ta_uri": "https://rpki.afrinic.net/repository/AfriNIC.cer",
"tal_bytes": 496,
"tal_download": "200 496 1.361326",
"tal_elapsed_s": 1.238,
"tal_path": "rpki_2/rpki/fixtures/live_20260619/tal/afrinic.tal",
"tal_sha256": "2838ef30ea27ce5705abf5f5adb131d8c35b1f50858338a2f3c84bb207c2fa35",
"tal_url": "https://rpki.afrinic.net/tal/afrinic.tal"
},
{
"rir": "apnic",
"ta_bytes": 1222,
"ta_download": "200 1222 1.012321",
"ta_elapsed_s": 0.921,
"ta_path": "rpki_2/rpki/fixtures/live_20260619/ta/apnic-ta.cer",
"ta_sha256": "2014230ad49b2777ac2bde0948ddfa4b8f207114c549e26d755de88c3593e3af",
"ta_uri": "https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer",
"tal_bytes": 532,
"tal_download": "200 466 1.007155",
"tal_elapsed_s": 0.917,
"tal_path": "rpki_2/rpki/fixtures/live_20260619/tal/apnic.tal",
"tal_sha256": "472e551f7c551c2e999e582b7c9437d3bee4900fe53afff62aeb28d4940ade94",
"tal_url": "https://tal.apnic.net/apnic.tal"
},
{
"rir": "arin",
"ta_bytes": 1143,
"ta_download": "200 1143 0.816714",
"ta_elapsed_s": 0.743,
"ta_path": "rpki_2/rpki/fixtures/live_20260619/ta/arin-ta.cer",
"ta_sha256": "5b3c2f6f04abd19261084487a43c156f778b0b8926d63801d48c7a93ed349492",
"ta_uri": "https://rrdp.arin.net/arin-rpki-ta.cer",
"tal_bytes": 1258,
"tal_download": "200 1258 0.848581",
"tal_elapsed_s": 0.774,
"tal_path": "rpki_2/rpki/fixtures/live_20260619/tal/arin.tal",
"tal_sha256": "1f8bdb03bcc30a3b8e11fd9a87102fba250c22137a3c8baa9c81b139cb412639",
"tal_url": "https://www.arin.net/resources/manage/rpki/arin.tal"
},
{
"rir": "lacnic",
"ta_bytes": 1166,
"ta_download": "200 1166 1.025273",
"ta_elapsed_s": 0.932,
"ta_path": "rpki_2/rpki/fixtures/live_20260619/ta/lacnic-ta.cer",
"ta_sha256": "f44bc51008fd6998de7597b72a79b07bf3ebcb0f14daa7c7c022c0e9d66e0ad0",
"ta_uri": "https://rrdp.lacnic.net/ta/rta-lacnic-rpki.cer",
"tal_bytes": 502,
"tal_download": "200 502 1.729122",
"tal_elapsed_s": 1.565,
"tal_path": "rpki_2/rpki/fixtures/live_20260619/tal/lacnic.tal",
"tal_sha256": "d44bb9394ab009c8b53e5efebf2a1c9450bab61a27efe00de5a3e4587a3a2f6a",
"tal_url": "https://www.lacnic.net/innovaportal/file/4983/1/lacnic.tal"
},
{
"rir": "ripe",
"ta_bytes": 1036,
"ta_download": "200 1036 1.060992",
"ta_elapsed_s": 4.536,
"ta_path": "rpki_2/rpki/fixtures/live_20260619/ta/ripe-ncc-ta.cer",
"ta_sha256": "3e3f7e4efc8d0cea03d9cc1fde6e168b45c26d7b3272e14abd8da4871886e539",
"ta_uri": "https://rpki.ripe.net/ta/ripe-ncc-ta.cer",
"tal_bytes": 482,
"tal_download": "200 482 1.092082",
"tal_elapsed_s": 0.992,
"tal_path": "rpki_2/rpki/fixtures/live_20260619/tal/ripe-ncc.tal",
"tal_sha256": "59ca27ef93f23682749fcefe7c6d70fbc723343549ff9e4d3996acaff79817fb",
"tal_url": "https://tal.rpki.ripe.net/ripe-ncc.tal"
}
]
}

BIN
fixtures/live_20260619/ta/afrinic-ta.cer
View File

Binary file not shown.

BIN
fixtures/live_20260619/ta/apnic-ta.cer
View File

Binary file not shown.

BIN
fixtures/live_20260619/ta/arin-ta.cer
View File

Binary file not shown.

BIN
fixtures/live_20260619/ta/lacnic-ta.cer
View File

Binary file not shown.

BIN
fixtures/live_20260619/ta/ripe-ncc-ta.cer
View File

Binary file not shown.

10
fixtures/live_20260619/tal/afrinic.tal
View File

@ -1,10 +0,0 @@
rsync://rpki.afrinic.net/repository/AfriNIC.cer
https://rpki.afrinic.net/repository/AfriNIC.cer
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxsAqAhWIO+ON2Ef9oRDM
pKxv+AfmSLIdLWJtjrvUyDxJPBjgR+kVrOHUeTaujygFUp49tuN5H2C1rUuQavTH
vve6xNF5fU3OkTcqEzMOZy+ctkbde2SRMVdvbO22+TH9gNhKDc9l7Vu01qU4LeJH
k3X0f5uu5346YrGAOSv6AaYBXVgXxa0s9ZvgqFpim50pReQe/WI3QwFKNgpPzfQL
6Y7fDPYdYaVOXPXSKtx7P4s4KLA/ZWmRL/bobw/i2fFviAGhDrjqqqum+/9w1hEl
L/vqihVnV18saKTnLvkItA/Bf5i11Yhw2K7qv573YWxyuqCknO/iYLTR1DToBZcZ
UQIDAQAB

10
fixtures/live_20260619/tal/apnic-rfc7730-https.tal
View File

@ -1,10 +0,0 @@
https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer
rsync://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx9RWSL61YAAYumEiU8z8
qH2ETVIL01ilxZlzIL9JYSORMN5Cmtf8V2JblIealSqgOTGjvSjEsiV73s67zYQI
7C/iSOb96uf3/s86NqbxDiFQGN8qG7RNcdgVuUlAidl8WxvLNI8VhqbAB5uSg/Mr
LeSOvXRja041VptAxIhcGzDMvlAJRwkrYK/Mo8P4E2rSQgwqCgae0ebY1CsJ3Cjf
i67C1nw7oXqJJovvXJ4apGmEv8az23OLC6Ki54Ul/E6xk227BFttqFV3YMtKx42H
cCcDVZZy01n7JjzvO8ccaXmHIgR7utnqhBRNNq5Xc5ZhbkrUsNtiJmrZzVlgU6Ou
0wIDAQAB

10
fixtures/live_20260619/tal/apnic.tal
View File

@ -1,10 +0,0 @@
https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer
rsync://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx9RWSL61YAAYumEiU8z8
qH2ETVIL01ilxZlzIL9JYSORMN5Cmtf8V2JblIealSqgOTGjvSjEsiV73s67zYQI
7C/iSOb96uf3/s86NqbxDiFQGN8qG7RNcdgVuUlAidl8WxvLNI8VhqbAB5uSg/Mr
LeSOvXRja041VptAxIhcGzDMvlAJRwkrYK/Mo8P4E2rSQgwqCgae0ebY1CsJ3Cjf
i67C1nw7oXqJJovvXJ4apGmEv8az23OLC6Ki54Ul/E6xk227BFttqFV3YMtKx42H
cCcDVZZy01n7JjzvO8ccaXmHIgR7utnqhBRNNq5Xc5ZhbkrUsNtiJmrZzVlgU6Ou
0wIDAQAB

19
fixtures/live_20260619/tal/arin.tal
View File

@ -1,19 +0,0 @@
# THIS TRUST ANCHOR LOCATOR IS PROVIDED BY THE AMERICAN REGISTRY FOR
# INTERNET NUMBERS (ARIN) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL ARIN BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS PUBLIC KEY, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
rsync://rpki.arin.net/repository/arin-rpki-ta.cer
https://rrdp.arin.net/arin-rpki-ta.cer
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3lZPjbHvMRV5sDDqfLc/685th5FnreHMJjg8
pEZUbG8Y8TQxSBsDebbsDpl3Ov3Cj1WtdrJ3CIfQODCPrrJdOBSrMATeUbPC+JlNf2SRP3UB+VJFgtTj
0RN8cEYIuhBW5t6AxQbHhdNQH+A1F/OJdw0q9da2U29Lx85nfFxvnC1EpK9CbLJS4m37+RlpNbT1cba+
b+loXpx0Qcb1C4UpJCGDy7uNf5w6/+l7RpATAHqqsX4qCtwwDYlbHzp2xk9owF3mkCxzl0HwncO+sEHH
eaL3OjtwdIGrRGeHi2Mpt+mvWHhtQqVG+51MHTyg+nIjWFKKGx1Q9+KDx4wJStwveQIDAQAB

4
fixtures/live_20260619/tal/lacnic.tal
View File

@ -1,4 +0,0 @@
https://rrdp.lacnic.net/ta/rta-lacnic-rpki.cer
rsync://repository.lacnic.net/rpki/lacnic/rta-lacnic-rpki.cer
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqZEzhYK0+PtDOPfub/KRc3MeWx3neXx4/wbnJWGbNAtbYqXg3uU5J4HFzPgk/VIppgSKAhlO0H60DRP48by9gr5/yDHu2KXhOmnMg46sYsUIpfgtBS9+VtrqWziJfb+pkGtuOWeTnj6zBmBNZKK+5AlMCW1WPhrylIcB+XSZx8tk9GS/3SMQ+YfMVwwAyYjsex14Uzto4GjONALE5oh1M3+glRQduD6vzSwOD+WahMbc9vCOTED+2McLHRKgNaQf0YJ9a1jG9oJIvDkKXEqdfqDRktwyoD74cV57bW3tBAexB7GglITbInyQAsmdngtfg2LUMrcROHHP86QPZINjDQIDAQAB

10
fixtures/live_20260619/tal/ripe-ncc.tal
View File

@ -1,10 +0,0 @@
https://rpki.ripe.net/ta/ripe-ncc-ta.cer
rsync://rpki.ripe.net/ta/ripe-ncc-ta.cer
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0URYSGqUz2myBsOzeW1j
Q6NsxNvlLMyhWknvnl8NiBCs/T/S2XuNKQNZ+wBZxIgPPV2pFBFeQAvoH/WK83Hw
A26V2siwm/MY2nKZ+Olw+wlpzlZ1p3Ipj2eNcKrmit8BwBC8xImzuCGaV0jkRB0G
Z0hoH6Ml03umLprRsn6v0xOP0+l6Qc1ZHMFVFb385IQ7FQQTcVIxrdeMsoyJq9eM
kE6DoclHhF/NlSllXubASQ9KUWqJ0+Ot3QCXr4LXECMfkpkVR2TZT+v5v658bHVs
6ZxRD1b6Uk1uQKAyHUbn/tXvP8lrjAibGzVsXDT2L0x4Edx+QdixPgOji3gBMyL2
VwIDAQAB

2734
model.txt
View File

File diff suppressed because it is too large Load Diff

220
monitor/README.md
View File

@ -1,220 +0,0 @@
# Ours RP Prometheus / Grafana Monitor
本目录提供本地开发监控栈，用于采集 `rpki_artifact_metrics` 暴露的 ours RP soak 指标。
## 前置条件
1. Docker + Docker Compose v2；
2. 宿主机已启动 `rpki_artifact_metrics`，并监听 Docker 网桥可访问的地址，例如 `0.0.0.0:9556`；
3. Prometheus 容器通过 `host.docker.internal:9556` 访问宿主 sidecar。
Linux Docker 下 compose 已配置：
```yaml
extra_hosts:
- host.docker.internal:host-gateway
```
## 启动
```bash
cd rpki_2/rpki/monitor
docker compose up -d
```
默认镜像使用官方 Docker Hub 镜像：
```text
prom/prometheus:v2.55.1
grafana/grafana:11.3.1
```
如需切到其它镜像源：
```bash
PROMETHEUS_IMAGE=<mirror>/prom/prometheus:v2.55.1 \
GRAFANA_IMAGE=<mirror>/grafana/grafana:11.3.1 \
docker compose up -d
```
默认端口：
- Prometheus: <http://localhost:9090>
- Grafana: <http://localhost:3000>
- Grafana 默认账号密码：`admin` / `admin`
如端口冲突：
```bash
PROMETHEUS_PORT=19090 GRAFANA_PORT=13000 docker compose up -d
```
Prometheus 默认保留 7 天数据；可通过 `PROMETHEUS_RETENTION` 覆盖：
```bash
PROMETHEUS_RETENTION=7d docker compose up -d
```
## 长期稳定性测试
portable soak package 内置 `run_24h_soak_with_metrics.sh`，用于连续运行 ours RP、启动 metrics sidecar、启动本监控栈，并每小时生成报告：
```bash
cd /path/to/portable-soak
SOAK_DURATION_SECS=0 \
HOURLY_REPORT_INTERVAL_SECS=3600 \
SOAK_RETAIN_RUNS=100 \
CLEAN_TMP_AFTER_RUN=1 \
PROMETHEUS_RETENTION=7d \
STOP_MONITOR_STACK_ON_EXIT=0 \
FEISHU_WEBHOOK_SCRIPT=/home/yuyr/.codex/skills/user/feishu-webhook/scripts/send_feishu_text.py \
./run_24h_soak_with_metrics.sh
```
`SOAK_DURATION_SECS=0` 表示持续运行不自动停止；如需 24 小时自然停止，可设置为 `86400`，脚本会等当前 run 完成后退出，不会直接 kill 半轮验证。
关键产物：
- `runs/run_xxxx/`：最近 100 个 run 原始产物；
- `hourly_reports/hour_*.md`：小时级报告；
- `hourly_reports/hourly_summary.jsonl`：小时级结构化汇总；
- `incident_runs/run_xxxx/`：异常 run 固化副本；
- `logs/metrics.*`、`logs/24h-soak.*`、`logs/hourly-reporter.*`：运行日志。
短周期联调可把 `SOAK_DURATION_SECS` 和 `HOURLY_REPORT_INTERVAL_SECS` 调小，并设置 `FEISHU_DRY_RUN=1` 避免真实飞书通知。
## 停止
```bash
cd rpki_2/rpki/monitor
docker compose down
```
保留数据 volume。若要清理数据：
```bash
docker compose down -v
```
## 典型本地联调命令
先启动 APNIC soak 和 metrics sidecar，例如：
```bash
# soak .env 关键配置
MAX_RUNS=-1
RIRS=apnic
RETAIN_RUNS=5
INTERVAL_SECS=0
# metrics sidecar
rpki_artifact_metrics \
--run-root /path/to/portable-soak \
--listen 0.0.0.0:9556 \
--poll-secs 5 \
--instance local-apnic-continuous
```
再启动监控栈：
```bash
cd rpki_2/rpki/monitor
docker compose up -d
```
## 验证
Prometheus target：
```bash
curl -s 'http://localhost:9090/api/v1/targets' | python3 -m json.tool
```
Prometheus query：
```bash
curl -G 'http://localhost:9090/api/v1/query' \
--data-urlencode 'query=up{job="ours-rp-artifact-metrics"}'
curl -G 'http://localhost:9090/api/v1/query' \
--data-urlencode 'query=ours_rp_run_completed_total{status="success"}'
```
Grafana health：
```bash
curl -s http://localhost:3000/api/health | python3 -m json.tool
```
Grafana dashboard：
- 打开 <http://localhost:3000/d/ours-rp-soak-overview/ours-rp-soak-overview>
## 主要指标
- `ours_rp_metrics_service_up`
- `ours_rp_run_completed_total`
- `ours_rp_run_duration_seconds`
- `ours_rp_run_max_rss_bytes`
- `ours_rp_vrps{kind="total|unique"}`：`total` 为去重前 VRP 条目数，`unique` 按 `(ASN, IP Prefix, Max Length)` 去重。
- `ours_rp_vaps`
- `ours_rp_publication_points`
- `ours_rp_repo_sync_phase_count`
- `ours_rp_large_publication_points{object_count_gt="10|50|100|..."}`
- `ours_rp_cir_objects`
- `ours_rp_ccr_state_items`
## Inter-RP 持续对比监控
`rpki_inter_rp_metrics` 用于汇总三方 RP 的最新产物：
- ours RP：读取当前 portable soak 的 `runs/run_xxxx/run-summary.json`、`result.ccr`、CSV 产物；
- Routinator：读取远端200同步来的 `routinator/latest/run-meta.json`、`vrps.csv`、`vaps.csv`；
- rpki-client 9.8：读取远端200同步来的 `rpki-client/latest/run-meta.json`、`vrps.csv`、`vaps.csv`、`result.ccr`。
远端231 启动 sidecar 示例：
```bash
rpki_inter_rp_metrics \
--ours-run-root /root/rpki_20260608_2_feature062_24h_20260608T075547Z/portable-soak \
--peer-root /root/inter-rp-aggregator/synced-from-200 \
--listen 0.0.0.0:9557 \
--poll-secs 30 \
--instance remote231-inter-rp
```
Prometheus 已新增 `ours-rp-inter-rp-metrics` scrape job，默认访问 `host.docker.internal:9557`。
远端200 runner 与远端231同步脚本位于：
```text
scripts/inter_rp/run_remote200_rp_loops.sh
scripts/inter_rp/run_single_rp_with_rss.sh
scripts/inter_rp/sync_remote200_to_231.sh
scripts/inter_rp/run_inter_rp_metrics_sidecar.sh
scripts/inter_rp/inter-rp.env.example
```
如需从本机独立开关远端200上的 Routinator 或 rpki-client，使用：
```bash
scripts/inter_rp/control_remote200_rp.sh status all
scripts/inter_rp/control_remote200_rp.sh stop routinator
scripts/inter_rp/control_remote200_rp.sh start routinator
scripts/inter_rp/control_remote200_rp.sh restart rpki-client
```
默认远端为 `root@43.110.128.200`，可通过 `REMOTE_HOST=...` 覆盖；脚本只管理指定 RP 的 loop 和当前子进程，不会自动影响另一个 RP。
关键指标：
- `inter_rp_run_wall_seconds{rp="ours-rp|routinator|rpki-client"}`
- `inter_rp_run_max_rss_bytes{rp="...",kind="aggregate_peak"}`
- `inter_rp_vrps{rp="..."}`：按 `(ASN, IP Prefix, Max Length)` 去重。
- `inter_rp_vaps{rp="..."}`：按 `(Customer ASN, Providers)` 去重，Routinator 使用 `--enable-aspa` JSON 输出转换，rpki-client 使用 `-j` JSON 输出转换。
- `inter_rp_ccr_digest_match{left="ours-rp",right="rpki-client",state="overall|mfts|vrps|vaps|tas|rks"}`
- `inter_rp_sync_age_seconds`
Grafana dashboard：
- <http://localhost:3000/d/ours-rp-inter-rp/ours-rp-inter-rp>

38
monitor/docker-compose.yml
View File

@ -1,38 +0,0 @@
services:
prometheus:
image: ${PROMETHEUS_IMAGE:-prom/prometheus:v2.55.1}
container_name: ours-rp-prometheus
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-7d}
- --web.enable-lifecycle
extra_hosts:
- host.docker.internal:host-gateway
ports:
- "${PROMETHEUS_PORT:-9090}:9090"
volumes:
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- prometheus-data:/prometheus
restart: unless-stopped
grafana:
image: ${GRAFANA_IMAGE:-grafana/grafana:11.3.1}
container_name: ours-rp-grafana
depends_on:
- prometheus
ports:
- "${GRAFANA_PORT:-3000}:3000"
environment:
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
GF_USERS_ALLOW_SIGN_UP: "false"
volumes:
- grafana-data:/var/lib/grafana
- ./grafana/provisioning:/etc/grafana/provisioning:ro
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
restart: unless-stopped
volumes:
prometheus-data:
grafana-data:

826
monitor/grafana/dashboards/ours-rp-inter-rp.json
View File

@ -1,826 +0,0 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"id": 1,
"title": "Ours Only Repo Count",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"expr": "max(inter_rp_repo_sync_overlap_total{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_ours\"})",
"legendFormat": "only ours",
"refId": "A",
"instant": true
}
]
},
{
"id": 2,
"title": "Routinator Only Repo Count",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"expr": "max(inter_rp_repo_sync_overlap_total{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_routinator\"})",
"legendFormat": "only routinator",
"refId": "A",
"instant": true
}
]
},
{
"id": 3,
"title": "Ours vs Routinator VAP Diff",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"expr": "max(inter_rp_vaps_diff{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"})",
"legendFormat": "vap diff",
"refId": "A",
"instant": true
}
]
},
{
"id": 4,
"title": "Ours vs Routinator VRP Diff",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"expr": "max(inter_rp_vrps_diff{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"})",
"legendFormat": "vrp diff",
"refId": "A",
"instant": true
}
]
},
{
"id": 5,
"title": "Wall Time by RP",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 4
},
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "inter_rp_run_wall_seconds{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
"legendFormat": "{{rp}}",
"refId": "A"
}
]
},
{
"id": 6,
"title": "Max RSS Aggregate Peak by RP",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 4
},
"fieldConfig": {
"defaults": {
"unit": "bytes",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "inter_rp_run_max_rss_bytes{exported_instance=~\".*inter-rp\",kind=\"aggregate_peak\",rp=~\"ours-rp|routinator\"}",
"legendFormat": "{{rp}}",
"refId": "A"
}
]
},
{
"id": 7,
"title": "VRPs by RP (unique ASN/Prefix/MaxLen)",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 12
},
"fieldConfig": {
"defaults": {
"unit": "none",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "inter_rp_vrps{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
"legendFormat": "{{rp}}",
"refId": "A"
}
]
},
{
"id": 8,
"title": "VAPs / ASPAs by RP (unique Customer/Providers)",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 12
},
"fieldConfig": {
"defaults": {
"unit": "none",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "inter_rp_vaps{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
"legendFormat": "{{rp}}",
"refId": "A"
}
]
},
{
"id": 9,
"title": "Latest RP Runs",
"type": "table",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 20
},
"fieldConfig": {
"defaults": {
"unit": "none",
"decimals": 0
},
"overrides": []
},
"options": {
"showHeader": true,
"sortBy": []
},
"targets": [
{
"expr": "inter_rp_run_seq{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
"format": "table",
"instant": true,
"legendFormat": "{{rp}} seq",
"refId": "A"
},
{
"expr": "inter_rp_run_success{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
"format": "table",
"instant": true,
"legendFormat": "{{rp}} success",
"refId": "B"
},
{
"expr": "inter_rp_run_wall_seconds{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
"format": "table",
"instant": true,
"legendFormat": "{{rp}} wall",
"refId": "C"
}
]
},
{
"id": 10,
"title": "Output Count Diffs (unique)",
"type": "table",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 20
},
"fieldConfig": {
"defaults": {
"unit": "none",
"decimals": 0
},
"overrides": []
},
"options": {
"showHeader": true,
"sortBy": []
},
"targets": [
{
"expr": "inter_rp_vrps_diff{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"}",
"format": "table",
"instant": true,
"legendFormat": "vrps ours-rp-routinator",
"refId": "A"
},
{
"expr": "inter_rp_vaps_diff{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"}",
"format": "table",
"instant": true,
"legendFormat": "vaps ours-rp-routinator",
"refId": "B"
}
]
},
{
"id": 15,
"title": "VRP Diff Trend by Class",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 28
},
"fieldConfig": {
"defaults": {
"unit": "none",
"min": 0,
"decimals": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "inter_rp_vrps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"total\"}",
"legendFormat": "total diff",
"refId": "A"
},
{
"expr": "inter_rp_vrps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_ours\"}",
"legendFormat": "only ours",
"refId": "B"
},
{
"expr": "inter_rp_vrps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_routinator\"}",
"legendFormat": "only routinator",
"refId": "C"
}
]
},
{
"id": 16,
"title": "VAP Diff Trend by Class",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 28
},
"fieldConfig": {
"defaults": {
"unit": "none",
"min": 0,
"decimals": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "inter_rp_vaps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"total\"}",
"legendFormat": "total diff",
"refId": "A"
},
{
"expr": "inter_rp_vaps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_ours\"}",
"legendFormat": "only ours",
"refId": "B"
},
{
"expr": "inter_rp_vaps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_routinator\"}",
"legendFormat": "only routinator",
"refId": "C"
}
]
},
{
"id": 11,
"title": "Artifact Age by RP",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 36
},
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "inter_rp_artifact_age_seconds{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
"legendFormat": "{{rp}}",
"refId": "A"
}
]
},
{
"id": 12,
"title": "Repo Sync Availability by RP",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 44
},
"fieldConfig": {
"defaults": {
"unit": "none",
"min": 0,
"decimals": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "inter_rp_repo_sync_total{exported_instance=~\".*inter-rp\",state=~\"available|failed\",rp=~\"ours-rp|routinator\"}",
"legendFormat": "{{rp}} {{state}}",
"refId": "A"
}
]
},
{
"id": 13,
"title": "Repo Sync Overlap Classes",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 44
},
"fieldConfig": {
"defaults": {
"unit": "none",
"min": 0,
"decimals": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "inter_rp_repo_sync_overlap_total{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"}",
"legendFormat": "{{class}}",
"refId": "A"
}
]
},
{
"id": 14,
"title": "Repo Sync Diff URIs",
"type": "table",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 52
},
"fieldConfig": {
"defaults": {
"unit": "none",
"decimals": 0
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "uri"
},
"properties": [
{
"id": "custom.width",
"value": 760
}
]
},
{
"matcher": {
"id": "byName",
"options": "class"
},
"properties": [
{
"id": "custom.width",
"value": 140
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": "^(mft|crl|crt|roa|aspa)$"
},
"properties": [
{
"id": "custom.align",
"value": "right"
},
{
"id": "custom.width",
"value": 80
}
]
}
]
},
"options": {
"showHeader": true,
"sortBy": []
},
"targets": [
{
"expr": "inter_rp_repo_sync_diff_info{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"}",
"format": "table",
"instant": true,
"legendFormat": "{{class}} #{{rank}}",
"refId": "A"
}
],
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"Value": true,
"__name__": true,
"exported_instance": true,
"instance": true,
"job": true,
"left": true,
"right": true,
"rank": true,
"routinator_duration": true
},
"indexByName": {
"class": 0,
"uri": 1,
"mft": 2,
"crl": 3,
"crt": 4,
"roa": 5,
"aspa": 6
},
"renameByName": {
"class": "class",
"uri": "uri",
"mft": "mft",
"crl": "crl",
"crt": "crt",
"roa": "roa",
"aspa": "aspa"
}
}
}
]
}
],
"refresh": "10s",
"schemaVersion": 40,
"tags": [
"rpki",
"inter-rp",
"routinator"
],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "browser",
"title": "Ours RP vs Routinator",
"uid": "ours-rp-inter-rp",
"version": 4
}

761
monitor/grafana/dashboards/ours-rp-repo-sync.json
View File

@ -1,761 +0,0 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_publication_points",
"legendFormat": "publication points",
"refId": "A"
}
],
"title": "Publication Points",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 0
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_ok\"}",
"legendFormat": "rrdp ok",
"refId": "A"
}
],
"title": "RRDP OK Points",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 0
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "sum(ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}) or vector(0)",
"legendFormat": "fallback",
"refId": "A"
}
],
"title": "Rsync Fallback Points",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 0
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
"legendFormat": "failed no cache",
"refId": "A"
}
],
"title": "Failed No Cache Points",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 4
},
"id": 5,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"repo_sync_total\"}",
"legendFormat": "repo sync total",
"refId": "A"
},
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"rrdp_download_total\"}",
"legendFormat": "rrdp download",
"refId": "B"
},
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"rsync_download_total\"}",
"legendFormat": "rsync download",
"refId": "C"
}
],
"title": "Repo Sync Download Durations",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 12,
"w": 12,
"h": 8
},
"id": 6,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_repo_sync_phase_count",
"legendFormat": "{{phase}}",
"refId": "A"
}
],
"title": "Repo Sync Phase Counts",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 12,
"w": 12,
"h": 8
},
"id": 7,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}",
"legendFormat": "rrdp failed, rsync ok",
"refId": "A"
},
{
"expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_failed\"}",
"legendFormat": "rrdp failed, rsync failed",
"refId": "B"
},
{
"expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
"legendFormat": "failed no cache",
"refId": "C"
},
{
"expr": "ours_rp_tree_instances{state=\"failed\"}",
"legendFormat": "tree failed",
"refId": "D"
}
],
"title": "Repo Failure / Fallback Counts",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 20,
"w": 12,
"h": 8
},
"id": 8,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_ok\"}",
"legendFormat": "rsync fallback duration",
"refId": "A"
},
{
"expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_failed\"}",
"legendFormat": "failed duration",
"refId": "B"
},
{
"expr": "ours_rp_repo_terminal_state_duration_seconds_total{terminal_state=\"failed_no_cache\"}",
"legendFormat": "failed no cache duration",
"refId": "C"
}
],
"title": "Repo Failure / Fallback Durations",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 29,
"w": 12,
"h": 9
},
"id": 9,
"options": {
"showHeader": true,
"cellHeight": "sm",
"footer": {
"show": false,
"reducer": [
"sum"
],
"countRows": false,
"fields": ""
}
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_rrdp_rsync_failed_repository_duration_seconds",
"format": "table",
"instant": true,
"legendFormat": "",
"refId": "A"
}
],
"title": "RRDP + Rsync Failed Repositories",
"type": "table",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"job": true,
"terminal_state": true,
"rank": true,
"transport": true,
"__name__": true,
"publication_points": true,
"instance": true,
"repo_id": true,
"pp_id": true,
"exported_instance": true,
"rp": true,
"source": true
},
"indexByName": {
"Time": 0,
"host": 1,
"phase": 2,
"uri": 3,
"Value": 4
},
"renameByName": {
"Value": "duration"
}
}
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 29,
"w": 12,
"h": 9
},
"id": 11,
"options": {
"showHeader": true,
"cellHeight": "sm",
"footer": {
"show": false,
"reducer": [
"sum"
],
"countRows": false,
"fields": ""
}
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "topk(20, ours_rp_top_repository_sync_duration_seconds)",
"format": "table",
"instant": true,
"legendFormat": "",
"refId": "A"
}
],
"title": "Top 20 Repositories by Sync Duration",
"type": "table",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"job": true,
"terminal_state": true,
"__name__": true,
"publication_points": true,
"instance": true,
"repo_id": true,
"phase": true,
"pp_id": true,
"exported_instance": true,
"rp": true,
"source": true
},
"indexByName": {
"Time": 0,
"host": 1,
"rank": 2,
"transport": 3,
"uri": 4,
"Value": 5
},
"renameByName": {
"Value": "value"
}
}
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 38,
"w": 24,
"h": 9
},
"id": 10,
"options": {
"showHeader": true,
"cellHeight": "sm",
"footer": {
"show": false,
"reducer": [
"sum"
],
"countRows": false,
"fields": ""
}
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "topk(20, ours_rp_top_publication_point_object_count)",
"format": "table",
"instant": true,
"legendFormat": "",
"refId": "A"
}
],
"title": "Top Publication Points by Objects",
"type": "table",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"job": true,
"__name__": true,
"publication_points": true,
"instance": true,
"repo_id": true,
"phase": true,
"pp_id": true,
"exported_instance": true,
"rp": true,
"source": true
},
"indexByName": {
"Time": 0,
"host": 1,
"rank": 2,
"terminal_state": 3,
"transport": 4,
"uri": 5,
"Value": 6
},
"renameByName": {}
}
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"description": "Per-repository sync success in the latest successful run; 1 means successful, 0 means failed or failed_no_cache.",
"fieldConfig": {
"defaults": {
"unit": "bool"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 47
},
"id": 12,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"expr": "ours_rp_repository_sync_success",
"legendFormat": "{{host}} {{repo_id}}",
"refId": "A"
}
],
"title": "Repository Sync Success by Repo",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"description": "Per-repository total sync duration aggregated from publication point repo_sync_duration_ms.",
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 55
},
"id": 13,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"expr": "ours_rp_repository_sync_duration_seconds{stat=\"sum\"}",
"legendFormat": "{{host}} {{repo_id}}",
"refId": "A"
}
],
"title": "Repository Sync Duration by Repo",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"description": "Per-repository downloaded bytes attributed from report.json downloads events.",
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 63
},
"id": 14,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"expr": "ours_rp_repository_download_bytes",
"legendFormat": "{{host}} {{repo_id}}",
"refId": "A"
}
],
"title": "Repository Download Bytes by Repo",
"type": "timeseries"
}
],
"refresh": "5s",
"schemaVersion": 40,
"tags": [
"ours-rp",
"rpki",
"soak",
"repo-sync"
],
"templating": {
"list": []
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Ours RP Repo Sync",
"uid": "ours-rp-repo-sync",
"version": 3,
"weekStart": ""
}

666
monitor/grafana/dashboards/ours-rp-rtr-overview.json
View File

@ -1,666 +0,0 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"id": 1,
"title": "RTR Metrics Enabled",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 4,
"w": 4,
"x": 0,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"expr": "max(ours_rp_rtr_metrics_enabled)",
"legendFormat": "enabled",
"refId": "A",
"instant": true
}
]
},
{
"id": 2,
"title": "Refresh Success",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 4,
"w": 5,
"x": 4,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"expr": "max(ours_rp_rtr_source_refresh_status{status=\"success\"})",
"legendFormat": "success",
"refId": "A",
"instant": true
}
]
},
{
"id": 3,
"title": "Consecutive Failures",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 4,
"w": 5,
"x": 9,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"expr": "max(ours_rp_rtr_source_refresh_consecutive_failures)",
"legendFormat": "failures",
"refId": "A",
"instant": true
}
]
},
{
"id": 4,
"title": "Last Success Age",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 4,
"w": 5,
"x": 14,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "s",
"decimals": 0,
"min": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"expr": "max(ours_rp_rtr_source_last_success_age_seconds)",
"legendFormat": "age",
"refId": "A",
"instant": true
}
]
},
{
"id": 5,
"title": "RTR RSS",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 4,
"w": 5,
"x": 19,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "bytes",
"decimals": 0,
"min": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"targets": [
{
"expr": "max(ours_rp_rtr_process_rss_bytes)",
"legendFormat": "rss",
"refId": "A",
"instant": true
}
]
},
{
"id": 6,
"title": "Data Quality Totals",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 4
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_rtr_data_quality_items{stage=~\"ccr_input|before_slurm|after_slurm\",type=\"total\"}",
"legendFormat": "{{stage}} total",
"refId": "A"
},
{
"expr": "ours_rp_rtr_data_quality_items{stage=\"after_slurm\",type=\"vrp\"}",
"legendFormat": "after_slurm vrp",
"refId": "B"
},
{
"expr": "ours_rp_rtr_data_quality_items{stage=\"after_slurm\",type=\"aspa\"}",
"legendFormat": "after_slurm aspa",
"refId": "C"
}
]
},
{
"id": 7,
"title": "SLURM Filters / Assertions",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 4
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_rtr_slurm_filters",
"legendFormat": "filter {{type}}",
"refId": "A"
},
{
"expr": "ours_rp_rtr_slurm_assertions",
"legendFormat": "assert {{type}}",
"refId": "B"
}
]
},
{
"id": 8,
"title": "Cache Ready / Delta Window",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 12
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_rtr_cache_ready",
"legendFormat": "ready",
"refId": "A"
},
{
"expr": "ours_rp_rtr_cache_delta_window_length",
"legendFormat": "v{{version}} length",
"refId": "B"
}
]
},
{
"id": 9,
"title": "Cache Snapshot Items",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 12
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_rtr_cache_snapshot_items",
"legendFormat": "v{{version}} {{type}}",
"refId": "A"
}
]
},
{
"id": 10,
"title": "Latest Delta Items",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 12
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_rtr_cache_delta_items",
"legendFormat": "v{{version}} {{direction}} {{type}}",
"refId": "A"
}
]
},
{
"id": 11,
"title": "Connections",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 20
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_rtr_active_connections",
"legendFormat": "active",
"refId": "A"
},
{
"expr": "ours_rp_rtr_connections",
"legendFormat": "{{transport}}",
"refId": "B"
}
]
},
{
"id": 12,
"title": "Connection Utilization / Max",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 20
},
"fieldConfig": {
"defaults": {
"min": 0
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": ".*utilization.*"
},
"properties": [
{
"id": "unit",
"value": "percentunit"
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": ".*max.*"
},
"properties": [
{
"id": "unit",
"value": "short"
}
]
}
]
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_rtr_connection_utilization",
"legendFormat": "utilization",
"refId": "A"
},
{
"expr": "ours_rp_rtr_max_connections",
"legendFormat": "max",
"refId": "B"
}
]
},
{
"id": 13,
"title": "Report Age",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 20
},
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_rtr_source_report_age_seconds",
"legendFormat": "source",
"refId": "A"
},
{
"expr": "ours_rp_rtr_runtime_report_age_seconds",
"legendFormat": "runtime",
"refId": "B"
},
{
"expr": "ours_rp_rtr_clients_report_age_seconds",
"legendFormat": "clients",
"refId": "C"
}
]
}
],
"refresh": "10s",
"schemaVersion": 40,
"tags": [
"rpki",
"inter-rp",
"routinator"
],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "browser",
"title": "RTR Service Overview",
"uid": "ours-rp-rtr-overview",
"version": 1
}

875
monitor/grafana/dashboards/ours-rp-soak-overview.json
View File

@ -1,875 +0,0 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 0,
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 0,
"w": 6,
"h": 4
},
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_cir_trust_anchors",
"legendFormat": "RIRs",
"refId": "A"
}
],
"title": "Current Run RIRs",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"x": 6,
"y": 0,
"w": 6,
"h": 4
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_run_duration_seconds",
"legendFormat": "wall",
"refId": "A"
}
],
"title": "Latest Wall Time",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 0,
"w": 6,
"h": 4
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_run_max_rss_bytes",
"legendFormat": "rss",
"refId": "A"
}
],
"title": "Latest Max RSS",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 0,
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 18,
"y": 0,
"w": 6,
"h": 4
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_publication_points",
"legendFormat": "publication points",
"refId": "A"
}
],
"title": "Publication Points",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 0,
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 4,
"w": 6,
"h": 4
},
"id": 9,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_run_sequence",
"legendFormat": "seq",
"refId": "A"
}
],
"title": "Latest Run Sequence",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 2,
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "orange",
"value": 90
},
{
"color": "green",
"value": 98
}
]
}
},
"overrides": []
},
"gridPos": {
"x": 6,
"y": 4,
"w": 6,
"h": 4
},
"id": 10,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "100 * sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"publication_point_cache\"}) / sum by (job, instance, exported_instance) (ours_rp_publication_points)",
"legendFormat": "PP cache hit ratio",
"refId": "A"
}
],
"title": "Latest PP Cache Hit Ratio",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none",
"decimals": 0
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 4,
"w": 6,
"h": 4
},
"id": 11,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_vrps{kind=\"total\"}",
"legendFormat": "VRPs raw",
"refId": "A"
}
],
"title": "VRPs",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none",
"decimals": 0
},
"overrides": []
},
"gridPos": {
"x": 18,
"y": 4,
"w": 6,
"h": 4
},
"id": 12,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_vaps",
"legendFormat": "VAPs",
"refId": "A"
}
],
"title": "VAPs",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"id": 5,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_run_duration_seconds",
"legendFormat": "wall",
"refId": "A"
},
{
"expr": "ours_rp_stage_duration_seconds{stage=\"validation\"}",
"legendFormat": "validation",
"refId": "B"
}
],
"title": "Run / Validation Duration",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 0,
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"id": 6,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_vrps{kind=\"total\"}",
"legendFormat": "VRPs raw",
"refId": "A"
},
{
"expr": "ours_rp_vrps{kind=\"unique\"}",
"legendFormat": "VRPs unique",
"refId": "D"
},
{
"expr": "ours_rp_vaps",
"legendFormat": "VAPs",
"refId": "B"
},
{
"expr": "ours_rp_cir_objects",
"legendFormat": "CIR objects",
"refId": "C"
}
],
"title": "Output and Input Sizes",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 0,
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"id": 8,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_large_publication_points",
"legendFormat": "> {{object_count_gt}} objects",
"refId": "A"
}
],
"title": "Large Publication Points by Object Count",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"id": 13,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"validation\"}",
"legendFormat": "validation",
"refId": "A"
},
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"report_write\"}",
"legendFormat": "report write",
"refId": "E"
},
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"ccr_write\"}",
"legendFormat": "ccr write",
"refId": "F"
},
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"cir_write\"}",
"legendFormat": "cir write",
"refId": "G"
}
],
"title": "Output Stage Durations",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "bytes",
"decimals": 2
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"id": 14,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_run_max_rss_bytes",
"legendFormat": "Max RSS",
"refId": "A"
}
],
"title": "Max RSS Over Time",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "percent",
"decimals": 2,
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "orange",
"value": 90
},
{
"color": "green",
"value": 98
}
]
}
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"id": 17,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"min",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "100 * sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"publication_point_cache\"}) / sum by (job, instance, exported_instance) (ours_rp_publication_points)",
"legendFormat": "PP cache hit ratio",
"refId": "A"
}
],
"title": "PP Cache Hit Ratio",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "bytes",
"decimals": 2
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 32,
"w": 24,
"h": 8
},
"id": 15,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_state_db_size_bytes",
"legendFormat": "{{db}}",
"refId": "A"
}
],
"title": "State DB Size Over Time",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none",
"decimals": 0
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 40,
"w": 24,
"h": 8
},
"id": 16,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_state_db_files",
"legendFormat": "{{db}}",
"refId": "A"
}
],
"title": "State DB File Count Over Time",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none",
"decimals": 0,
"min": 0
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 48,
"w": 24,
"h": 8
},
"id": 18,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"fresh\"})",
"legendFormat": "fresh pp",
"refId": "A"
},
{
"expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"roa\"})",
"legendFormat": "fresh roa",
"refId": "B"
},
{
"expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"manifest\"})",
"legendFormat": "fresh mft",
"refId": "C"
},
{
"expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"certificate\"})",
"legendFormat": "fresh crt",
"refId": "D"
},
{
"expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"crl\"})",
"legendFormat": "fresh crl",
"refId": "E"
}
],
"title": "Fresh PP / Object Counts by Run",
"type": "timeseries"
}
],
"refresh": "5s",
"schemaVersion": 40,
"tags": [
"ours-rp",
"rpki",
"soak"
],
"templating": {
"list": []
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Ours RP Soak Overview",
"uid": "ours-rp-soak-overview",
"version": 4,
"weekStart": ""
}

12
monitor/grafana/provisioning/dashboards/dashboard.yml
View File

@ -1,12 +0,0 @@
apiVersion: 1
providers:
- name: ours-rp
orgId: 1
folder: Ours RP
type: file
disableDeletion: false
updateIntervalSeconds: 10
allowUiUpdates: true
options:
path: /var/lib/grafana/dashboards

10
monitor/grafana/provisioning/datasources/prometheus.yml
View File

@ -1,10 +0,0 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: Prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: true

20
monitor/prometheus/prometheus.yml
View File

@ -1,20 +0,0 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
scrape_configs:
- job_name: ours-rp-artifact-metrics
metrics_path: /metrics
static_configs:
- targets:
- host.docker.internal:9556
labels:
rp: ours-rp
source: artifact-sidecar
- job_name: ours-rp-inter-rp-metrics
metrics_path: /metrics
static_configs:
- targets:
- host.docker.internal:9557
labels:
source: inter-rp-sidecar

70
scripts/benchmark/README.md
View File

@ -1,70 +0,0 @@
# RPKI Benchmarks (Stage2, selected_der_v2)
This directory contains a reproducible, one-click benchmark to measure **decode + profile validate**
performance for all supported object types and compare **OURS** against the **Routinator baseline**
(`rpki` crate `=0.19.1` with `repository` feature).
## What it measures
Dataset:
- Fixtures: `rpki/tests/benchmark/selected_der_v2/`
- Objects: `cer`, `crl`, `manifest` (`.mft`), `roa`, `aspa` (`.asa`)
- Samples: 10 quantiles per type (`min/p01/p10/p25/p50/p75/p90/p95/p99/max`) → 50 files total
Metrics:
- **decode+validate**: `decode_der` (parse + profile validate) for each object file
- **landing** (OURS only): `PackFile::from_bytes_compute_sha256` + CBOR encode + `RocksDB put_raw`
- **compare**: ratio `ours_ns/op ÷ rout_ns/op` for decode+validate
## Default benchmark settings
Both OURS and Routinator baseline use the same run settings:
- warmup: `10` iterations
- rounds: `3`
- adaptive loop target: `min_round_ms=200` (with an internal max of `1_000_000` iters)
- strict DER: `true` (baseline)
- cert inspect: `false` (baseline)
You can override the settings via environment variables in the runner script:
- `BENCH_WARMUP_ITERS` (default `10`)
- `BENCH_ROUNDS` (default `3`)
- `BENCH_MIN_ROUND_MS` (default `200`)
## One-click run (OURS + Routinator compare)
From the `rpki/` crate directory:
```bash
./scripts/benchmark/run_stage2_selected_der_v2_release.sh
```
Outputs are written under:
- `rpki/target/bench/`
- OURS decode+validate: `stage2_selected_der_v2_decode_release_<TS>.{md,csv}`
- OURS landing: `stage2_selected_der_v2_landing_release_<TS>.{md,csv}`
- Routinator: `stage2_selected_der_v2_routinator_decode_release_<TS>.{md,csv}`
- Compare: `stage2_selected_der_v2_compare_ours_vs_routinator_decode_release_<TS>.{md,csv}`
- Summary: `stage2_selected_der_v2_compare_summary_<TS>.md`
### Why decode and landing are separated
The underlying benchmark can run in `BENCH_MODE=both`, but the **landing** part writes to RocksDB
and may trigger background work (e.g., compactions) that can **skew subsequent decode timings**.
For a fair OURS-vs-Routinator comparison, the runner script:
- runs `BENCH_MODE=decode_validate` for comparison, and
- runs `BENCH_MODE=landing` separately for landing-only numbers.
## Notes
- The Routinator baseline benchmark is implemented in-repo under:
- `rpki/benchmark/routinator_object_bench/`
- It pins `rpki = "=0.19.1"` in its `Cargo.toml`.
- This benchmark is implemented as an `#[ignore]` integration test:
- `rpki/tests/bench_stage2_decode_profile_selected_der_v2.rs`
- The runner script invokes it with `cargo test --release ... -- --ignored --nocapture`.

123
scripts/benchmark/run_stage2_selected_der_v2_release.sh
View File

@ -1,123 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Stage2 (selected_der_v2) decode+profile validate benchmark.
# Runs:
# 1) OURS decode+validate benchmark and writes MD/CSV.
# 2) OURS landing benchmark and writes MD/CSV.
# 3) Routinator baseline decode benchmark (rpki crate =0.19.1).
# 4) Produces a joined compare CSV/MD and a short geomean summary.
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
cd "$ROOT_DIR"
OUT_DIR="$ROOT_DIR/target/bench"
mkdir -p "$OUT_DIR"
TS="$(date -u +%Y%m%dT%H%M%SZ)"
WARMUP_ITERS="${BENCH_WARMUP_ITERS:-10}"
ROUNDS="${BENCH_ROUNDS:-3}"
MIN_ROUND_MS="${BENCH_MIN_ROUND_MS:-200}"
OURS_MD="$OUT_DIR/stage2_selected_der_v2_decode_release_${TS}.md"
OURS_CSV="$OUT_DIR/stage2_selected_der_v2_decode_release_${TS}.csv"
OURS_LANDING_MD="$OUT_DIR/stage2_selected_der_v2_landing_release_${TS}.md"
OURS_LANDING_CSV="$OUT_DIR/stage2_selected_der_v2_landing_release_${TS}.csv"
ROUT_MD="$OUT_DIR/stage2_selected_der_v2_routinator_decode_release_${TS}.md"
ROUT_CSV="$OUT_DIR/stage2_selected_der_v2_routinator_decode_release_${TS}.csv"
COMPARE_MD="$OUT_DIR/stage2_selected_der_v2_compare_ours_vs_routinator_decode_release_${TS}.md"
COMPARE_CSV="$OUT_DIR/stage2_selected_der_v2_compare_ours_vs_routinator_decode_release_${TS}.csv"
SUMMARY_MD="$OUT_DIR/stage2_selected_der_v2_compare_summary_${TS}.md"
echo "[1/4] OURS: decode+validate benchmark (release)..." >&2
BENCH_MODE="decode_validate" \
BENCH_WARMUP_ITERS="$WARMUP_ITERS" \
BENCH_ROUNDS="$ROUNDS" \
BENCH_MIN_ROUND_MS="$MIN_ROUND_MS" \
BENCH_OUT_MD="$OURS_MD" \
BENCH_OUT_CSV="$OURS_CSV" \
cargo test --release --test bench_stage2_decode_profile_selected_der_v2 -- --ignored --nocapture >/dev/null
echo "[2/4] OURS: landing benchmark (release)..." >&2
BENCH_MODE="landing" \
BENCH_WARMUP_ITERS="$WARMUP_ITERS" \
BENCH_ROUNDS="$ROUNDS" \
BENCH_MIN_ROUND_MS="$MIN_ROUND_MS" \
BENCH_OUT_MD_LANDING="$OURS_LANDING_MD" \
BENCH_OUT_CSV_LANDING="$OURS_LANDING_CSV" \
cargo test --release --test bench_stage2_decode_profile_selected_der_v2 -- --ignored --nocapture >/dev/null
echo "[3/4] Routinator baseline + compare join..." >&2
OURS_CSV="$OURS_CSV" \
ROUT_CSV="$ROUT_CSV" \
ROUT_MD="$ROUT_MD" \
COMPARE_CSV="$COMPARE_CSV" \
COMPARE_MD="$COMPARE_MD" \
WARMUP_ITERS="$WARMUP_ITERS" \
ROUNDS="$ROUNDS" \
MIN_ROUND_MS="$MIN_ROUND_MS" \
scripts/stage2_perf_compare_m4.sh >/dev/null
echo "[4/4] Summary (geomean ratios)..." >&2
python3 - "$COMPARE_CSV" "$SUMMARY_MD" <<'PY'
import csv
import math
import sys
from pathlib import Path
from datetime import datetime, timezone
in_csv = Path(sys.argv[1])
out_md = Path(sys.argv[2])
rows = list(csv.DictReader(in_csv.open(newline="")))
ratios = {}
for r in rows:
ratios.setdefault(r["type"], []).append(float(r["ratio_ours_over_rout"]))
def geomean(vals):
return math.exp(sum(math.log(v) for v in vals) / len(vals))
def p50(vals):
v = sorted(vals)
n = len(v)
if n % 2 == 1:
return v[n // 2]
return (v[n // 2 - 1] + v[n // 2]) / 2.0
all_vals = [float(r["ratio_ours_over_rout"]) for r in rows]
types = ["all"] + sorted(ratios.keys())
now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
lines = []
lines.append("# Stage2 selected_der_v2 compare summary (release)\n\n")
lines.append(f"- recorded_at_utc: `{now}`\n")
lines.append(f"- inputs_csv: `{in_csv}`\n\n")
lines.append("| type | n | min | p50 | geomean | max | >1 count |\n")
lines.append("|---|---:|---:|---:|---:|---:|---:|\n")
for t in types:
vals = all_vals if t == "all" else ratios[t]
vals_sorted = sorted(vals)
lines.append(
f"| {t} | {len(vals_sorted)} | {vals_sorted[0]:.4f} | {p50(vals_sorted):.4f} | "
f"{geomean(vals_sorted):.4f} | {vals_sorted[-1]:.4f} | {sum(1 for v in vals_sorted if v>1.0)} |\n"
)
out_md.write_text("".join(lines), encoding="utf-8")
print(out_md)
PY
echo "Done." >&2
echo "- OURS decode MD: $OURS_MD" >&2
echo "- OURS decode CSV: $OURS_CSV" >&2
echo "- OURS landing MD: $OURS_LANDING_MD" >&2
echo "- OURS landing CSV: $OURS_LANDING_CSV" >&2
echo "- Routinator: $ROUT_MD" >&2
echo "- Compare MD: $COMPARE_MD" >&2
echo "- Compare CSV: $COMPARE_CSV" >&2
echo "- Summary MD: $SUMMARY_MD" >&2

56
scripts/cir/README.md
View File

@ -1,56 +0,0 @@
# CIR Scripts
## `cir-rsync-wrapper`
一个用于 CIR 黑盒 replay 的 rsync wrapper。
### 环境变量
- `REAL_RSYNC_BIN`
- 真实 rsync 二进制路径
- 默认优先 `/usr/bin/rsync`
- `CIR_MIRROR_ROOT`
- 本地镜像树根目录
- 当命令行中出现 `rsync://...` source 时必需
### 语义
- 仅改写 `rsync://host/path` 类型参数
- 其它参数原样透传给真实 rsync
- 改写目标：
- `rsync://example.net/repo/a.roa`
- →
- `<CIR_MIRROR_ROOT>/example.net/repo/a.roa`
### 兼容目标
- Routinator `--rsync-command`
- `rpki-client -e rsync_prog`
## 其它脚本
- `run_cir_replay_ours.sh`
- `run_cir_replay_routinator.sh`
- `run_cir_replay_rpki_client.sh`
- `run_cir_replay_matrix.sh`
## `cir-local-link-sync.py`
当 `CIR_LOCAL_LINK_MODE=1` 且 wrapper 检测到 source 已经被改写为本地 mirror 路径时，
wrapper 不再调用真实 `rsync`，而是调用这个 helper 完成：
- `hardlink` 优先的本地树同步
- 失败时回退到 copy
- 支持 `--delete`
`run_cir_replay_matrix.sh` 会顺序执行：
- `ours`
- Routinator
- `rpki-client`
并汇总生成：
- `summary.json`
- `summary.md`
- `detail.md`

136
scripts/cir/cir-local-link-sync.py
View File

@ -1,136 +0,0 @@
#!/usr/bin/env python3
import argparse
import errno
import os
import shutil
from pathlib import Path
def _same_inode(src: Path, dst: Path) -> bool:
try:
src_stat = src.stat()
dst_stat = dst.stat()
except FileNotFoundError:
return False
return (src_stat.st_dev, src_stat.st_ino) == (dst_stat.st_dev, dst_stat.st_ino)
def _remove_path(path: Path) -> None:
if not path.exists() and not path.is_symlink():
return
if path.is_dir() and not path.is_symlink():
shutil.rmtree(path)
else:
path.unlink()
def _prune_empty_dirs(root: Path) -> None:
if not root.exists():
return
for path in sorted((p for p in root.rglob("*") if p.is_dir()), key=lambda p: len(p.parts), reverse=True):
try:
path.rmdir()
except OSError:
pass
def _link_or_copy(src: Path, dst: Path) -> str:
dst.parent.mkdir(parents=True, exist_ok=True)
if dst.exists() or dst.is_symlink():
if _same_inode(src, dst):
return "reused"
_remove_path(dst)
try:
os.link(src, dst)
return "linked"
except OSError as err:
if err.errno not in (errno.EXDEV, errno.EPERM, errno.EMLINK, errno.ENOTSUP, errno.EACCES):
raise
shutil.copy2(src, dst)
return "copied"
def _file_map(src_arg: str, dest_arg: str) -> tuple[Path, dict[str, Path]]:
src = Path(src_arg.rstrip(os.sep))
if not src.exists():
raise FileNotFoundError(src)
mapping: dict[str, Path] = {}
if src.is_dir():
copy_contents = src_arg.endswith(os.sep)
if copy_contents:
root = src
for path in root.rglob("*"):
if path.is_file():
mapping[path.relative_to(root).as_posix()] = path
else:
root = src
base = src.name
for path in root.rglob("*"):
if path.is_file():
rel = Path(base) / path.relative_to(root)
mapping[rel.as_posix()] = path
else:
dest_path = Path(dest_arg)
if dest_arg.endswith(os.sep) or dest_path.is_dir():
mapping[src.name] = src
else:
mapping[dest_path.name] = src
return Path(dest_arg), mapping
def sync_local_tree(src_arg: str, dst_arg: str, delete: bool) -> dict[str, int]:
dst_root, mapping = _file_map(src_arg, dst_arg)
dst_root.mkdir(parents=True, exist_ok=True)
expected = {dst_root / rel for rel in mapping.keys()}
deleted = 0
if delete and dst_root.exists():
for path in sorted(dst_root.rglob("*"), key=lambda p: len(p.parts), reverse=True):
if path.is_dir():
continue
if path not in expected:
_remove_path(path)
deleted += 1
_prune_empty_dirs(dst_root)
linked = 0
copied = 0
reused = 0
for rel, src in mapping.items():
dst = dst_root / rel
result = _link_or_copy(src, dst)
if result == "linked":
linked += 1
elif result == "copied":
copied += 1
else:
reused += 1
return {
"files": len(mapping),
"linked": linked,
"copied": copied,
"reused": reused,
"deleted": deleted,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Sync a local CIR mirror tree using hardlinks when possible.")
parser.add_argument("--delete", action="store_true", help="Delete target files not present in source")
parser.add_argument("source")
parser.add_argument("dest")
args = parser.parse_args()
summary = sync_local_tree(args.source, args.dest, args.delete)
print(
"local-link-sync files={files} linked={linked} copied={copied} reused={reused} deleted={deleted}".format(
**summary
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())

127
scripts/cir/cir-rsync-wrapper
View File

@ -1,127 +0,0 @@
#!/usr/bin/env python3
import os
import shutil
import sys
from pathlib import Path
from urllib.parse import urlparse
def real_rsync_bin() -> str:
env = os.environ.get("REAL_RSYNC_BIN")
if env:
return env
default = "/usr/bin/rsync"
if Path(default).exists():
return default
found = shutil.which("rsync")
if found:
return found
raise SystemExit("cir-rsync-wrapper: REAL_RSYNC_BIN is not set and rsync was not found")
def rewrite_arg(arg: str, mirror_root: str | None) -> str:
if not arg.startswith("rsync://"):
return arg
if not mirror_root:
raise SystemExit(
"cir-rsync-wrapper: CIR_MIRROR_ROOT is required when an rsync:// source is present"
)
parsed = urlparse(arg)
if parsed.scheme != "rsync" or not parsed.hostname:
raise SystemExit(f"cir-rsync-wrapper: invalid rsync URI: {arg}")
path = parsed.path.lstrip("/")
local = Path(mirror_root).resolve() / parsed.hostname
if path:
local = local / path
local_str = str(local)
if local.exists() and local.is_dir() and not local_str.endswith("/"):
local_str += "/"
elif arg.endswith("/") and not local_str.endswith("/"):
local_str += "/"
return local_str
def filter_args(args: list[str]) -> list[str]:
mirror_root = os.environ.get("CIR_MIRROR_ROOT")
rewritten_any = any(arg.startswith("rsync://") for arg in args)
out: list[str] = []
i = 0
while i < len(args):
arg = args[i]
if rewritten_any:
if arg == "--address":
i += 2
continue
if arg.startswith("--address="):
i += 1
continue
if arg == "--contimeout":
i += 2
continue
if arg.startswith("--contimeout="):
i += 1
continue
out.append(rewrite_arg(arg, mirror_root))
i += 1
return out
def local_link_mode_enabled() -> bool:
value = os.environ.get("CIR_LOCAL_LINK_MODE", "")
return value.lower() in {"1", "true", "yes", "on"}
def extract_source_and_dest(args: list[str]) -> tuple[str, str]:
expects_value = {
"--timeout",
"--min-size",
"--max-size",
"--include",
"--exclude",
"--compare-dest",
}
positionals: list[str] = []
i = 0
while i < len(args):
arg = args[i]
if arg in expects_value:
i += 2
continue
if any(arg.startswith(prefix + "=") for prefix in expects_value):
i += 1
continue
if arg.startswith("-"):
i += 1
continue
positionals.append(arg)
i += 1
if len(positionals) < 2:
raise SystemExit("cir-rsync-wrapper: expected source and destination arguments")
return positionals[-2], positionals[-1]
def maybe_exec_local_link_sync(args: list[str], rewritten_any: bool) -> None:
if not rewritten_any or not local_link_mode_enabled():
return
source, dest = extract_source_and_dest(args)
if source.startswith("rsync://"):
raise SystemExit("cir-rsync-wrapper: expected rewritten local source for CIR_LOCAL_LINK_MODE")
helper = Path(__file__).with_name("cir-local-link-sync.py")
cmd = [sys.executable, str(helper)]
if "--delete" in args:
cmd.append("--delete")
cmd.extend([source, dest])
os.execv(sys.executable, cmd)
def main() -> int:
args = sys.argv[1:]
rewritten_any = any(arg.startswith("rsync://") for arg in args)
rewritten = filter_args(args)
maybe_exec_local_link_sync(rewritten, rewritten_any)
os.execv(real_rsync_bin(), [real_rsync_bin(), *rewritten])
return 127
if __name__ == "__main__":
raise SystemExit(main())

32
scripts/cir/fetch_cir_sequence_from_remote.sh
View File

@ -1,32 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/fetch_cir_sequence_from_remote.sh \
--ssh-target <user@host> \
--remote-path <path> \
--local-path <path>
EOF
}
SSH_TARGET=""
REMOTE_PATH=""
LOCAL_PATH=""
while [[ $# -gt 0 ]]; do
case "$1" in
--ssh-target) SSH_TARGET="$2"; shift 2 ;;
--remote-path) REMOTE_PATH="$2"; shift 2 ;;
--local-path) LOCAL_PATH="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$SSH_TARGET" && -n "$REMOTE_PATH" && -n "$LOCAL_PATH" ]] || { usage >&2; exit 2; }
mkdir -p "$(dirname "$LOCAL_PATH")"
rsync -a "$SSH_TARGET:$REMOTE_PATH/" "$LOCAL_PATH/"
echo "done: $LOCAL_PATH"

50
scripts/cir/json_to_vaps_csv.py
View File

@ -1,50 +0,0 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import csv
import json
from pathlib import Path
def normalize_asn(value: str | int) -> str:
text = str(value).strip().upper()
if text.startswith("AS"):
text = text[2:]
return f"AS{int(text)}"
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--input", required=True, type=Path)
parser.add_argument("--csv-out", required=True, type=Path)
args = parser.parse_args()
obj = json.loads(args.input.read_text(encoding="utf-8"))
rows: list[tuple[str, str, str]] = []
for aspa in obj.get("aspas", []):
providers = sorted(
{normalize_asn(item) for item in aspa.get("providers", [])},
key=lambda s: int(s[2:]),
)
rows.append(
(
normalize_asn(aspa["customer"]),
";".join(providers),
str(aspa.get("ta", "")).strip().lower(),
)
)
rows.sort(key=lambda row: (int(row[0][2:]), row[1], row[2]))
args.csv_out.parent.mkdir(parents=True, exist_ok=True)
with args.csv_out.open("w", encoding="utf-8", newline="") as fh:
writer = csv.writer(fh)
writer.writerow(["Customer ASN", "Providers", "Trust Anchor"])
writer.writerows(rows)
print(args.csv_out)
return 0
if __name__ == "__main__":
raise SystemExit(main())

77
scripts/cir/run_cir_drop_sequence.sh
View File

@ -1,77 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/run_cir_drop_sequence.sh \
--sequence-root <path> \
[--drop-bin <path>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
SEQUENCE_ROOT=""
DROP_BIN="${DROP_BIN:-$ROOT_DIR/target/release/cir_drop_report}"
while [[ $# -gt 0 ]]; do
case "$1" in
--sequence-root) SEQUENCE_ROOT="$2"; shift 2 ;;
--drop-bin) DROP_BIN="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$SEQUENCE_ROOT" ]] || { usage >&2; exit 2; }
python3 - <<'PY' "$SEQUENCE_ROOT" "$DROP_BIN"
import json
import subprocess
import sys
from pathlib import Path
sequence_root = Path(sys.argv[1]).resolve()
drop_bin = sys.argv[2]
sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
repo_bytes_db = sequence_root / sequence["repoBytesDbPath"]
summaries = []
for step in sequence["steps"]:
step_id = step["stepId"]
out_dir = sequence_root / "drop" / step_id
out_dir.mkdir(parents=True, exist_ok=True)
cmd = [
drop_bin,
"--cir",
str(sequence_root / step["cirPath"]),
"--ccr",
str(sequence_root / step["ccrPath"]),
"--report-json",
str(sequence_root / step["reportPath"]),
"--json-out",
str(out_dir / "drop.json"),
"--md-out",
str(out_dir / "drop.md"),
]
cmd.extend(["--repo-bytes-db", str(repo_bytes_db)])
proc = subprocess.run(cmd, capture_output=True, text=True)
if proc.returncode != 0:
raise SystemExit(
f"drop report failed for {step_id}: stdout={proc.stdout} stderr={proc.stderr}"
)
result = json.loads((out_dir / "drop.json").read_text(encoding="utf-8"))
summaries.append(
{
"stepId": step_id,
"droppedVrpCount": result["summary"]["droppedVrpCount"],
"droppedObjectCount": result["summary"]["droppedObjectCount"],
"reportPath": str(out_dir / "drop.json"),
}
)
summary = {"version": 1, "steps": summaries}
(sequence_root / "drop-summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
PY
echo "done: $SEQUENCE_ROOT"

173
scripts/cir/run_cir_record_full_delta.sh
View File

@ -1,173 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/run_cir_record_full_delta.sh \
--out-dir <path> \
--tal-path <path> \
--ta-path <path> \
--cir-tal-uri <url> \
--payload-replay-archive <path> \
--payload-replay-locks <path> \
--payload-base-archive <path> \
--payload-base-locks <path> \
--payload-delta-archive <path> \
--payload-delta-locks <path> \
[--base-validation-time <rfc3339>] \
[--delta-validation-time <rfc3339>] \
[--max-depth <n>] \
[--max-instances <n>] \
[--rpki-bin <path>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
OUT_DIR=""
TAL_PATH=""
TA_PATH=""
CIR_TAL_URI=""
PAYLOAD_REPLAY_ARCHIVE=""
PAYLOAD_REPLAY_LOCKS=""
PAYLOAD_BASE_ARCHIVE=""
PAYLOAD_BASE_LOCKS=""
PAYLOAD_DELTA_ARCHIVE=""
PAYLOAD_DELTA_LOCKS=""
BASE_VALIDATION_TIME=""
DELTA_VALIDATION_TIME=""
MAX_DEPTH=0
MAX_INSTANCES=1
RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
while [[ $# -gt 0 ]]; do
case "$1" in
--out-dir) OUT_DIR="$2"; shift 2 ;;
--tal-path) TAL_PATH="$2"; shift 2 ;;
--ta-path) TA_PATH="$2"; shift 2 ;;
--cir-tal-uri) CIR_TAL_URI="$2"; shift 2 ;;
--payload-replay-archive) PAYLOAD_REPLAY_ARCHIVE="$2"; shift 2 ;;
--payload-replay-locks) PAYLOAD_REPLAY_LOCKS="$2"; shift 2 ;;
--payload-base-archive) PAYLOAD_BASE_ARCHIVE="$2"; shift 2 ;;
--payload-base-locks) PAYLOAD_BASE_LOCKS="$2"; shift 2 ;;
--payload-delta-archive) PAYLOAD_DELTA_ARCHIVE="$2"; shift 2 ;;
--payload-delta-locks) PAYLOAD_DELTA_LOCKS="$2"; shift 2 ;;
--base-validation-time) BASE_VALIDATION_TIME="$2"; shift 2 ;;
--delta-validation-time) DELTA_VALIDATION_TIME="$2"; shift 2 ;;
--max-depth) MAX_DEPTH="$2"; shift 2 ;;
--max-instances) MAX_INSTANCES="$2"; shift 2 ;;
--rpki-bin) RPKI_BIN="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$OUT_DIR" && -n "$TAL_PATH" && -n "$TA_PATH" && -n "$CIR_TAL_URI" && -n "$PAYLOAD_REPLAY_ARCHIVE" && -n "$PAYLOAD_REPLAY_LOCKS" && -n "$PAYLOAD_BASE_ARCHIVE" && -n "$PAYLOAD_BASE_LOCKS" && -n "$PAYLOAD_DELTA_ARCHIVE" && -n "$PAYLOAD_DELTA_LOCKS" ]] || {
usage >&2
exit 2
}
if [[ ! -x "$RPKI_BIN" ]]; then
(
cd "$ROOT_DIR"
cargo build --release --bin rpki
)
fi
resolve_validation_time() {
local path="$1"
python3 - <<'PY' "$path"
import json, sys
print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['validationTime'])
PY
}
if [[ -z "$BASE_VALIDATION_TIME" ]]; then
BASE_VALIDATION_TIME="$(resolve_validation_time "$PAYLOAD_REPLAY_LOCKS")"
fi
if [[ -z "$DELTA_VALIDATION_TIME" ]]; then
DELTA_VALIDATION_TIME="$(resolve_validation_time "$PAYLOAD_DELTA_LOCKS")"
fi
rm -rf "$OUT_DIR"
mkdir -p "$OUT_DIR/full" "$OUT_DIR/delta-001"
REPO_BYTES_DB="$OUT_DIR/repo-bytes.db"
FULL_DB="$OUT_DIR/full/db"
DELTA_DB="$OUT_DIR/delta-001/db"
"$RPKI_BIN" \
--db "$FULL_DB" \
--tal-path "$TAL_PATH" \
--ta-path "$TA_PATH" \
--payload-replay-archive "$PAYLOAD_REPLAY_ARCHIVE" \
--payload-replay-locks "$PAYLOAD_REPLAY_LOCKS" \
--validation-time "$BASE_VALIDATION_TIME" \
--max-depth "$MAX_DEPTH" \
--max-instances "$MAX_INSTANCES" \
--ccr-out "$OUT_DIR/full/result.ccr" \
--report-json "$OUT_DIR/full/report.json" \
--cir-enable \
--cir-out "$OUT_DIR/full/input.cir" \
--repo-bytes-db "$REPO_BYTES_DB" \
--cir-tal-uri "$CIR_TAL_URI" \
>"$OUT_DIR/full/run.stdout.log" 2>"$OUT_DIR/full/run.stderr.log"
"$RPKI_BIN" \
--db "$DELTA_DB" \
--tal-path "$TAL_PATH" \
--ta-path "$TA_PATH" \
--payload-base-archive "$PAYLOAD_BASE_ARCHIVE" \
--payload-base-locks "$PAYLOAD_BASE_LOCKS" \
--payload-delta-archive "$PAYLOAD_DELTA_ARCHIVE" \
--payload-delta-locks "$PAYLOAD_DELTA_LOCKS" \
--payload-base-validation-time "$BASE_VALIDATION_TIME" \
--validation-time "$DELTA_VALIDATION_TIME" \
--max-depth "$MAX_DEPTH" \
--max-instances "$MAX_INSTANCES" \
--ccr-out "$OUT_DIR/delta-001/result.ccr" \
--report-json "$OUT_DIR/delta-001/report.json" \
--cir-enable \
--cir-out "$OUT_DIR/delta-001/input.cir" \
--repo-bytes-db "$REPO_BYTES_DB" \
--cir-tal-uri "$CIR_TAL_URI" \
>"$OUT_DIR/delta-001/run.stdout.log" 2>"$OUT_DIR/delta-001/run.stderr.log"
python3 - <<'PY' "$OUT_DIR" "$BASE_VALIDATION_TIME" "$DELTA_VALIDATION_TIME"
import json
import os
import sys
from pathlib import Path
out = Path(sys.argv[1])
base_validation_time = sys.argv[2]
delta_validation_time = sys.argv[3]
summary = {
"version": 1,
"kind": "cir_pair",
"baseValidationTime": base_validation_time,
"deltaValidationTime": delta_validation_time,
"repoBytesDbPath": "repo-bytes.db",
"steps": [
{
"kind": "full",
"cirPath": "full/input.cir",
"ccrPath": "full/result.ccr",
"reportPath": "full/report.json",
},
{
"kind": "delta",
"cirPath": "delta-001/input.cir",
"ccrPath": "delta-001/result.ccr",
"reportPath": "delta-001/report.json",
"previous": "full",
},
],
"repoBytesDbExists": (out / "repo-bytes.db").exists(),
}
(out / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
PY
echo "done: $OUT_DIR"

129
scripts/cir/run_cir_record_sequence_multi_rir_offline.sh
View File

@ -1,129 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/run_cir_record_sequence_multi_rir_offline.sh \
[--bundle-root <path>] \
[--rir <afrinic,apnic,arin,lacnic,ripe>] \
[--delta-count <n>] \
[--full-repo] \
[--out-root <path>] \
[--rpki-bin <path>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
CASE_INFO="$ROOT_DIR/scripts/payload_replay/multi_rir_case_info.py"
SINGLE_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_record_sequence_offline.sh"
BUNDLE_ROOT="/home/yuyr/dev/rust_playground/routinator/bench/multi_rir_demo/runs/20260316-112341-multi-final3"
RIRS="afrinic,apnic,arin,lacnic,ripe"
DELTA_COUNT=2
FULL_REPO=0
OUT_ROOT="$ROOT_DIR/target/replay/cir_sequence_multi_rir_offline_$(date -u +%Y%m%dT%H%M%SZ)"
RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
while [[ $# -gt 0 ]]; do
case "$1" in
--bundle-root) BUNDLE_ROOT="$2"; shift 2 ;;
--rir) RIRS="$2"; shift 2 ;;
--delta-count) DELTA_COUNT="$2"; shift 2 ;;
--full-repo) FULL_REPO=1; shift 1 ;;
--out-root) OUT_ROOT="$2"; shift 2 ;;
--rpki-bin) RPKI_BIN="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
mkdir -p "$OUT_ROOT"
SUMMARY_JSON="$OUT_ROOT/summary.json"
SUMMARY_MD="$OUT_ROOT/summary.md"
IFS=',' read -r -a RIR_ITEMS <<< "$RIRS"
for rir in "${RIR_ITEMS[@]}"; do
CASE_JSON="$(python3 "$CASE_INFO" --bundle-root "$BUNDLE_ROOT" --repo-root "$ROOT_DIR" --rir "$rir")"
TAL_PATH="$(python3 - <<'PY' "$CASE_JSON"
import json,sys
print(json.loads(sys.argv[1])['tal_path'])
PY
)"
TA_PATH="$(python3 - <<'PY' "$CASE_JSON"
import json,sys
print(json.loads(sys.argv[1])['ta_path'])
PY
)"
BASE_ARCHIVE="$(python3 - <<'PY' "$CASE_JSON"
import json,sys
print(json.loads(sys.argv[1])['base_archive'])
PY
)"
BASE_LOCKS="$(python3 - <<'PY' "$CASE_JSON"
import json,sys
print(json.loads(sys.argv[1])['base_locks'])
PY
)"
DELTA_ARCHIVE="$(python3 - <<'PY' "$CASE_JSON"
import json,sys
print(json.loads(sys.argv[1])['delta_archive'])
PY
)"
DELTA_LOCKS="$(python3 - <<'PY' "$CASE_JSON"
import json,sys
print(json.loads(sys.argv[1])['delta_locks'])
PY
)"
OUT_DIR="$OUT_ROOT/$rir"
args=(
"$SINGLE_SCRIPT"
--out-dir "$OUT_DIR" \
--tal-path "$TAL_PATH" \
--ta-path "$TA_PATH" \
--cir-tal-uri "https://example.test/$rir.tal" \
--payload-replay-archive "$BASE_ARCHIVE" \
--payload-replay-locks "$BASE_LOCKS" \
--payload-base-archive "$BASE_ARCHIVE" \
--payload-base-locks "$BASE_LOCKS" \
--payload-delta-archive "$DELTA_ARCHIVE" \
--payload-delta-locks "$DELTA_LOCKS" \
--delta-count "$DELTA_COUNT" \
--rpki-bin "$RPKI_BIN"
)
if [[ "$FULL_REPO" -ne 1 ]]; then
args+=(--max-depth 0 --max-instances 1)
else
args+=(--full-repo)
fi
"${args[@]}"
done
python3 - <<'PY' "$OUT_ROOT" "$RIRS" "$SUMMARY_JSON" "$SUMMARY_MD"
import json, sys
from pathlib import Path
out_root = Path(sys.argv[1])
rirs = [item for item in sys.argv[2].split(',') if item]
summary_json = Path(sys.argv[3])
summary_md = Path(sys.argv[4])
items = []
for rir in rirs:
root = out_root / rir
seq = json.loads((root / "sequence.json").read_text(encoding="utf-8"))
summ = json.loads((root / "summary.json").read_text(encoding="utf-8"))
items.append({
"rir": rir,
"root": str(root),
"stepCount": len(seq["steps"]),
"repoBytesDbExists": summ.get("repoBytesDbExists", False),
})
summary = {"version": 1, "rirs": items}
summary_json.write_text(json.dumps(summary, indent=2), encoding="utf-8")
lines = ["# Multi-RIR Offline CIR Sequence Summary", ""]
for item in items:
lines.append(f"- `{item['rir']}`: `stepCount={item['stepCount']}` `repoBytesDbExists={item['repoBytesDbExists']}` `root={item['root']}`")
summary_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
PY
echo "done: $OUT_ROOT"

208
scripts/cir/run_cir_record_sequence_offline.sh
View File

@ -1,208 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/run_cir_record_sequence_offline.sh \
--out-dir <path> \
--tal-path <path> \
--ta-path <path> \
--cir-tal-uri <url> \
--payload-replay-archive <path> \
--payload-replay-locks <path> \
--payload-base-archive <path> \
--payload-base-locks <path> \
--payload-delta-archive <path> \
--payload-delta-locks <path> \
[--delta-count <n>] \
[--base-validation-time <rfc3339>] \
[--delta-validation-time <rfc3339>] \
[--full-repo] \
[--max-depth <n>] \
[--max-instances <n>] \
[--rpki-bin <path>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
OUT_DIR=""
TAL_PATH=""
TA_PATH=""
CIR_TAL_URI=""
PAYLOAD_REPLAY_ARCHIVE=""
PAYLOAD_REPLAY_LOCKS=""
PAYLOAD_BASE_ARCHIVE=""
PAYLOAD_BASE_LOCKS=""
PAYLOAD_DELTA_ARCHIVE=""
PAYLOAD_DELTA_LOCKS=""
BASE_VALIDATION_TIME=""
DELTA_VALIDATION_TIME=""
DELTA_COUNT=2
FULL_REPO=0
MAX_DEPTH=0
MAX_INSTANCES=1
RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
while [[ $# -gt 0 ]]; do
case "$1" in
--out-dir) OUT_DIR="$2"; shift 2 ;;
--tal-path) TAL_PATH="$2"; shift 2 ;;
--ta-path) TA_PATH="$2"; shift 2 ;;
--cir-tal-uri) CIR_TAL_URI="$2"; shift 2 ;;
--payload-replay-archive) PAYLOAD_REPLAY_ARCHIVE="$2"; shift 2 ;;
--payload-replay-locks) PAYLOAD_REPLAY_LOCKS="$2"; shift 2 ;;
--payload-base-archive) PAYLOAD_BASE_ARCHIVE="$2"; shift 2 ;;
--payload-base-locks) PAYLOAD_BASE_LOCKS="$2"; shift 2 ;;
--payload-delta-archive) PAYLOAD_DELTA_ARCHIVE="$2"; shift 2 ;;
--payload-delta-locks) PAYLOAD_DELTA_LOCKS="$2"; shift 2 ;;
--base-validation-time) BASE_VALIDATION_TIME="$2"; shift 2 ;;
--delta-validation-time) DELTA_VALIDATION_TIME="$2"; shift 2 ;;
--delta-count) DELTA_COUNT="$2"; shift 2 ;;
--full-repo) FULL_REPO=1; shift 1 ;;
--max-depth) MAX_DEPTH="$2"; shift 2 ;;
--max-instances) MAX_INSTANCES="$2"; shift 2 ;;
--rpki-bin) RPKI_BIN="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$OUT_DIR" && -n "$TAL_PATH" && -n "$TA_PATH" && -n "$CIR_TAL_URI" && -n "$PAYLOAD_REPLAY_ARCHIVE" && -n "$PAYLOAD_REPLAY_LOCKS" && -n "$PAYLOAD_BASE_ARCHIVE" && -n "$PAYLOAD_BASE_LOCKS" && -n "$PAYLOAD_DELTA_ARCHIVE" && -n "$PAYLOAD_DELTA_LOCKS" ]] || {
usage >&2
exit 2
}
if [[ ! -x "$RPKI_BIN" ]]; then
(
cd "$ROOT_DIR"
cargo build --release --bin rpki
)
fi
resolve_validation_time() {
local path="$1"
python3 - <<'PY' "$path"
import json, sys
print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['validationTime'])
PY
}
if [[ -z "$BASE_VALIDATION_TIME" ]]; then
BASE_VALIDATION_TIME="$(resolve_validation_time "$PAYLOAD_REPLAY_LOCKS")"
fi
if [[ -z "$DELTA_VALIDATION_TIME" ]]; then
DELTA_VALIDATION_TIME="$(resolve_validation_time "$PAYLOAD_DELTA_LOCKS")"
fi
rm -rf "$OUT_DIR"
mkdir -p "$OUT_DIR/full"
REPO_BYTES_DB="$OUT_DIR/repo-bytes.db"
run_step() {
local kind="$1"
local step_dir="$2"
local db_dir="$3"
shift 3
mkdir -p "$step_dir"
local -a cmd=(
"$RPKI_BIN"
--db "$db_dir" \
--tal-path "$TAL_PATH" \
--ta-path "$TA_PATH" \
--ccr-out "$step_dir/result.ccr" \
--report-json "$step_dir/report.json" \
--cir-enable \
--cir-out "$step_dir/input.cir" \
--repo-bytes-db "$REPO_BYTES_DB" \
--cir-tal-uri "$CIR_TAL_URI"
)
if [[ "$FULL_REPO" -ne 1 ]]; then
cmd+=(--max-depth "$MAX_DEPTH" --max-instances "$MAX_INSTANCES")
fi
cmd+=("$@")
"${cmd[@]}" >"$step_dir/run.stdout.log" 2>"$step_dir/run.stderr.log"
}
run_step \
full \
"$OUT_DIR/full" \
"$OUT_DIR/full/db" \
--payload-replay-archive "$PAYLOAD_REPLAY_ARCHIVE" \
--payload-replay-locks "$PAYLOAD_REPLAY_LOCKS" \
--validation-time "$BASE_VALIDATION_TIME"
for idx in $(seq 1 "$DELTA_COUNT"); do
step_id="$(printf 'delta-%03d' "$idx")"
run_step \
delta \
"$OUT_DIR/$step_id" \
"$OUT_DIR/$step_id/db" \
--payload-base-archive "$PAYLOAD_BASE_ARCHIVE" \
--payload-base-locks "$PAYLOAD_BASE_LOCKS" \
--payload-delta-archive "$PAYLOAD_DELTA_ARCHIVE" \
--payload-delta-locks "$PAYLOAD_DELTA_LOCKS" \
--payload-base-validation-time "$BASE_VALIDATION_TIME" \
--validation-time "$DELTA_VALIDATION_TIME"
done
python3 - <<'PY' "$OUT_DIR" "$BASE_VALIDATION_TIME" "$DELTA_VALIDATION_TIME" "$DELTA_COUNT"
import json
import sys
from pathlib import Path
out = Path(sys.argv[1])
base_validation_time = sys.argv[2]
delta_validation_time = sys.argv[3]
delta_count = int(sys.argv[4])
steps = [
{
"stepId": "full",
"kind": "full",
"validationTime": base_validation_time,
"cirPath": "full/input.cir",
"ccrPath": "full/result.ccr",
"reportPath": "full/report.json",
"previousStepId": None,
}
]
previous = "full"
for idx in range(1, delta_count + 1):
step_id = f"delta-{idx:03d}"
steps.append(
{
"stepId": step_id,
"kind": "delta",
"validationTime": delta_validation_time,
"cirPath": f"{step_id}/input.cir",
"ccrPath": f"{step_id}/result.ccr",
"reportPath": f"{step_id}/report.json",
"previousStepId": previous,
}
)
previous = step_id
summary = {
"version": 1,
"kind": "cir_sequence_offline",
"repoBytesDbPath": "repo-bytes.db",
"steps": steps,
}
(out / "sequence.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
(out / "summary.json").write_text(
json.dumps(
{
"version": 1,
"stepCount": len(steps),
"repoBytesDbPath": "repo-bytes.db",
"repoBytesDbExists": (out / "repo-bytes.db").exists(),
},
indent=2,
),
encoding="utf-8",
)
PY
echo "done: $OUT_DIR"

246
scripts/cir/run_cir_record_sequence_remote.sh
View File

@ -1,246 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/run_cir_record_sequence_remote.sh \
--rir <name> \
--remote-root <path> \
[--ssh-target <user@host>] \
[--out-subdir <path>] \
[--delta-count <n>] \
[--sleep-secs <n>] \
[--full-repo] \
[--max-depth <n>] \
[--max-instances <n>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
RIR=""
REMOTE_ROOT=""
OUT_SUBDIR=""
DELTA_COUNT=2
SLEEP_SECS=30
FULL_REPO=0
MAX_DEPTH=0
MAX_INSTANCES=1
while [[ $# -gt 0 ]]; do
case "$1" in
--rir) RIR="$2"; shift 2 ;;
--remote-root) REMOTE_ROOT="$2"; shift 2 ;;
--ssh-target) SSH_TARGET="$2"; shift 2 ;;
--out-subdir) OUT_SUBDIR="$2"; shift 2 ;;
--delta-count) DELTA_COUNT="$2"; shift 2 ;;
--sleep-secs) SLEEP_SECS="$2"; shift 2 ;;
--full-repo) FULL_REPO=1; shift 1 ;;
--max-depth) MAX_DEPTH="$2"; shift 2 ;;
--max-instances) MAX_INSTANCES="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$RIR" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
case "$RIR" in
afrinic) TAL_REL="tests/fixtures/tal/afrinic.tal"; TA_REL="tests/fixtures/ta/afrinic-ta.cer" ;;
apnic) TAL_REL="tests/fixtures/tal/apnic-rfc7730-https.tal"; TA_REL="tests/fixtures/ta/apnic-ta.cer" ;;
arin) TAL_REL="tests/fixtures/tal/arin.tal"; TA_REL="tests/fixtures/ta/arin-ta.cer" ;;
lacnic) TAL_REL="tests/fixtures/tal/lacnic.tal"; TA_REL="tests/fixtures/ta/lacnic-ta.cer" ;;
ripe) TAL_REL="tests/fixtures/tal/ripe-ncc.tal"; TA_REL="tests/fixtures/ta/ripe-ncc-ta.cer" ;;
*) echo "unsupported rir: $RIR" >&2; exit 2 ;;
esac
rsync -a --delete \
--exclude target \
--exclude .git \
"$ROOT_DIR/" "$SSH_TARGET:$REMOTE_ROOT/"
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/target/release'"
rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_ROOT/target/release/"
ssh "$SSH_TARGET" \
RIR="$RIR" \
REMOTE_ROOT="$REMOTE_ROOT" \
OUT_SUBDIR="$OUT_SUBDIR" \
DELTA_COUNT="$DELTA_COUNT" \
SLEEP_SECS="$SLEEP_SECS" \
FULL_REPO="$FULL_REPO" \
MAX_DEPTH="$MAX_DEPTH" \
MAX_INSTANCES="$MAX_INSTANCES" \
TAL_REL="$TAL_REL" \
TA_REL="$TA_REL" \
'bash -s' <<'EOS'
set -euo pipefail
cd "$REMOTE_ROOT"
if [[ -n "${OUT_SUBDIR}" ]]; then
OUT="${OUT_SUBDIR}"
else
OUT="target/replay/cir_sequence_remote_${RIR}_$(date -u +%Y%m%dT%H%M%SZ)"
fi
mkdir -p "$OUT"
DB="$OUT/work-db"
RAW_STORE_DB="$OUT/raw-store.db"
REPO_BYTES_DB="$OUT/repo-bytes.db"
ROWS="$OUT/.sequence_rows.tsv"
: > "$ROWS"
write_step_timing() {
local path="$1"
local start_ms="$2"
local end_ms="$3"
local started_at="$4"
local finished_at="$5"
python3 - <<'PY' "$path" "$start_ms" "$end_ms" "$started_at" "$finished_at"
import json, sys
path, start_ms, end_ms, started_at, finished_at = sys.argv[1:]
start_ms = int(start_ms)
end_ms = int(end_ms)
with open(path, "w", encoding="utf-8") as fh:
json.dump(
{
"durationMs": end_ms - start_ms,
"startedAt": started_at,
"finishedAt": finished_at,
},
fh,
indent=2,
)
PY
}
run_step() {
local step_id="$1"
local kind="$2"
local previous_step_id="$3"
shift 3
local started_at_iso started_at_ms finished_at_iso finished_at_ms prefix
started_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
started_at_ms="$(python3 - <<'PY'
import time
print(int(time.time() * 1000))
PY
)"
prefix="${started_at_iso}-test"
local cir_out="$OUT/${prefix}.cir"
local ccr_out="$OUT/${prefix}.ccr"
local report_out="$OUT/${prefix}.report.json"
local timing_out="$OUT/${prefix}.timing.json"
local stdout_out="$OUT/${prefix}.stdout.log"
local stderr_out="$OUT/${prefix}.stderr.log"
local -a cmd=(
target/release/rpki
--db "$DB"
--raw-store-db "$RAW_STORE_DB"
--repo-bytes-db "$REPO_BYTES_DB"
--tal-path "$TAL_REL"
--ta-path "$TA_REL"
--ccr-out "$ccr_out"
--report-json "$report_out"
--cir-enable
--cir-out "$cir_out"
--cir-tal-uri "https://example.test/${RIR}.tal"
)
if [[ "$FULL_REPO" -ne 1 ]]; then
cmd+=(--max-depth "$MAX_DEPTH" --max-instances "$MAX_INSTANCES")
fi
cmd+=("$@")
env RPKI_PROGRESS_LOG=1 "${cmd[@]}" >"$stdout_out" 2>"$stderr_out"
finished_at_ms="$(python3 - <<'PY'
import time
print(int(time.time() * 1000))
PY
)"
finished_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
write_step_timing "$timing_out" "$started_at_ms" "$finished_at_ms" "$started_at_iso" "$finished_at_iso"
local validation_time
validation_time="$(python3 - <<'PY' "$report_out"
import json, sys
print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['meta']['validation_time_rfc3339_utc'])
PY
)"
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
"$step_id" \
"$kind" \
"$validation_time" \
"$(basename "$cir_out")" \
"$(basename "$ccr_out")" \
"$(basename "$report_out")" \
"$(basename "$timing_out")" \
"$(basename "$stdout_out")" \
"$(basename "$stderr_out")" >> "$ROWS"
}
run_step "full" "full" ""
prev="full"
for idx in $(seq 1 "$DELTA_COUNT"); do
sleep "$SLEEP_SECS"
step="$(printf 'delta-%03d' "$idx")"
run_step "$step" "delta" "$prev"
prev="$step"
done
python3 - <<'PY' "$OUT" "$ROWS" "$RIR"
import json, sys
from pathlib import Path
out = Path(sys.argv[1])
rows = Path(sys.argv[2]).read_text(encoding='utf-8').splitlines()
rir = sys.argv[3]
steps = []
for idx, row in enumerate(rows):
step_id, kind, validation_time, cir_name, ccr_name, report_name, timing_name, stdout_name, stderr_name = row.split('\t')
steps.append({
"stepId": step_id,
"kind": kind,
"validationTime": validation_time,
"cirPath": cir_name,
"ccrPath": ccr_name,
"reportPath": report_name,
"timingPath": timing_name,
"stdoutLogPath": stdout_name,
"stderrLogPath": stderr_name,
"artifactPrefix": cir_name[:-4], # strip .cir
"previousStepId": None if idx == 0 else steps[idx - 1]["stepId"],
})
(out / "sequence.json").write_text(
json.dumps({"version": 1, "repoBytesDbPath": "repo-bytes.db", "steps": steps}, indent=2),
encoding="utf-8",
)
summary = {
"version": 1,
"rir": rir,
"stepCount": len(steps),
"steps": [],
}
for step in steps:
timing = json.loads((out / step["timingPath"]).read_text(encoding="utf-8"))
summary["steps"].append({
"stepId": step["stepId"],
"kind": step["kind"],
"validationTime": step["validationTime"],
"artifactPrefix": step["artifactPrefix"],
**timing,
})
(out / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
PY
rm -f "$ROWS"
echo "$OUT"
EOS

72
scripts/cir/run_cir_record_sequence_remote_multi_rir.sh
View File

@ -1,72 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/run_cir_record_sequence_remote_multi_rir.sh \
--remote-root <path> \
[--rir <afrinic,apnic,arin,lacnic,ripe>] \
[--ssh-target <user@host>] \
[--out-subdir-root <path>] \
[--delta-count <n>] \
[--sleep-secs <n>] \
[--full-repo] \
[--max-depth <n>] \
[--max-instances <n>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
REMOTE_ROOT=""
RIRS="afrinic,apnic,arin,lacnic,ripe"
OUT_SUBDIR_ROOT=""
DELTA_COUNT=2
SLEEP_SECS=30
FULL_REPO=0
MAX_DEPTH=0
MAX_INSTANCES=1
SINGLE="$ROOT_DIR/scripts/cir/run_cir_record_sequence_remote.sh"
while [[ $# -gt 0 ]]; do
case "$1" in
--remote-root) REMOTE_ROOT="$2"; shift 2 ;;
--rir) RIRS="$2"; shift 2 ;;
--ssh-target) SSH_TARGET="$2"; shift 2 ;;
--out-subdir-root) OUT_SUBDIR_ROOT="$2"; shift 2 ;;
--delta-count) DELTA_COUNT="$2"; shift 2 ;;
--sleep-secs) SLEEP_SECS="$2"; shift 2 ;;
--full-repo) FULL_REPO=1; shift 1 ;;
--max-depth) MAX_DEPTH="$2"; shift 2 ;;
--max-instances) MAX_INSTANCES="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
if [[ -z "$OUT_SUBDIR_ROOT" ]]; then
OUT_SUBDIR_ROOT="target/replay/cir_sequence_remote_multi_rir_$(date -u +%Y%m%dT%H%M%SZ)"
fi
IFS=',' read -r -a ITEMS <<< "$RIRS"
for rir in "${ITEMS[@]}"; do
args=(
"$SINGLE"
--rir "$rir" \
--remote-root "$REMOTE_ROOT" \
--ssh-target "$SSH_TARGET" \
--out-subdir "$OUT_SUBDIR_ROOT/$rir" \
--delta-count "$DELTA_COUNT" \
--sleep-secs "$SLEEP_SECS" \
)
if [[ "$FULL_REPO" -eq 1 ]]; then
args+=(--full-repo)
else
args+=(--max-depth "$MAX_DEPTH" --max-instances "$MAX_INSTANCES")
fi
"${args[@]}"
done
echo "$OUT_SUBDIR_ROOT"

119
scripts/cir/run_cir_record_sequence_ta_only_multi_rir.sh
View File

@ -1,119 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/run_cir_record_sequence_ta_only_multi_rir.sh \
[--rir <afrinic,apnic,arin,lacnic,ripe>] \
[--delta-count <n>] \
[--out-root <path>] \
[--rpki-bin <path>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
HELPER_BIN="${HELPER_BIN:-$ROOT_DIR/target/release/cir_ta_only_fixture}"
MATERIALIZE_BIN="${MATERIALIZE_BIN:-$ROOT_DIR/target/release/cir_materialize}"
EXTRACT_BIN="${EXTRACT_BIN:-$ROOT_DIR/target/release/cir_extract_inputs}"
WRAPPER="$ROOT_DIR/scripts/cir/cir-rsync-wrapper"
RIRS="afrinic,apnic,arin,lacnic,ripe"
DELTA_COUNT=2
OUT_ROOT="$ROOT_DIR/target/replay/cir_sequence_multi_rir_ta_only_$(date -u +%Y%m%dT%H%M%SZ)"
RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
while [[ $# -gt 0 ]]; do
case "$1" in
--rir) RIRS="$2"; shift 2 ;;
--delta-count) DELTA_COUNT="$2"; shift 2 ;;
--out-root) OUT_ROOT="$2"; shift 2 ;;
--rpki-bin) RPKI_BIN="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
if [[ ! -x "$HELPER_BIN" ]]; then
(
cd "$ROOT_DIR"
cargo build --release --bin cir_ta_only_fixture --bin rpki --bin cir_materialize --bin cir_extract_inputs
)
fi
case_paths() {
case "$1" in
afrinic) echo "tests/fixtures/tal/afrinic.tal tests/fixtures/ta/afrinic-ta.cer" ;;
apnic) echo "tests/fixtures/tal/apnic-rfc7730-https.tal tests/fixtures/ta/apnic-ta.cer" ;;
arin) echo "tests/fixtures/tal/arin.tal tests/fixtures/ta/arin-ta.cer" ;;
lacnic) echo "tests/fixtures/tal/lacnic.tal tests/fixtures/ta/lacnic-ta.cer" ;;
ripe) echo "tests/fixtures/tal/ripe-ncc.tal tests/fixtures/ta/ripe-ncc-ta.cer" ;;
*) return 1 ;;
esac
}
mkdir -p "$OUT_ROOT"
IFS=',' read -r -a ITEMS <<< "$RIRS"
for rir in "${ITEMS[@]}"; do
read -r tal_rel ta_rel < <(case_paths "$rir")
rir_root="$OUT_ROOT/$rir"
mkdir -p "$rir_root/full"
repo_bytes_db="$rir_root/repo-bytes.db"
"$HELPER_BIN" \
--tal-path "$ROOT_DIR/$tal_rel" \
--ta-path "$ROOT_DIR/$ta_rel" \
--tal-uri "https://example.test/$rir.tal" \
--validation-time "2026-04-09T00:00:00Z" \
--cir-out "$rir_root/full/input.cir" \
--repo-bytes-db "$repo_bytes_db"
"$EXTRACT_BIN" --cir "$rir_root/full/input.cir" --tals-dir "$rir_root/.tmp/tals" --meta-json "$rir_root/.tmp/meta.json"
"$MATERIALIZE_BIN" --cir "$rir_root/full/input.cir" --repo-bytes-db "$repo_bytes_db" --mirror-root "$rir_root/.tmp/mirror"
FIRST_TAL="$(python3 - <<'PY' "$rir_root/.tmp/meta.json"
import json,sys
print(json.load(open(sys.argv[1]))["talFiles"][0]["path"])
PY
)"
export CIR_MIRROR_ROOT="$rir_root/.tmp/mirror"
export REAL_RSYNC_BIN=/usr/bin/rsync
export CIR_LOCAL_LINK_MODE=1
"$RPKI_BIN" \
--db "$rir_root/full/db" \
--tal-path "$FIRST_TAL" \
--disable-rrdp \
--rsync-command "$WRAPPER" \
--validation-time "2026-04-09T00:00:00Z" \
--ccr-out "$rir_root/full/result.ccr" \
--report-json "$rir_root/full/report.json" >/dev/null 2>&1
for idx in $(seq 1 "$DELTA_COUNT"); do
step="$(printf 'delta-%03d' "$idx")"
mkdir -p "$rir_root/$step"
cp "$rir_root/full/input.cir" "$rir_root/$step/input.cir"
cp "$rir_root/full/result.ccr" "$rir_root/$step/result.ccr"
cp "$rir_root/full/report.json" "$rir_root/$step/report.json"
done
python3 - <<'PY' "$rir_root" "$DELTA_COUNT"
import json, sys
from pathlib import Path
root = Path(sys.argv[1]); delta_count = int(sys.argv[2])
steps = [{"stepId":"full","kind":"full","validationTime":"2026-04-09T00:00:00Z","cirPath":"full/input.cir","ccrPath":"full/result.ccr","reportPath":"full/report.json","previousStepId":None}]
prev = "full"
for i in range(1, delta_count + 1):
step = f"delta-{i:03d}"
steps.append({"stepId":step,"kind":"delta","validationTime":"2026-04-09T00:00:00Z","cirPath":f"{step}/input.cir","ccrPath":f"{step}/result.ccr","reportPath":f"{step}/report.json","previousStepId":prev})
prev = step
(root/"sequence.json").write_text(json.dumps({"version":1,"repoBytesDbPath":"repo-bytes.db","steps":steps}, indent=2), encoding="utf-8")
(root/"summary.json").write_text(json.dumps({"version":1,"stepCount":len(steps)}, indent=2), encoding="utf-8")
PY
done
python3 - <<'PY' "$OUT_ROOT" "$RIRS"
import json, sys
from pathlib import Path
root = Path(sys.argv[1]); rirs = [x for x in sys.argv[2].split(',') if x]
items=[]
for rir in rirs:
seq=json.loads((root/rir/'sequence.json').read_text())
items.append({"rir":rir,"stepCount":len(seq['steps'])})
(root/'summary.json').write_text(json.dumps({"version":1,"rirs":items}, indent=2), encoding='utf-8')
PY
echo "done: $OUT_ROOT"

49
scripts/cir/run_cir_record_sequence_ta_only_remote_multi_rir.sh
View File

@ -1,49 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/run_cir_record_sequence_ta_only_remote_multi_rir.sh \
--remote-root <path> \
[--ssh-target <user@host>] \
[--rir <afrinic,apnic,arin,lacnic,ripe>] \
[--delta-count <n>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
REMOTE_ROOT=""
RIRS="afrinic,apnic,arin,lacnic,ripe"
DELTA_COUNT=2
while [[ $# -gt 0 ]]; do
case "$1" in
--remote-root) REMOTE_ROOT="$2"; shift 2 ;;
--ssh-target) SSH_TARGET="$2"; shift 2 ;;
--rir) RIRS="$2"; shift 2 ;;
--delta-count) DELTA_COUNT="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
rsync -a --delete \
--exclude target \
--exclude .git \
"$ROOT_DIR/" "$SSH_TARGET:$REMOTE_ROOT/"
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/target/release'"
for bin in rpki cir_ta_only_fixture cir_materialize cir_extract_inputs; do
rsync -a "$ROOT_DIR/target/release/$bin" "$SSH_TARGET:$REMOTE_ROOT/target/release/"
done
ssh "$SSH_TARGET" "bash -lc '
set -euo pipefail
cd $REMOTE_ROOT
OUT=target/replay/cir_sequence_remote_ta_only_\$(date -u +%Y%m%dT%H%M%SZ)
./scripts/cir/run_cir_record_sequence_ta_only_multi_rir.sh --rir $RIRS --delta-count $DELTA_COUNT --out-root \"\$OUT\"
echo \"\$OUT\"
'"

293
scripts/cir/run_cir_replay_matrix.sh
View File

@ -1,293 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/run_cir_replay_matrix.sh \
--cir <path> \
--repo-bytes-db <path> \
--out-dir <path> \
--reference-ccr <path> \
--rpki-client-build-dir <path> \
[--keep-db] \
[--rpki-bin <path>] \
[--routinator-root <path>] \
[--routinator-bin <path>] \
[--real-rsync-bin <path>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
CIR=""
REPO_BYTES_DB=""
OUT_DIR=""
REFERENCE_CCR=""
RPKI_CLIENT_BUILD_DIR=""
KEEP_DB=0
RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
ROUTINATOR_ROOT="${ROUTINATOR_ROOT:-/home/yuyr/dev/rust_playground/routinator}"
ROUTINATOR_BIN="${ROUTINATOR_BIN:-$ROUTINATOR_ROOT/target/debug/routinator}"
REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
OURS_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_ours.sh"
ROUTINATOR_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_routinator.sh"
RPKI_CLIENT_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_rpki_client.sh"
while [[ $# -gt 0 ]]; do
case "$1" in
--cir) CIR="$2"; shift 2 ;;
--repo-bytes-db) REPO_BYTES_DB="$2"; shift 2 ;;
--out-dir) OUT_DIR="$2"; shift 2 ;;
--reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
--rpki-client-build-dir) RPKI_CLIENT_BUILD_DIR="$2"; shift 2 ;;
--keep-db) KEEP_DB=1; shift ;;
--rpki-bin) RPKI_BIN="$2"; shift 2 ;;
--routinator-root) ROUTINATOR_ROOT="$2"; shift 2 ;;
--routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
--real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$CIR" && -n "$REPO_BYTES_DB" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" && -n "$RPKI_CLIENT_BUILD_DIR" ]] || {
usage >&2
exit 2
}
mkdir -p "$OUT_DIR"
run_with_timing() {
local summary_path="$1"
local timing_path="$2"
shift 2
local start end status
start="$(python3 - <<'PY'
import time
print(time.perf_counter_ns())
PY
)"
if "$@"; then
status=0
else
status=$?
fi
end="$(python3 - <<'PY'
import time
print(time.perf_counter_ns())
PY
)"
python3 - <<'PY' "$summary_path" "$timing_path" "$status" "$start" "$end"
import json, sys
summary_path, timing_path, status, start, end = sys.argv[1:]
duration_ms = max(0, (int(end) - int(start)) // 1_000_000)
data = {"exitCode": int(status), "durationMs": duration_ms}
try:
with open(summary_path, "r", encoding="utf-8") as f:
data["compare"] = json.load(f)
except FileNotFoundError:
data["compare"] = None
with open(timing_path, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2)
PY
return "$status"
}
OURS_OUT="$OUT_DIR/ours"
ROUTINATOR_OUT="$OUT_DIR/routinator"
RPKI_CLIENT_OUT="$OUT_DIR/rpki-client"
mkdir -p "$OURS_OUT" "$ROUTINATOR_OUT" "$RPKI_CLIENT_OUT"
ours_cmd=(
"$OURS_SCRIPT"
--cir "$CIR"
--repo-bytes-db "$REPO_BYTES_DB"
--out-dir "$OURS_OUT"
--reference-ccr "$REFERENCE_CCR"
--rpki-bin "$RPKI_BIN"
--real-rsync-bin "$REAL_RSYNC_BIN"
)
routinator_cmd=(
"$ROUTINATOR_SCRIPT"
--cir "$CIR"
--repo-bytes-db "$REPO_BYTES_DB"
--out-dir "$ROUTINATOR_OUT"
--reference-ccr "$REFERENCE_CCR"
--routinator-root "$ROUTINATOR_ROOT"
--routinator-bin "$ROUTINATOR_BIN"
--real-rsync-bin "$REAL_RSYNC_BIN"
)
rpki_client_cmd=(
"$RPKI_CLIENT_SCRIPT"
--cir "$CIR"
--repo-bytes-db "$REPO_BYTES_DB"
--out-dir "$RPKI_CLIENT_OUT"
--reference-ccr "$REFERENCE_CCR"
--build-dir "$RPKI_CLIENT_BUILD_DIR"
--real-rsync-bin "$REAL_RSYNC_BIN"
)
if [[ "$KEEP_DB" -eq 1 ]]; then
ours_cmd+=(--keep-db)
routinator_cmd+=(--keep-db)
rpki_client_cmd+=(--keep-db)
fi
ours_status=0
routinator_status=0
rpki_client_status=0
if run_with_timing "$OURS_OUT/compare-summary.json" "$OURS_OUT/timing.json" "${ours_cmd[@]}"; then
:
else
ours_status=$?
fi
if run_with_timing "$ROUTINATOR_OUT/compare-summary.json" "$ROUTINATOR_OUT/timing.json" "${routinator_cmd[@]}"; then
:
else
routinator_status=$?
fi
if run_with_timing "$RPKI_CLIENT_OUT/compare-summary.json" "$RPKI_CLIENT_OUT/timing.json" "${rpki_client_cmd[@]}"; then
:
else
rpki_client_status=$?
fi
SUMMARY_JSON="$OUT_DIR/summary.json"
SUMMARY_MD="$OUT_DIR/summary.md"
DETAIL_MD="$OUT_DIR/detail.md"
python3 - <<'PY' \
"$CIR" \
"$REPO_BYTES_DB" \
"$REFERENCE_CCR" \
"$OURS_OUT" \
"$ROUTINATOR_OUT" \
"$RPKI_CLIENT_OUT" \
"$SUMMARY_JSON" \
"$SUMMARY_MD" \
"$DETAIL_MD"
import json
import sys
from pathlib import Path
cir_path, repo_bytes_db, reference_ccr, ours_out, routinator_out, rpki_client_out, summary_json, summary_md, detail_md = sys.argv[1:]
participants = []
all_match = True
for name, out_dir in [
("ours", ours_out),
("routinator", routinator_out),
("rpki-client", rpki_client_out),
]:
out = Path(out_dir)
timing = json.loads((out / "timing.json").read_text(encoding="utf-8"))
compare = timing.get("compare") or {}
vrps = compare.get("vrps") or {}
vaps = compare.get("vaps") or {}
participant = {
"name": name,
"outDir": str(out),
"tmpRoot": str(out / ".tmp"),
"mirrorPath": str(out / ".tmp" / "mirror"),
"timingPath": str(out / "timing.json"),
"summaryPath": str(out / "compare-summary.json"),
"exitCode": timing["exitCode"],
"durationMs": timing["durationMs"],
"compareMode": compare.get("compareMode"),
"talCount": compare.get("talCount"),
"talPaths": compare.get("talPaths", []),
"vrps": vrps,
"vaps": vaps,
"match": bool(vrps.get("match")) and bool(vaps.get("match")) and timing["exitCode"] == 0,
"logPaths": [str(path) for path in sorted(out.glob("*.log"))],
}
participants.append(participant)
all_match = all_match and participant["match"]
summary = {
"cirPath": cir_path,
"repoBytesDb": repo_bytes_db,
"referenceCcr": reference_ccr,
"participants": participants,
"allMatch": all_match,
}
Path(summary_json).write_text(json.dumps(summary, indent=2), encoding="utf-8")
lines = [
"# CIR Replay Matrix Summary",
"",
f"- `cir`: `{cir_path}`",
f"- `repo_bytes_db`: `{repo_bytes_db}`",
f"- `reference_ccr`: `{reference_ccr}`",
f"- `all_match`: `{all_match}`",
"",
"| Participant | Exit | Duration (ms) | TALs | Compare mode | VRP actual/ref | VRP match | VAP actual/ref | VAP match | Log |",
"| --- | ---: | ---: | ---: | --- | --- | --- | --- | --- | --- |",
]
for participant in participants:
vrps = participant["vrps"] or {}
vaps = participant["vaps"] or {}
log_path = participant["logPaths"][0] if participant["logPaths"] else ""
lines.append(
"| {name} | {exit_code} | {duration_ms} | {tal_count} | {compare_mode} | {vrp_actual}/{vrp_ref} | {vrp_match} | {vap_actual}/{vap_ref} | {vap_match} | `{log_path}` |".format(
name=participant["name"],
exit_code=participant["exitCode"],
duration_ms=participant["durationMs"],
tal_count=participant.get("talCount") if participant.get("talCount") is not None else "-",
compare_mode=participant.get("compareMode") or "-",
vrp_actual=vrps.get("actual", "-"),
vrp_ref=vrps.get("reference", "-"),
vrp_match=vrps.get("match", False),
vap_actual=vaps.get("actual", "-"),
vap_ref=vaps.get("reference", "-"),
vap_match=vaps.get("match", False),
log_path=log_path,
)
)
Path(summary_md).write_text("\n".join(lines) + "\n", encoding="utf-8")
detail_lines = [
"# CIR Replay Matrix Detail",
"",
]
for participant in participants:
vrps = participant["vrps"] or {}
vaps = participant["vaps"] or {}
detail_lines.extend([
f"## {participant['name']}",
f"- `exit_code`: `{participant['exitCode']}`",
f"- `duration_ms`: `{participant['durationMs']}`",
f"- `out_dir`: `{participant['outDir']}`",
f"- `tmp_root`: `{participant['tmpRoot']}`",
f"- `mirror_path`: `{participant['mirrorPath']}`",
f"- `summary_path`: `{participant['summaryPath']}`",
f"- `timing_path`: `{participant['timingPath']}`",
f"- `compare_mode`: `{participant.get('compareMode')}`",
f"- `tal_count`: `{participant.get('talCount')}`",
f"- `log_paths`: `{', '.join(participant['logPaths'])}`",
f"- `vrps`: `actual={vrps.get('actual', '-')}` `reference={vrps.get('reference', '-')}` `match={vrps.get('match', False)}`",
f"- `vaps`: `actual={vaps.get('actual', '-')}` `reference={vaps.get('reference', '-')}` `match={vaps.get('match', False)}`",
f"- `vrps.only_in_actual`: `{vrps.get('only_in_actual', [])}`",
f"- `vrps.only_in_reference`: `{vrps.get('only_in_reference', [])}`",
f"- `vaps.only_in_actual`: `{vaps.get('only_in_actual', [])}`",
f"- `vaps.only_in_reference`: `{vaps.get('only_in_reference', [])}`",
"",
])
Path(detail_md).write_text("\n".join(detail_lines), encoding="utf-8")
PY
if [[ "$ours_status" -ne 0 || "$routinator_status" -ne 0 || "$rpki_client_status" -ne 0 ]]; then
exit 1
fi
all_match="$(python3 - <<'PY' "$SUMMARY_JSON"
import json,sys
print("true" if json.load(open(sys.argv[1]))["allMatch"] else "false")
PY
)"
if [[ "$all_match" != "true" ]]; then
exit 1
fi
echo "done: $OUT_DIR"

252
scripts/cir/run_cir_replay_ours.sh
View File

@ -1,252 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/run_cir_replay_ours.sh \
--cir <path> \
--repo-bytes-db <path> \
--out-dir <path> \
--reference-ccr <path> \
[--keep-db] \
[--write-actual-ccr] \
[--write-report-json] \
[--report-json-compact] \
[--phase2-object-workers <n>] \
[--phase2-worker-queue-capacity <n>] \
[--rpki-bin <path>] \
[--real-rsync-bin <path>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
CIR=""
REPO_BYTES_DB=""
OUT_DIR=""
REFERENCE_CCR=""
KEEP_DB=0
WRITE_ACTUAL_CCR=0
WRITE_REPORT_JSON=0
REPORT_JSON_COMPACT=0
PHASE2_OBJECT_WORKERS="${CIR_REPLAY_PHASE2_OBJECT_WORKERS:-4}"
PHASE2_WORKER_QUEUE_CAPACITY="${CIR_REPLAY_PHASE2_WORKER_QUEUE_CAPACITY:-64}"
RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
CIR_MATERIALIZE_BIN="${CIR_MATERIALIZE_BIN:-$ROOT_DIR/target/release/cir_materialize}"
CIR_EXTRACT_INPUTS_BIN="${CIR_EXTRACT_INPUTS_BIN:-$ROOT_DIR/target/release/cir_extract_inputs}"
CCR_TO_COMPARE_VIEWS_BIN="${CCR_TO_COMPARE_VIEWS_BIN:-$ROOT_DIR/target/release/ccr_to_compare_views}"
REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
WRAPPER="$ROOT_DIR/scripts/cir/cir-rsync-wrapper"
while [[ $# -gt 0 ]]; do
case "$1" in
--cir) CIR="$2"; shift 2 ;;
--repo-bytes-db) REPO_BYTES_DB="$2"; shift 2 ;;
--out-dir) OUT_DIR="$2"; shift 2 ;;
--reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
--keep-db) KEEP_DB=1; shift ;;
--write-actual-ccr) WRITE_ACTUAL_CCR=1; shift ;;
--write-report-json) WRITE_REPORT_JSON=1; shift ;;
--report-json-compact) WRITE_REPORT_JSON=1; REPORT_JSON_COMPACT=1; shift ;;
--phase2-object-workers) PHASE2_OBJECT_WORKERS="$2"; shift 2 ;;
--phase2-worker-queue-capacity) PHASE2_WORKER_QUEUE_CAPACITY="$2"; shift 2 ;;
--rpki-bin) RPKI_BIN="$2"; shift 2 ;;
--real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$CIR" && -n "$REPO_BYTES_DB" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
usage >&2
exit 2
}
mkdir -p "$OUT_DIR"
needs_build=0
if [[ ! -x "$RPKI_BIN" || ! -x "$CIR_MATERIALIZE_BIN" || ! -x "$CIR_EXTRACT_INPUTS_BIN" || ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
needs_build=1
elif [[ "$RPKI_BIN" == "$ROOT_DIR/target/release/rpki" ]] && find "$ROOT_DIR/src" "$ROOT_DIR/Cargo.toml" -newer "$RPKI_BIN" -print -quit | grep -q .; then
needs_build=1
fi
if [[ "$needs_build" -eq 1 ]]; then
(
cd "$ROOT_DIR"
cargo build --release --bin rpki --bin cir_materialize --bin cir_extract_inputs --bin ccr_to_compare_views
)
fi
TMP_ROOT="$OUT_DIR/.tmp"
TALS_DIR="$TMP_ROOT/tals"
META_JSON="$TMP_ROOT/meta.json"
MIRROR_ROOT="$TMP_ROOT/mirror"
DB_DIR="$TMP_ROOT/work-db"
REPLAY_RAW_STORE_DB="$TMP_ROOT/replay-raw-store.db"
REPLAY_REPO_BYTES_DB="$TMP_ROOT/replay-repo-bytes.db"
ACTUAL_CCR="$OUT_DIR/actual.ccr"
ACTUAL_REPORT="$OUT_DIR/report.json"
ACTUAL_VRPS="$OUT_DIR/actual-vrps.csv"
ACTUAL_VAPS="$OUT_DIR/actual-vaps.csv"
REF_VRPS="$OUT_DIR/reference-vrps.csv"
REF_VAPS="$OUT_DIR/reference-vaps.csv"
COMPARE_JSON="$OUT_DIR/compare-summary.json"
RUN_LOG="$OUT_DIR/run.log"
rm -rf "$TMP_ROOT"
mkdir -p "$TMP_ROOT"
"$CIR_EXTRACT_INPUTS_BIN" --cir "$CIR" --tals-dir "$TALS_DIR" --meta-json "$META_JSON"
materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --repo-bytes-db "$REPO_BYTES_DB" --mirror-root "$MIRROR_ROOT")
if [[ "$KEEP_DB" -eq 1 ]]; then
materialize_cmd+=(--keep-db)
fi
"${materialize_cmd[@]}"
VALIDATION_TIME="$(python3 - <<'PY' "$META_JSON"
import json,sys
print(json.load(open(sys.argv[1]))["validationTime"])
PY
)"
mapfile -t TAL_PATHS < <(python3 - <<'PY' "$META_JSON"
import json, sys
for item in json.load(open(sys.argv[1], encoding="utf-8"))["talFiles"]:
print(item["path"])
PY
)
TAL_ARGS=()
for tal_path in "${TAL_PATHS[@]}"; do
TAL_ARGS+=(--tal-path "$tal_path")
done
export CIR_MIRROR_ROOT="$(python3 - <<'PY' "$MIRROR_ROOT"
from pathlib import Path
import sys
print(Path(sys.argv[1]).resolve())
PY
)"
export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
export CIR_LOCAL_LINK_MODE=1
REPORT_JSON_ARGS=(--skip-report-build)
VCIR_ARGS=(--skip-vcir-persist)
if [[ "$WRITE_REPORT_JSON" -eq 1 ]]; then
REPORT_JSON_ARGS=(--report-json "$ACTUAL_REPORT")
if [[ "$REPORT_JSON_COMPACT" -eq 1 ]]; then
REPORT_JSON_ARGS+=(--report-json-compact)
fi
fi
CCR_ARGS=()
if [[ "$WRITE_ACTUAL_CCR" -eq 1 ]]; then
CCR_ARGS=(--ccr-out "$ACTUAL_CCR")
fi
"$RPKI_BIN" \
--db "$DB_DIR" \
--raw-store-db "$REPLAY_RAW_STORE_DB" \
--repo-bytes-db "$REPLAY_REPO_BYTES_DB" \
"${TAL_ARGS[@]}" \
--parallel-phase2-object-workers "$PHASE2_OBJECT_WORKERS" \
--parallel-phase2-worker-queue-capacity "$PHASE2_WORKER_QUEUE_CAPACITY" \
--disable-rrdp \
--rsync-command "$WRAPPER" \
--validation-time "$VALIDATION_TIME" \
"${CCR_ARGS[@]}" \
--vrps-csv-out "$ACTUAL_VRPS" \
--vaps-csv-out "$ACTUAL_VAPS" \
--compare-view-trust-anchor unknown \
"${VCIR_ARGS[@]}" \
"${REPORT_JSON_ARGS[@]}" \
>"$RUN_LOG" 2>&1
sort_compare_csv() {
local path="$1"
local tmp="${path}.sorted.tmp"
{
head -n 1 "$path"
tail -n +2 "$path" | LC_ALL=C sort -u
} >"$tmp"
mv "$tmp" "$path"
}
sort_compare_csv "$ACTUAL_VRPS"
sort_compare_csv "$ACTUAL_VAPS"
"$CCR_TO_COMPARE_VIEWS_BIN" --ccr "$REFERENCE_CCR" --vrps-out "$REF_VRPS" --vaps-out "$REF_VAPS" --trust-anchor unknown
python3 - <<'PY' "$ACTUAL_VRPS" "$REF_VRPS" "$ACTUAL_VAPS" "$REF_VAPS" "$COMPARE_JSON" "$META_JSON" "$WRITE_REPORT_JSON" "$WRITE_ACTUAL_CCR" "$PHASE2_OBJECT_WORKERS" "$PHASE2_WORKER_QUEUE_CAPACITY"
import csv, json, sys
def next_row(reader):
try:
return tuple(next(reader))
except StopIteration:
return None
def compare_sorted_csv(actual_path, ref_path):
actual_count = 0
ref_count = 0
only_actual_count = 0
only_ref_count = 0
only_actual_sample = []
only_ref_sample = []
with open(actual_path, newline="") as actual_file, open(ref_path, newline="") as ref_file:
actual_reader = csv.reader(actual_file)
ref_reader = csv.reader(ref_file)
next(actual_reader, None)
next(ref_reader, None)
actual = next_row(actual_reader)
ref = next_row(ref_reader)
while actual is not None or ref is not None:
if ref is None or (actual is not None and actual < ref):
actual_count += 1
only_actual_count += 1
if len(only_actual_sample) < 20:
only_actual_sample.append(list(actual))
actual = next_row(actual_reader)
elif actual is None or ref < actual:
ref_count += 1
only_ref_count += 1
if len(only_ref_sample) < 20:
only_ref_sample.append(list(ref))
ref = next_row(ref_reader)
else:
actual_count += 1
ref_count += 1
actual = next_row(actual_reader)
ref = next_row(ref_reader)
return {
"actual": actual_count,
"reference": ref_count,
"only_in_actual": only_actual_sample,
"only_in_reference": only_ref_sample,
"only_in_actual_count": only_actual_count,
"only_in_reference_count": only_ref_count,
"match": only_actual_count == 0 and only_ref_count == 0,
}
vrps = compare_sorted_csv(sys.argv[1], sys.argv[2])
vaps = compare_sorted_csv(sys.argv[3], sys.argv[4])
meta = json.load(open(sys.argv[6], encoding="utf-8"))
summary = {
"compareMode": "trust-anchor-agnostic",
"talCount": len(meta["talFiles"]),
"talPaths": [item["path"] for item in meta["talFiles"]],
"actualCcrWritten": sys.argv[8] == "1",
"reportJsonWritten": sys.argv[7] == "1",
"replayParallelism": {
"phase2ObjectWorkers": int(sys.argv[9]),
"phase2WorkerQueueCapacity": int(sys.argv[10]),
},
"vrps": vrps,
"vaps": vaps,
}
with open(sys.argv[5], "w") as f:
json.dump(summary, f, indent=2)
PY
if [[ "$KEEP_DB" -ne 1 ]]; then
rm -rf "$TMP_ROOT"
fi
echo "done: $OUT_DIR"

239
scripts/cir/run_cir_replay_routinator.sh
View File

@ -1,239 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/run_cir_replay_routinator.sh \
--cir <path> \
--repo-bytes-db <path> \
--out-dir <path> \
--reference-ccr <path> \
[--keep-db] \
[--routinator-root <path>] \
[--routinator-bin <path>] \
[--real-rsync-bin <path>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
RPKI_DEV_ROOT="${RPKI_DEV_ROOT:-$ROOT_DIR}"
CIR=""
REPO_BYTES_DB=""
OUT_DIR=""
REFERENCE_CCR=""
KEEP_DB=0
ROUTINATOR_ROOT="${ROUTINATOR_ROOT:-/home/yuyr/dev/rust_playground/routinator}"
ROUTINATOR_BIN="${ROUTINATOR_BIN:-$ROUTINATOR_ROOT/target/debug/routinator}"
REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
CIR_MATERIALIZE_BIN="${CIR_MATERIALIZE_BIN:-$ROOT_DIR/target/release/cir_materialize}"
CIR_EXTRACT_INPUTS_BIN="${CIR_EXTRACT_INPUTS_BIN:-$ROOT_DIR/target/release/cir_extract_inputs}"
CCR_TO_COMPARE_VIEWS_BIN="${CCR_TO_COMPARE_VIEWS_BIN:-$ROOT_DIR/target/release/ccr_to_compare_views}"
WRAPPER="$ROOT_DIR/scripts/cir/cir-rsync-wrapper"
JSON_TO_VAPS="$ROOT_DIR/scripts/cir/json_to_vaps_csv.py"
FAKETIME_LIB="${FAKETIME_LIB:-$ROOT_DIR/target/tools/faketime_pkg/extracted/libfaketime/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1}"
while [[ $# -gt 0 ]]; do
case "$1" in
--cir) CIR="$2"; shift 2 ;;
--repo-bytes-db) REPO_BYTES_DB="$2"; shift 2 ;;
--out-dir) OUT_DIR="$2"; shift 2 ;;
--reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
--keep-db) KEEP_DB=1; shift ;;
--routinator-root) ROUTINATOR_ROOT="$2"; shift 2 ;;
--routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
--real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$CIR" && -n "$REPO_BYTES_DB" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
usage >&2
exit 2
}
if [[ ! -x "$ROUTINATOR_BIN" ]]; then
echo "routinator binary not executable: $ROUTINATOR_BIN" >&2
exit 2
fi
mkdir -p "$OUT_DIR"
if [[ ! -x "$CIR_MATERIALIZE_BIN" || ! -x "$CIR_EXTRACT_INPUTS_BIN" || ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
(
cd "$ROOT_DIR"
cargo build --release --bin cir_materialize --bin cir_extract_inputs --bin ccr_to_compare_views
)
fi
TMP_ROOT="$OUT_DIR/.tmp"
TALS_DIR="$TMP_ROOT/tals"
META_JSON="$TMP_ROOT/meta.json"
MIRROR_ROOT="$TMP_ROOT/mirror"
WORK_REPO="$TMP_ROOT/repository"
RUN_LOG="$OUT_DIR/routinator.log"
ACTUAL_VRPS="$OUT_DIR/actual-vrps.csv"
ACTUAL_VAPS_JSON="$OUT_DIR/actual-vaps.json"
ACTUAL_VAPS="$OUT_DIR/actual-vaps.csv"
REF_VRPS="$OUT_DIR/reference-vrps.csv"
REF_VAPS="$OUT_DIR/reference-vaps.csv"
SUMMARY_JSON="$OUT_DIR/compare-summary.json"
rm -rf "$TMP_ROOT"
mkdir -p "$TMP_ROOT"
"$CIR_EXTRACT_INPUTS_BIN" --cir "$CIR" --tals-dir "$TALS_DIR" --meta-json "$META_JSON"
python3 - <<'PY' "$TALS_DIR"
from pathlib import Path
import sys
for tal in Path(sys.argv[1]).glob("*.tal"):
lines = tal.read_text(encoding="utf-8").splitlines()
rsync_uris = [line for line in lines if line.startswith("rsync://")]
base64_lines = []
seen_sep = False
for line in lines:
if seen_sep:
if line.strip():
base64_lines.append(line)
elif line.strip() == "":
seen_sep = True
tal.write_text("\n".join(rsync_uris) + "\n\n" + "\n".join(base64_lines) + "\n", encoding="utf-8")
PY
materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --repo-bytes-db "$REPO_BYTES_DB" --mirror-root "$MIRROR_ROOT")
if [[ "$KEEP_DB" -eq 1 ]]; then
materialize_cmd+=(--keep-db)
fi
"${materialize_cmd[@]}"
VALIDATION_TIME="$(python3 - <<'PY' "$META_JSON"
import json,sys
print(json.load(open(sys.argv[1]))["validationTime"])
PY
)"
mapfile -t TAL_PATHS < <(python3 - <<'PY' "$META_JSON"
import json, sys
for item in json.load(open(sys.argv[1], encoding="utf-8"))["talFiles"]:
print(item["path"])
PY
)
COMPARE_TRUST_ANCHOR="unknown"
FAKE_EPOCH="$(python3 - <<'PY' "$VALIDATION_TIME"
from datetime import datetime, timezone
import sys
dt = datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).astimezone(timezone.utc)
print(int(dt.timestamp()))
PY
)"
export CIR_MIRROR_ROOT="$(python3 - <<'PY' "$MIRROR_ROOT"
from pathlib import Path
import sys
print(Path(sys.argv[1]).resolve())
PY
)"
export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
export CIR_LOCAL_LINK_MODE=1
env \
LD_PRELOAD="$FAKETIME_LIB" \
FAKETIME_FMT=%s \
FAKETIME="$FAKE_EPOCH" \
FAKETIME_DONT_FAKE_MONOTONIC=1 \
"$ROUTINATOR_BIN" \
--repository-dir "$WORK_REPO" \
--disable-rrdp \
--rsync-command "$WRAPPER" \
--no-rir-tals \
--extra-tals-dir "$TALS_DIR" \
--enable-aspa \
update --complete >"$RUN_LOG" 2>&1 || true
env \
LD_PRELOAD="$FAKETIME_LIB" \
FAKETIME_FMT=%s \
FAKETIME="$FAKE_EPOCH" \
FAKETIME_DONT_FAKE_MONOTONIC=1 \
"$ROUTINATOR_BIN" \
--repository-dir "$WORK_REPO" \
--disable-rrdp \
--rsync-command "$WRAPPER" \
--no-rir-tals \
--extra-tals-dir "$TALS_DIR" \
--enable-aspa \
vrps --noupdate -o "$ACTUAL_VRPS" >>"$RUN_LOG" 2>&1
env \
LD_PRELOAD="$FAKETIME_LIB" \
FAKETIME_FMT=%s \
FAKETIME="$FAKE_EPOCH" \
FAKETIME_DONT_FAKE_MONOTONIC=1 \
"$ROUTINATOR_BIN" \
--repository-dir "$WORK_REPO" \
--disable-rrdp \
--rsync-command "$WRAPPER" \
--no-rir-tals \
--extra-tals-dir "$TALS_DIR" \
--enable-aspa \
vrps --noupdate --format json -o "$ACTUAL_VAPS_JSON" >>"$RUN_LOG" 2>&1
python3 "$JSON_TO_VAPS" --input "$ACTUAL_VAPS_JSON" --csv-out "$ACTUAL_VAPS"
normalize_trust_anchor_csv() {
python3 - <<'PY' "$1" "$2"
import csv
import sys
from pathlib import Path
path = Path(sys.argv[1])
trust_anchor = sys.argv[2]
rows = list(csv.reader(path.open(newline="", encoding="utf-8")))
if rows:
for row in rows[1:]:
if row:
row[-1] = trust_anchor
with path.open("w", newline="", encoding="utf-8") as fh:
csv.writer(fh).writerows(rows)
PY
}
normalize_trust_anchor_csv "$ACTUAL_VRPS" "$COMPARE_TRUST_ANCHOR"
normalize_trust_anchor_csv "$ACTUAL_VAPS" "$COMPARE_TRUST_ANCHOR"
"$CCR_TO_COMPARE_VIEWS_BIN" --ccr "$REFERENCE_CCR" --vrps-out "$REF_VRPS" --vaps-out "$REF_VAPS" --trust-anchor "$COMPARE_TRUST_ANCHOR"
python3 - <<'PY' "$ACTUAL_VRPS" "$REF_VRPS" "$ACTUAL_VAPS" "$REF_VAPS" "$SUMMARY_JSON" "$META_JSON"
import csv, json, sys
def rows(path):
with open(path, newline="") as f:
return list(csv.reader(f))[1:]
actual_vrps = {tuple(r) for r in rows(sys.argv[1])}
ref_vrps = {tuple(r) for r in rows(sys.argv[2])}
actual_vaps = {tuple(r) for r in rows(sys.argv[3])}
ref_vaps = {tuple(r) for r in rows(sys.argv[4])}
meta = json.load(open(sys.argv[6], encoding="utf-8"))
summary = {
"compareMode": "trust-anchor-agnostic",
"talCount": len(meta["talFiles"]),
"talPaths": [item["path"] for item in meta["talFiles"]],
"vrps": {
"actual": len(actual_vrps),
"reference": len(ref_vrps),
"match": actual_vrps == ref_vrps,
"only_in_actual": sorted(actual_vrps - ref_vrps)[:20],
"only_in_reference": sorted(ref_vrps - actual_vrps)[:20],
},
"vaps": {
"actual": len(actual_vaps),
"reference": len(ref_vaps),
"match": actual_vaps == ref_vaps,
"only_in_actual": sorted(actual_vaps - ref_vaps)[:20],
"only_in_reference": sorted(ref_vaps - actual_vaps)[:20],
}
}
with open(sys.argv[5], "w") as f:
json.dump(summary, f, indent=2)
PY
if [[ "$KEEP_DB" -ne 1 ]]; then
rm -rf "$TMP_ROOT"
fi
echo "done: $OUT_DIR"

248
scripts/cir/run_cir_replay_rpki_client.sh
View File

@ -1,248 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/run_cir_replay_rpki_client.sh \
--cir <path> \
--repo-bytes-db <path> \
--out-dir <path> \
--reference-ccr <path> \
[--build-dir <path> | --rpki-client-bin <path>] \
[--keep-db] \
[--real-rsync-bin <path>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
CIR=""
REPO_BYTES_DB=""
OUT_DIR=""
REFERENCE_CCR=""
BUILD_DIR=""
RPKI_CLIENT_BIN=""
KEEP_DB=0
REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
CIR_MATERIALIZE_BIN="${CIR_MATERIALIZE_BIN:-$ROOT_DIR/target/release/cir_materialize}"
CIR_EXTRACT_INPUTS_BIN="${CIR_EXTRACT_INPUTS_BIN:-$ROOT_DIR/target/release/cir_extract_inputs}"
CCR_TO_COMPARE_VIEWS_BIN="${CCR_TO_COMPARE_VIEWS_BIN:-$ROOT_DIR/target/release/ccr_to_compare_views}"
WRAPPER="$ROOT_DIR/scripts/cir/cir-rsync-wrapper"
while [[ $# -gt 0 ]]; do
case "$1" in
--cir) CIR="$2"; shift 2 ;;
--repo-bytes-db) REPO_BYTES_DB="$2"; shift 2 ;;
--out-dir) OUT_DIR="$2"; shift 2 ;;
--reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
--build-dir) BUILD_DIR="$2"; shift 2 ;;
--rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
--keep-db) KEEP_DB=1; shift ;;
--real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$CIR" && -n "$REPO_BYTES_DB" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
usage >&2
exit 2
}
if [[ -z "$BUILD_DIR" && -z "$RPKI_CLIENT_BIN" ]]; then
usage >&2
exit 2
fi
if [[ -z "$RPKI_CLIENT_BIN" ]]; then
RPKI_CLIENT_BIN="$BUILD_DIR/src/rpki-client"
fi
if [[ ! -x "$RPKI_CLIENT_BIN" ]]; then
echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2
exit 2
fi
mkdir -p "$OUT_DIR"
if [[ ! -x "$CIR_MATERIALIZE_BIN" || ! -x "$CIR_EXTRACT_INPUTS_BIN" || ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
(
cd "$ROOT_DIR"
cargo build --release --bin cir_materialize --bin cir_extract_inputs --bin ccr_to_compare_views
)
fi
TMP_ROOT="$OUT_DIR/.tmp"
TALS_DIR="$TMP_ROOT/tals"
META_JSON="$TMP_ROOT/meta.json"
MIRROR_ROOT="$TMP_ROOT/mirror"
CACHE_DIR="$TMP_ROOT/cache"
OUT_CCR_DIR="$TMP_ROOT/out"
RUN_LOG="$OUT_DIR/rpki-client.log"
ACTUAL_VRPS="$OUT_DIR/actual-vrps.csv"
ACTUAL_VAPS="$OUT_DIR/actual-vaps.csv"
ACTUAL_VAPS_META="$OUT_DIR/actual-vaps-meta.json"
ACTUAL_VRPS_META="$OUT_DIR/actual-vrps-meta.json"
REF_VRPS="$OUT_DIR/reference-vrps.csv"
REF_VAPS="$OUT_DIR/reference-vaps.csv"
SUMMARY_JSON="$OUT_DIR/compare-summary.json"
rm -rf "$TMP_ROOT"
mkdir -p "$TMP_ROOT"
"$CIR_EXTRACT_INPUTS_BIN" --cir "$CIR" --tals-dir "$TALS_DIR" --meta-json "$META_JSON"
python3 - <<'PY' "$TALS_DIR"
from pathlib import Path
import sys
for tal in Path(sys.argv[1]).glob("*.tal"):
lines = tal.read_text(encoding="utf-8").splitlines()
rsync_uris = [line for line in lines if line.startswith("rsync://")]
base64_lines = []
seen_sep = False
for line in lines:
if seen_sep:
if line.strip():
base64_lines.append(line)
elif line.strip() == "":
seen_sep = True
tal.write_text("\n".join(rsync_uris) + "\n\n" + "\n".join(base64_lines) + "\n", encoding="utf-8")
PY
materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --repo-bytes-db "$REPO_BYTES_DB" --mirror-root "$MIRROR_ROOT")
if [[ "$KEEP_DB" -eq 1 ]]; then
materialize_cmd+=(--keep-db)
fi
"${materialize_cmd[@]}"
VALIDATION_EPOCH="$(python3 - <<'PY' "$META_JSON"
from datetime import datetime, timezone
import json, sys
vt = json.load(open(sys.argv[1]))["validationTime"]
dt = datetime.fromisoformat(vt.replace("Z", "+00:00")).astimezone(timezone.utc)
print(int(dt.timestamp()))
PY
)"
mapfile -t TAL_PATHS < <(python3 - <<'PY' "$META_JSON"
import json, sys
for item in json.load(open(sys.argv[1], encoding="utf-8"))["talFiles"]:
print(item["path"])
PY
)
CLIENT_TAL_ARGS=()
for tal_path in "${TAL_PATHS[@]}"; do
CLIENT_TAL_ARGS+=(-t "$tal_path")
done
COMPARE_TRUST_ANCHOR="unknown"
export CIR_MIRROR_ROOT="$(python3 - <<'PY' "$MIRROR_ROOT"
from pathlib import Path
import sys
print(Path(sys.argv[1]).resolve())
PY
)"
export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
export CIR_LOCAL_LINK_MODE=1
mkdir -p "$CACHE_DIR" "$OUT_CCR_DIR"
chmod -R 0777 "$TMP_ROOT"
"$RPKI_CLIENT_BIN" \
-R \
-e "$WRAPPER" \
-P "$VALIDATION_EPOCH" \
"${CLIENT_TAL_ARGS[@]}" \
-d "$CACHE_DIR" \
"$OUT_CCR_DIR" >"$RUN_LOG" 2>&1
if [[ -f "$OUT_CCR_DIR/rpki.ccr" ]]; then
"$CCR_TO_COMPARE_VIEWS_BIN" \
--ccr "$OUT_CCR_DIR/rpki.ccr" \
--vrps-out "$ACTUAL_VRPS" \
--vaps-out "$ACTUAL_VAPS" \
--trust-anchor "$COMPARE_TRUST_ANCHOR"
else
python3 - <<'PY' "$OUT_CCR_DIR/json" "$ACTUAL_VRPS" "$ACTUAL_VAPS" "$COMPARE_TRUST_ANCHOR"
import csv
import json
import sys
from pathlib import Path
json_path = Path(sys.argv[1])
vrps_out = Path(sys.argv[2])
vaps_out = Path(sys.argv[3])
compare_ta = sys.argv[4]
if not json_path.is_file():
raise SystemExit(f"rpki-client output has neither rpki.ccr nor json: {json_path}")
data = json.loads(json_path.read_text(encoding="utf-8"))
vrps_out.parent.mkdir(parents=True, exist_ok=True)
with vrps_out.open("w", newline="", encoding="utf-8") as fh:
writer = csv.writer(fh)
writer.writerow(["ASN", "IP Prefix", "Max Length", "Trust Anchor"])
for roa in data.get("roas", []):
writer.writerow([
f"AS{roa['asn']}",
roa["prefix"],
str(roa["maxLength"]),
compare_ta,
])
with vaps_out.open("w", newline="", encoding="utf-8") as fh:
writer = csv.writer(fh)
writer.writerow(["Customer ASN", "Providers", "Trust Anchor"])
for aspa in data.get("aspas", []):
providers = ";".join(f"AS{item}" for item in sorted(aspa.get("providers", [])))
writer.writerow([
f"AS{aspa['customer_asid']}",
providers,
compare_ta,
])
PY
fi
python3 - <<'PY' "$ACTUAL_VRPS" "$ACTUAL_VAPS" "$ACTUAL_VRPS_META" "$ACTUAL_VAPS_META"
import csv, json, sys
def count_rows(path):
with open(path, newline="") as f:
rows = list(csv.reader(f))
return max(len(rows) - 1, 0)
json.dump({"count": count_rows(sys.argv[1])}, open(sys.argv[3], "w"), indent=2)
json.dump({"count": count_rows(sys.argv[2])}, open(sys.argv[4], "w"), indent=2)
PY
"$CCR_TO_COMPARE_VIEWS_BIN" --ccr "$REFERENCE_CCR" --vrps-out "$REF_VRPS" --vaps-out "$REF_VAPS" --trust-anchor "$COMPARE_TRUST_ANCHOR"
python3 - <<'PY' "$ACTUAL_VRPS" "$REF_VRPS" "$ACTUAL_VAPS" "$REF_VAPS" "$SUMMARY_JSON" "$META_JSON"
import csv, json, sys
def rows(path):
with open(path, newline="") as f:
return list(csv.reader(f))[1:]
actual_vrps = {tuple(r) for r in rows(sys.argv[1])}
ref_vrps = {tuple(r) for r in rows(sys.argv[2])}
actual_vaps = {tuple(r) for r in rows(sys.argv[3])}
ref_vaps = {tuple(r) for r in rows(sys.argv[4])}
meta = json.load(open(sys.argv[6], encoding="utf-8"))
summary = {
"compareMode": "trust-anchor-agnostic",
"talCount": len(meta["talFiles"]),
"talPaths": [item["path"] for item in meta["talFiles"]],
"vrps": {
"actual": len(actual_vrps),
"reference": len(ref_vrps),
"match": actual_vrps == ref_vrps,
"only_in_actual": sorted(actual_vrps - ref_vrps)[:20],
"only_in_reference": sorted(ref_vrps - actual_vrps)[:20],
},
"vaps": {
"actual": len(actual_vaps),
"reference": len(ref_vaps),
"match": actual_vaps == ref_vaps,
"only_in_actual": sorted(actual_vaps - ref_vaps)[:20],
"only_in_reference": sorted(ref_vaps - actual_vaps)[:20],
}
}
with open(sys.argv[5], "w") as f:
json.dump(summary, f, indent=2)
PY
if [[ "$KEEP_DB" -ne 1 ]]; then
rm -rf "$TMP_ROOT"
fi
echo "done: $OUT_DIR"

147
scripts/cir/run_cir_replay_sequence_ours.sh
View File

@ -1,147 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/run_cir_replay_sequence_ours.sh \
--sequence-root <path> \
[--rpki-bin <path>] \
[--real-rsync-bin <path>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
SEQUENCE_ROOT=""
RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
STEP_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_ours.sh"
while [[ $# -gt 0 ]]; do
case "$1" in
--sequence-root) SEQUENCE_ROOT="$2"; shift 2 ;;
--rpki-bin) RPKI_BIN="$2"; shift 2 ;;
--real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$SEQUENCE_ROOT" ]] || { usage >&2; exit 2; }
SEQUENCE_ROOT="$(python3 - <<'PY' "$SEQUENCE_ROOT"
from pathlib import Path
import sys
print(Path(sys.argv[1]).resolve())
PY
)"
SUMMARY_JSON="$SEQUENCE_ROOT/sequence-summary.json"
SUMMARY_MD="$SEQUENCE_ROOT/sequence-summary.md"
DETAIL_JSON="$SEQUENCE_ROOT/sequence-detail.json"
python3 - <<'PY' "$SEQUENCE_ROOT" "$SUMMARY_JSON" "$SUMMARY_MD" "$DETAIL_JSON" "$STEP_SCRIPT" "$RPKI_BIN" "$REAL_RSYNC_BIN"
import json
import subprocess
import sys
from pathlib import Path
sequence_root = Path(sys.argv[1])
summary_json = Path(sys.argv[2])
summary_md = Path(sys.argv[3])
detail_json = Path(sys.argv[4])
step_script = Path(sys.argv[5])
rpki_bin = sys.argv[6]
real_rsync_bin = sys.argv[7]
sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
repo_bytes_db = sequence_root / sequence["repoBytesDbPath"]
steps = sequence["steps"]
results = []
all_match = True
for step in steps:
step_id = step["stepId"]
out_dir = sequence_root / "replay-ours" / step_id
out_dir.parent.mkdir(parents=True, exist_ok=True)
cmd = [
str(step_script),
"--cir",
str(sequence_root / step["cirPath"]),
"--out-dir",
str(out_dir),
"--reference-ccr",
str(sequence_root / step["ccrPath"]),
"--rpki-bin",
rpki_bin,
"--real-rsync-bin",
real_rsync_bin,
]
cmd.extend(["--repo-bytes-db", str(repo_bytes_db)])
proc = subprocess.run(cmd, capture_output=True, text=True)
if proc.returncode != 0:
raise SystemExit(
f"ours sequence replay failed for {step_id}: stdout={proc.stdout} stderr={proc.stderr}"
)
compare = json.loads((out_dir / "compare-summary.json").read_text(encoding="utf-8"))
timing = json.loads((out_dir / "timing.json").read_text(encoding="utf-8")) if (out_dir / "timing.json").exists() else {}
record = {
"stepId": step_id,
"kind": step["kind"],
"validationTime": step["validationTime"],
"outDir": str(out_dir),
"comparePath": str(out_dir / "compare-summary.json"),
"timingPath": str(out_dir / "timing.json"),
"compareMode": compare.get("compareMode"),
"talCount": compare.get("talCount"),
"talPaths": compare.get("talPaths", []),
"compare": compare,
"timing": timing,
"match": bool(compare["vrps"]["match"]) and bool(compare["vaps"]["match"]),
}
all_match = all_match and record["match"]
results.append(record)
summary = {
"version": 1,
"participant": "ours",
"sequenceRoot": str(sequence_root),
"stepCount": len(results),
"allMatch": all_match,
"steps": results,
}
summary_json.write_text(json.dumps(summary, indent=2), encoding="utf-8")
detail_json.write_text(json.dumps(results, indent=2), encoding="utf-8")
lines = [
"# Ours CIR Sequence Replay Summary",
"",
f"- `sequence_root`: `{sequence_root}`",
f"- `step_count`: `{len(results)}`",
f"- `all_match`: `{all_match}`",
"",
"| Step | Kind | TALs | Compare mode | VRP actual/ref | VRP match | VAP actual/ref | VAP match | Duration (ms) |",
"| --- | --- | ---: | --- | --- | --- | --- | --- | ---: |",
]
for item in results:
compare = item["compare"]
timing = item.get("timing") or {}
lines.append(
"| {step} | {kind} | {tal_count} | {compare_mode} | {va}/{vr} | {vm} | {aa}/{ar} | {am} | {dur} |".format(
step=item["stepId"],
kind=item["kind"],
tal_count=item.get("talCount") if item.get("talCount") is not None else "-",
compare_mode=item.get("compareMode") or "-",
va=compare["vrps"]["actual"],
vr=compare["vrps"]["reference"],
vm=compare["vrps"]["match"],
aa=compare["vaps"]["actual"],
ar=compare["vaps"]["reference"],
am=compare["vaps"]["match"],
dur=timing.get("durationMs", "-"),
)
)
summary_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
PY
echo "done: $SEQUENCE_ROOT"

146
scripts/cir/run_cir_replay_sequence_routinator.sh
View File

@ -1,146 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/run_cir_replay_sequence_routinator.sh \
--sequence-root <path> \
[--routinator-root <path>] \
[--routinator-bin <path>] \
[--real-rsync-bin <path>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
SEQUENCE_ROOT=""
ROUTINATOR_ROOT="${ROUTINATOR_ROOT:-/home/yuyr/dev/rust_playground/routinator}"
ROUTINATOR_BIN="${ROUTINATOR_BIN:-$ROUTINATOR_ROOT/target/debug/routinator}"
REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
STEP_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_routinator.sh"
while [[ $# -gt 0 ]]; do
case "$1" in
--sequence-root) SEQUENCE_ROOT="$2"; shift 2 ;;
--routinator-root) ROUTINATOR_ROOT="$2"; shift 2 ;;
--routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
--real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$SEQUENCE_ROOT" ]] || { usage >&2; exit 2; }
SEQUENCE_ROOT="$(python3 - <<'PY' "$SEQUENCE_ROOT"
from pathlib import Path
import sys
print(Path(sys.argv[1]).resolve())
PY
)"
SUMMARY_JSON="$SEQUENCE_ROOT/sequence-summary-routinator.json"
SUMMARY_MD="$SEQUENCE_ROOT/sequence-summary-routinator.md"
python3 - <<'PY' "$SEQUENCE_ROOT" "$SUMMARY_JSON" "$SUMMARY_MD" "$STEP_SCRIPT" "$ROUTINATOR_ROOT" "$ROUTINATOR_BIN" "$REAL_RSYNC_BIN"
import json
import subprocess
import sys
from pathlib import Path
sequence_root = Path(sys.argv[1])
summary_json = Path(sys.argv[2])
summary_md = Path(sys.argv[3])
step_script = Path(sys.argv[4])
routinator_root = sys.argv[5]
routinator_bin = sys.argv[6]
real_rsync_bin = sys.argv[7]
sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
repo_bytes_db = sequence_root / sequence["repoBytesDbPath"]
steps = sequence["steps"]
results = []
all_match = True
for step in steps:
step_id = step["stepId"]
out_dir = sequence_root / "replay-routinator" / step_id
out_dir.parent.mkdir(parents=True, exist_ok=True)
cmd = [
str(step_script),
"--cir",
str(sequence_root / step["cirPath"]),
"--out-dir",
str(out_dir),
"--reference-ccr",
str(sequence_root / step["ccrPath"]),
"--routinator-root",
routinator_root,
"--routinator-bin",
routinator_bin,
"--real-rsync-bin",
real_rsync_bin,
]
cmd.extend(["--repo-bytes-db", str(repo_bytes_db)])
proc = subprocess.run(cmd, capture_output=True, text=True)
if proc.returncode != 0:
raise SystemExit(
f"routinator sequence replay failed for {step_id}: stdout={proc.stdout} stderr={proc.stderr}"
)
compare = json.loads((out_dir / "compare-summary.json").read_text(encoding="utf-8"))
match = bool(compare["vrps"]["match"]) and bool(compare["vaps"]["match"])
all_match = all_match and match
results.append(
{
"stepId": step_id,
"kind": step["kind"],
"validationTime": step["validationTime"],
"outDir": str(out_dir),
"comparePath": str(out_dir / "compare-summary.json"),
"compareMode": compare.get("compareMode"),
"talCount": compare.get("talCount"),
"talPaths": compare.get("talPaths", []),
"match": match,
"compare": compare,
}
)
summary = {
"version": 1,
"participant": "routinator",
"sequenceRoot": str(sequence_root),
"stepCount": len(results),
"allMatch": all_match,
"steps": results,
}
summary_json.write_text(json.dumps(summary, indent=2), encoding="utf-8")
lines = [
"# Routinator CIR Sequence Replay Summary",
"",
f"- `sequence_root`: `{sequence_root}`",
f"- `step_count`: `{len(results)}`",
f"- `all_match`: `{all_match}`",
"",
"| Step | Kind | TALs | Compare mode | VRP actual/ref | VRP match | VAP actual/ref | VAP match |",
"| --- | --- | ---: | --- | --- | --- | --- | --- |",
]
for item in results:
compare = item["compare"]
lines.append(
"| {step} | {kind} | {tal_count} | {compare_mode} | {va}/{vr} | {vm} | {aa}/{ar} | {am} |".format(
step=item["stepId"],
kind=item["kind"],
tal_count=item.get("talCount") if item.get("talCount") is not None else "-",
compare_mode=item.get("compareMode") or "-",
va=compare["vrps"]["actual"],
vr=compare["vrps"]["reference"],
vm=compare["vrps"]["match"],
aa=compare["vaps"]["actual"],
ar=compare["vaps"]["reference"],
am=compare["vaps"]["match"],
)
)
summary_md.write_text("\n".join(lines), encoding="utf-8")
PY
echo "done: $SEQUENCE_ROOT"

145
scripts/cir/run_cir_replay_sequence_rpki_client.sh
View File

@ -1,145 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/run_cir_replay_sequence_rpki_client.sh \
--sequence-root <path> \
[--build-dir <path> | --rpki-client-bin <path>] \
[--real-rsync-bin <path>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
SEQUENCE_ROOT=""
BUILD_DIR=""
RPKI_CLIENT_BIN=""
REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
STEP_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_rpki_client.sh"
while [[ $# -gt 0 ]]; do
case "$1" in
--sequence-root) SEQUENCE_ROOT="$2"; shift 2 ;;
--build-dir) BUILD_DIR="$2"; shift 2 ;;
--rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
--real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$SEQUENCE_ROOT" && ( -n "$BUILD_DIR" || -n "$RPKI_CLIENT_BIN" ) ]] || { usage >&2; exit 2; }
SEQUENCE_ROOT="$(python3 - <<'PY' "$SEQUENCE_ROOT"
from pathlib import Path
import sys
print(Path(sys.argv[1]).resolve())
PY
)"
SUMMARY_JSON="$SEQUENCE_ROOT/sequence-summary-rpki-client.json"
SUMMARY_MD="$SEQUENCE_ROOT/sequence-summary-rpki-client.md"
python3 - <<'PY' "$SEQUENCE_ROOT" "$SUMMARY_JSON" "$SUMMARY_MD" "$STEP_SCRIPT" "$BUILD_DIR" "$RPKI_CLIENT_BIN" "$REAL_RSYNC_BIN"
import json
import subprocess
import sys
from pathlib import Path
sequence_root = Path(sys.argv[1])
summary_json = Path(sys.argv[2])
summary_md = Path(sys.argv[3])
step_script = Path(sys.argv[4])
build_dir = sys.argv[5]
rpki_client_bin = sys.argv[6]
real_rsync_bin = sys.argv[7]
sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
repo_bytes_db = sequence_root / sequence["repoBytesDbPath"]
steps = sequence["steps"]
results = []
all_match = True
for step in steps:
step_id = step["stepId"]
out_dir = sequence_root / "replay-rpki-client" / step_id
out_dir.parent.mkdir(parents=True, exist_ok=True)
cmd = [
str(step_script),
"--cir",
str(sequence_root / step["cirPath"]),
"--out-dir",
str(out_dir),
"--reference-ccr",
str(sequence_root / step["ccrPath"]),
"--real-rsync-bin",
real_rsync_bin,
]
if rpki_client_bin:
cmd.extend(["--rpki-client-bin", rpki_client_bin])
else:
cmd.extend(["--build-dir", build_dir])
cmd.extend(["--repo-bytes-db", str(repo_bytes_db)])
proc = subprocess.run(cmd, capture_output=True, text=True)
if proc.returncode != 0:
raise SystemExit(
f"rpki-client sequence replay failed for {step_id}: stdout={proc.stdout} stderr={proc.stderr}"
)
compare = json.loads((out_dir / "compare-summary.json").read_text(encoding="utf-8"))
match = bool(compare["vrps"]["match"]) and bool(compare["vaps"]["match"])
all_match = all_match and match
results.append(
{
"stepId": step_id,
"kind": step["kind"],
"validationTime": step["validationTime"],
"outDir": str(out_dir),
"comparePath": str(out_dir / "compare-summary.json"),
"compareMode": compare.get("compareMode"),
"talCount": compare.get("talCount"),
"talPaths": compare.get("talPaths", []),
"match": match,
"compare": compare,
}
)
summary = {
"version": 1,
"participant": "rpki-client",
"sequenceRoot": str(sequence_root),
"stepCount": len(results),
"allMatch": all_match,
"steps": results,
}
summary_json.write_text(json.dumps(summary, indent=2), encoding="utf-8")
lines = [
"# rpki-client CIR Sequence Replay Summary",
"",
f"- `sequence_root`: `{sequence_root}`",
f"- `step_count`: `{len(results)}`",
f"- `all_match`: `{all_match}`",
"",
"| Step | Kind | TALs | Compare mode | VRP actual/ref | VRP match | VAP actual/ref | VAP match |",
"| --- | --- | ---: | --- | --- | --- | --- | --- |",
]
for item in results:
compare = item["compare"]
lines.append(
"| {step} | {kind} | {tal_count} | {compare_mode} | {va}/{vr} | {vm} | {aa}/{ar} | {am} |".format(
step=item["stepId"],
kind=item["kind"],
tal_count=item.get("talCount") if item.get("talCount") is not None else "-",
compare_mode=item.get("compareMode") or "-",
va=compare["vrps"]["actual"],
vr=compare["vrps"]["reference"],
vm=compare["vrps"]["match"],
aa=compare["vaps"]["actual"],
ar=compare["vaps"]["reference"],
am=compare["vaps"]["match"],
)
)
summary_md.write_text("\n".join(lines), encoding="utf-8")
PY
echo "done: $SEQUENCE_ROOT"

132
scripts/cir/run_cir_sequence_matrix_multi_rir.sh
View File

@ -1,132 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/cir/run_cir_sequence_matrix_multi_rir.sh \
--root <path> \
[--rir <afrinic,apnic,arin,lacnic,ripe>] \
[--rpki-bin <path>] \
[--routinator-root <path>] \
[--routinator-bin <path>] \
[--rpki-client-build-dir <path>] \
[--drop-bin <path>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
ROOT=""
RIRS="afrinic,apnic,arin,lacnic,ripe"
RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
ROUTINATOR_ROOT="${ROUTINATOR_ROOT:-/home/yuyr/dev/rust_playground/routinator}"
ROUTINATOR_BIN="${ROUTINATOR_BIN:-$ROUTINATOR_ROOT/target/debug/routinator}"
RPKI_CLIENT_BUILD_DIR="${RPKI_CLIENT_BUILD_DIR:-/home/yuyr/dev/rpki-client-9.7/build-m5}"
DROP_BIN="${DROP_BIN:-$ROOT_DIR/target/release/cir_drop_report}"
OURS_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_sequence_ours.sh"
ROUTINATOR_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_sequence_routinator.sh"
RPKIC_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_sequence_rpki_client.sh"
DROP_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_drop_sequence.sh"
while [[ $# -gt 0 ]]; do
case "$1" in
--root) ROOT="$2"; shift 2 ;;
--rir) RIRS="$2"; shift 2 ;;
--rpki-bin) RPKI_BIN="$2"; shift 2 ;;
--routinator-root) ROUTINATOR_ROOT="$2"; shift 2 ;;
--routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
--rpki-client-build-dir) RPKI_CLIENT_BUILD_DIR="$2"; shift 2 ;;
--drop-bin) DROP_BIN="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$ROOT" ]] || { usage >&2; exit 2; }
SUMMARY_JSON="$ROOT/final-summary.json"
SUMMARY_MD="$ROOT/final-summary.md"
IFS=',' read -r -a ITEMS <<< "$RIRS"
results=()
for rir in "${ITEMS[@]}"; do
seq_root="$ROOT/$rir"
"$OURS_SCRIPT" --sequence-root "$seq_root" --rpki-bin "$RPKI_BIN"
"$ROUTINATOR_SCRIPT" --sequence-root "$seq_root" --routinator-root "$ROUTINATOR_ROOT" --routinator-bin "$ROUTINATOR_BIN"
"$RPKIC_SCRIPT" --sequence-root "$seq_root" --build-dir "$RPKI_CLIENT_BUILD_DIR"
"$DROP_SCRIPT" --sequence-root "$seq_root" --drop-bin "$DROP_BIN"
done
python3 - <<'PY' "$ROOT" "$RIRS" "$SUMMARY_JSON" "$SUMMARY_MD"
import json, sys
from pathlib import Path
from collections import Counter
root = Path(sys.argv[1]).resolve()
rirs = [item for item in sys.argv[2].split(',') if item]
summary_json = Path(sys.argv[3])
summary_md = Path(sys.argv[4])
items = []
total_steps = 0
total_dropped_vrps = 0
total_dropped_objects = 0
reason_counter = Counter()
for rir in rirs:
seq_root = root / rir
ours = json.loads((seq_root / "sequence-summary.json").read_text(encoding="utf-8"))
routinator = json.loads((seq_root / "sequence-summary-routinator.json").read_text(encoding="utf-8"))
rpki_client = json.loads((seq_root / "sequence-summary-rpki-client.json").read_text(encoding="utf-8"))
drop = json.loads((seq_root / "drop-summary.json").read_text(encoding="utf-8"))
step_count = len(ours["steps"])
total_steps += step_count
rir_dropped_vrps = 0
rir_dropped_objects = 0
for step in drop["steps"]:
drop_path = Path(step["reportPath"])
detail = json.loads(drop_path.read_text(encoding="utf-8"))
summary = detail.get("summary", {})
rir_dropped_vrps += int(summary.get("droppedVrpCount", 0))
rir_dropped_objects += int(summary.get("droppedObjectCount", 0))
total_dropped_vrps += int(summary.get("droppedVrpCount", 0))
total_dropped_objects += int(summary.get("droppedObjectCount", 0))
for reason, count in summary.get("droppedByReason", {}).items():
reason_counter[reason] += int(count)
items.append({
"rir": rir,
"stepCount": step_count,
"oursAllMatch": ours["allMatch"],
"routinatorAllMatch": routinator["allMatch"],
"rpkiClientAllMatch": rpki_client["allMatch"],
"dropSummary": drop["steps"],
"droppedVrpCount": rir_dropped_vrps,
"droppedObjectCount": rir_dropped_objects,
})
summary = {
"version": 1,
"totalStepCount": total_steps,
"totalDroppedVrpCount": total_dropped_vrps,
"totalDroppedObjectCount": total_dropped_objects,
"topReasons": [{"reason": reason, "count": count} for reason, count in reason_counter.most_common(10)],
"rirs": items,
}
summary_json.write_text(json.dumps(summary, indent=2), encoding="utf-8")
lines = ["# Multi-RIR CIR Sequence Matrix Summary", ""]
lines.append(f"- `total_step_count`: `{total_steps}`")
lines.append(f"- `total_dropped_vrps`: `{total_dropped_vrps}`")
lines.append(f"- `total_dropped_objects`: `{total_dropped_objects}`")
lines.append("")
if reason_counter:
lines.append("## Top Drop Reasons")
lines.append("")
for reason, count in reason_counter.most_common(10):
lines.append(f"- `{reason}`: `{count}`")
lines.append("")
for item in items:
lines.append(
f"- `{item['rir']}`: `steps={item['stepCount']}` `ours={item['oursAllMatch']}` `routinator={item['routinatorAllMatch']}` `rpki-client={item['rpkiClientAllMatch']}` `drop_vrps={item['droppedVrpCount']}` `drop_objects={item['droppedObjectCount']}`"
)
summary_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
PY
echo "done: $ROOT"

502
scripts/compare/run_perf_compare_quick_remote.sh
View File

@ -1,502 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/compare/run_perf_compare_quick_remote.sh \
--run-root <path> \
--remote-root <path> \
[--rir-set <mixed2|all5>] \
[--ssh-target <user@host>] \
[--rpki-client-bin <path>] \
[--libtls-path <path>] \
[--rp-run-mode <serial|parallel>] \
[--copy-rpki-client-cache] \
[--probe-rpki-client-cache] \
[--ours-extra-args '<args>'] \
[--dry-run]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
first_existing_executable() {
local fallback="$1"
shift
local candidate
for candidate in "$@"; do
if [[ -x "$candidate" ]]; then
printf '%s' "$candidate"
return
fi
done
printf '%s' "$fallback"
}
first_existing_file() {
local fallback="$1"
shift
local candidate
for candidate in "$@"; do
if [[ -f "$candidate" ]]; then
printf '%s' "$candidate"
return
fi
done
printf '%s' "$fallback"
}
RUN_ROOT=""
REMOTE_ROOT=""
SSH_TARGET="${SSH_TARGET:-root@47.251.56.108}"
RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-$(first_existing_executable \
"/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client" \
"$ROOT_DIR/../../.cache/rpki-client-9.7-build/bin/rpki-client" \
"$ROOT_DIR/../../.cache/rpki-client-9.7-build/src/rpki-client-9.7/src/rpki-client" \
"$ROOT_DIR/../../.cache/rpki-client-remote9.0/rpki-client" \
"/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client")}"
LIBTLS_PATH="${LIBTLS_PATH:-$(first_existing_file \
"/home/yuyr/dev/rpki-client-9.7/.deps/libtls/root/usr/lib/x86_64-linux-gnu/libtls.so.28.0.0" \
"$ROOT_DIR/../../.cache/rpki-client-9.7-build/runlib/libtls.so.28" \
"$ROOT_DIR/../../.cache/rpki-client-9.7-build/sysroot/usr/lib/x86_64-linux-gnu/libtls.so.28.0.0" \
"$ROOT_DIR/../../.cache/rpki-client-remote9.0/libtls.so.28" \
"/home/yuyr/dev/rpki-client-9.7/.deps/libtls/root/usr/lib/x86_64-linux-gnu/libtls.so.28.0.0")}"
RP_RUN_MODE="${RP_RUN_MODE:-serial}"
RIR_SET="${RIR_SET:-mixed2}"
OURS_EXTRA_ARGS="${OURS_EXTRA_ARGS:-}"
COPY_RPKI_CLIENT_CACHE="${COPY_RPKI_CLIENT_CACHE:-0}"
PROBE_RPKI_CLIENT_CACHE="${PROBE_RPKI_CLIENT_CACHE:-0}"
DRY_RUN=0
while [[ $# -gt 0 ]]; do
case "$1" in
--run-root) RUN_ROOT="$2"; shift 2 ;;
--remote-root) REMOTE_ROOT="$2"; shift 2 ;;
--rir-set) RIR_SET="$2"; shift 2 ;;
--ssh-target) SSH_TARGET="$2"; shift 2 ;;
--rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
--libtls-path) LIBTLS_PATH="$2"; shift 2 ;;
--rp-run-mode) RP_RUN_MODE="$2"; shift 2 ;;
--copy-rpki-client-cache) COPY_RPKI_CLIENT_CACHE=1; shift ;;
--probe-rpki-client-cache) PROBE_RPKI_CLIENT_CACHE=1; shift ;;
--ours-extra-args) OURS_EXTRA_ARGS="$2"; shift 2 ;;
--dry-run) DRY_RUN=1; shift ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$RUN_ROOT" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
[[ "$RP_RUN_MODE" == "serial" || "$RP_RUN_MODE" == "parallel" ]] || { echo "invalid --rp-run-mode: $RP_RUN_MODE" >&2; usage; exit 2; }
[[ "$RIR_SET" == "mixed2" || "$RIR_SET" == "all5" ]] || { echo "invalid --rir-set: $RIR_SET" >&2; usage; exit 2; }
[[ "$DRY_RUN" -eq 1 || -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
[[ "$DRY_RUN" -eq 1 || -f "$LIBTLS_PATH" ]] || { echo "libtls not found: $LIBTLS_PATH" >&2; exit 2; }
RUN_ROOT="$(python3 - <<'PY' "$RUN_ROOT"
from pathlib import Path
import sys
print(Path(sys.argv[1]).resolve())
PY
)"
mkdir -p "$RUN_ROOT/steps/step-001/ours" "$RUN_ROOT/steps/step-001/rpki-client" "$RUN_ROOT/steps/step-001/compare"
mkdir -p "$RUN_ROOT/steps/step-002/ours" "$RUN_ROOT/steps/step-002/rpki-client" "$RUN_ROOT/steps/step-002/compare"
tal_path_for_rir() {
case "$1" in
afrinic) printf '%s' "$ROOT_DIR/tests/fixtures/tal/afrinic.tal" ;;
apnic) printf '%s' "$ROOT_DIR/tests/fixtures/tal/apnic-rfc7730-https.tal" ;;
arin) printf '%s' "$ROOT_DIR/tests/fixtures/tal/arin.tal" ;;
lacnic) printf '%s' "$ROOT_DIR/tests/fixtures/tal/lacnic.tal" ;;
ripe) printf '%s' "$ROOT_DIR/tests/fixtures/tal/ripe-ncc.tal" ;;
*) echo "unknown rir: $1" >&2; exit 2 ;;
esac
}
ta_path_for_rir() {
case "$1" in
afrinic) printf '%s' "$ROOT_DIR/tests/fixtures/ta/afrinic-ta.cer" ;;
apnic) printf '%s' "$ROOT_DIR/tests/fixtures/ta/apnic-ta.cer" ;;
arin) printf '%s' "$ROOT_DIR/tests/fixtures/ta/arin-ta.cer" ;;
lacnic) printf '%s' "$ROOT_DIR/tests/fixtures/ta/lacnic-ta.cer" ;;
ripe) printf '%s' "$ROOT_DIR/tests/fixtures/ta/ripe-ncc-ta.cer" ;;
*) echo "unknown rir: $1" >&2; exit 2 ;;
esac
}
case "$RIR_SET" in
mixed2)
RIRS=(apnic arin)
SCOPE_LABEL="APNIC+ARIN mixed release two-step synchronized compare"
;;
all5)
RIRS=(afrinic apnic arin lacnic ripe)
SCOPE_LABEL="all-five-RIR mixed release two-step synchronized compare"
;;
esac
COPY_FILES=()
for rir in "${RIRS[@]}"; do
COPY_FILES+=("$(tal_path_for_rir "$rir")" "$(ta_path_for_rir "$rir")")
done
if [[ "$DRY_RUN" -eq 1 ]]; then
cat <<EOF2
workflow_name=性能对比测试快速版
scope=$SCOPE_LABEL
rir_set=$RIR_SET
rirs=${RIRS[*]}
run_root=$RUN_ROOT
remote_root=$REMOTE_ROOT
ssh_target=$SSH_TARGET
rp_run_mode=$RP_RUN_MODE
ours_extra_args=$OURS_EXTRA_ARGS
EOF2
exit 0
fi
cleanup_remote() {
if [[ "${KEEP_REMOTE:-0}" != "1" ]]; then
ssh "$SSH_TARGET" "rm -rf '$REMOTE_ROOT'" >/dev/null 2>&1 || true
fi
}
trap cleanup_remote EXIT
(
cd "$ROOT_DIR"
cargo build --release --bin rpki --bin ccr_to_compare_views --bin ccr_state_compare --bin cir_state_compare --bin cir_probe_rpki_client_cache
)
ssh "$SSH_TARGET" "set -e; systemctl disable --now rpki-client.timer >/dev/null 2>&1 || true; systemctl stop rpki-client.service >/dev/null 2>&1 || true; pkill -f '[/]rpki-client([[:space:]]|$)' >/dev/null 2>&1 || true; pkill -f '[/]routinator([[:space:]]|$)' >/dev/null 2>&1 || true; id -u _rpki-client >/dev/null 2>&1 || useradd -r -M -s /usr/sbin/nologin _rpki-client || true; rm -rf '$REMOTE_ROOT'; mkdir -p '$REMOTE_ROOT/bin' '$REMOTE_ROOT/lib' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client' '$REMOTE_ROOT/steps/step-001/ours' '$REMOTE_ROOT/steps/step-001/rpki-client' '$REMOTE_ROOT/steps/step-002/ours' '$REMOTE_ROOT/steps/step-002/rpki-client'"
scp "$ROOT_DIR/target/release/rpki" "${COPY_FILES[@]}" "$SSH_TARGET:$REMOTE_ROOT/"
if [[ "$PROBE_RPKI_CLIENT_CACHE" == "1" ]]; then
scp "$ROOT_DIR/target/release/cir_probe_rpki_client_cache" "$SSH_TARGET:$REMOTE_ROOT/bin/"
fi
scp "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_ROOT/bin/rpki-client"
scp "$LIBTLS_PATH" "$SSH_TARGET:$REMOTE_ROOT/lib/libtls.so.28"
printf '%s' "$OURS_EXTRA_ARGS" | ssh "$SSH_TARGET" "cat > '$REMOTE_ROOT/ours-extra-args.txt'"
printf '%s' "$RP_RUN_MODE" | ssh "$SSH_TARGET" "cat > '$REMOTE_ROOT/rp-run-mode.txt'"
printf '%s' "$RIR_SET" | ssh "$SSH_TARGET" "cat > '$REMOTE_ROOT/rir-set.txt'"
run_step() {
local step_id="$1"
local kind="$2"
local local_step="$RUN_ROOT/steps/$step_id"
ssh "$SSH_TARGET" bash -s -- "$REMOTE_ROOT" "$step_id" "$kind" <<'EOS'
set -euo pipefail
REMOTE_ROOT="$1"
STEP_ID="$2"
KIND="$3"
cd "$REMOTE_ROOT"
mkdir -p "steps/$STEP_ID/ours" "steps/$STEP_ID/rpki-client"
touch rpki-client-skiplist
chmod 0644 rpki-client-skiplist
OURS_EXTRA_ARGS="$(cat ours-extra-args.txt)"
RP_RUN_MODE="$(cat rp-run-mode.txt)"
RIR_SET="$(cat rir-set.txt)"
OURS_EXTRA_ARGV=()
if [[ -n "$OURS_EXTRA_ARGS" ]]; then
# shellcheck disable=SC2206
OURS_EXTRA_ARGV=($OURS_EXTRA_ARGS)
fi
case "$RIR_SET" in
mixed2) RIRS=(apnic arin) ;;
all5) RIRS=(afrinic apnic arin lacnic ripe) ;;
*) echo "invalid rir set: $RIR_SET" >&2; exit 2 ;;
esac
tal_file_for_rir() {
case "$1" in
afrinic) printf '%s' "afrinic.tal" ;;
apnic) printf '%s' "apnic-rfc7730-https.tal" ;;
arin) printf '%s' "arin.tal" ;;
lacnic) printf '%s' "lacnic.tal" ;;
ripe) printf '%s' "ripe-ncc.tal" ;;
*) echo "unknown rir: $1" >&2; exit 2 ;;
esac
}
tal_uri_for_rir() {
case "$1" in
afrinic) printf '%s' "https://rpki.afrinic.net/repository/AfriNIC.cer" ;;
apnic) printf '%s' "https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer" ;;
arin) printf '%s' "https://rrdp.arin.net/arin-rpki-ta.cer" ;;
lacnic) printf '%s' "https://rrdp.lacnic.net/ta/rta-lacnic-rpki.cer" ;;
ripe) printf '%s' "https://rpki.ripe.net/ta/ripe-ncc-ta.cer" ;;
*) echo "unknown rir: $1" >&2; exit 2 ;;
esac
}
ta_file_for_rir() {
case "$1" in
afrinic) printf '%s' "afrinic-ta.cer" ;;
apnic) printf '%s' "apnic-ta.cer" ;;
arin) printf '%s' "arin-ta.cer" ;;
lacnic) printf '%s' "lacnic-ta.cer" ;;
ripe) printf '%s' "ripe-ncc-ta.cer" ;;
*) echo "unknown rir: $1" >&2; exit 2 ;;
esac
}
refresh_ta_file_for_rir() {
local rir="$1"
local uri
local file
uri="$(tal_uri_for_rir "$rir")"
file="$(ta_file_for_rir "$rir")"
python3 - <<'PY' "$uri" "$file"
import sys
import urllib.request
uri, path = sys.argv[1:]
request = urllib.request.Request(uri, headers={"User-Agent": "rpki-dev/compare-fast-path"})
with urllib.request.urlopen(request, timeout=30) as response:
data = response.read()
if not data:
raise SystemExit(f"empty TA certificate response: {uri}")
with open(path, "wb") as output:
output.write(data)
PY
}
for rir in "${RIRS[@]}"; do
refresh_ta_file_for_rir "$rir"
done
OURS_TAL_ARGS=()
CLIENT_TAL_ARGS=()
OURS_CIR_TAL_ARGS=()
for rir in "${RIRS[@]}"; do
tal_file="$(tal_file_for_rir "$rir")"
ta_file="$(ta_file_for_rir "$rir")"
tal_uri="$(tal_uri_for_rir "$rir")"
OURS_TAL_ARGS+=(--tal-path "$tal_file" --ta-path "$ta_file")
OURS_CIR_TAL_ARGS+=(--cir-tal-uri "$tal_uri")
CLIENT_TAL_ARGS+=(-t "../../$tal_file")
done
if [[ "$KIND" == "snapshot" ]]; then
rm -rf state/ours/work-db state/ours/raw-store.db state/ours/repo-bytes.db state/rpki-client/cache state/rpki-client/out state/rpki-client/ta state/rpki-client/.ta
fi
mkdir -p state/ours/work-db state/ours/raw-store.db state/ours/repo-bytes.db state/rpki-client/cache state/rpki-client/out state/rpki-client/ta state/rpki-client/.ta
chmod 0777 state/ours/work-db state/ours/raw-store.db state/ours/repo-bytes.db
chmod -R 0777 state/rpki-client
touch state/rpki-client/rpki-client-skiplist
chmod 0644 state/rpki-client/rpki-client-skiplist
START_EPOCH="$(python3 - <<'PY'
import time
print(time.time() + 3.0)
PY
)"
run_ours() {
python3 - <<'PY' "$START_EPOCH"
import sys, time
x = float(sys.argv[1])
d = x - time.time()
if d > 0:
time.sleep(d)
PY
started_ms="$(python3 - <<'PY'
import time
print(int(time.time() * 1000))
PY
)"
set +e
env RPKI_PROGRESS_LOG=1 RPKI_PROGRESS_SLOW_SECS=0 ./rpki \
--db state/ours/work-db \
--raw-store-db state/ours/raw-store.db \
--repo-bytes-db state/ours/repo-bytes.db \
"${OURS_TAL_ARGS[@]}" \
"${OURS_EXTRA_ARGV[@]}" \
--ccr-out "steps/$STEP_ID/ours/result.ccr" \
--cir-enable \
--cir-out "steps/$STEP_ID/ours/result.cir" \
"${OURS_CIR_TAL_ARGS[@]}" \
--report-json "steps/$STEP_ID/ours/report.json" \
> "steps/$STEP_ID/ours/run.log" 2>&1
exit_code=$?
set -e
finished_ms="$(python3 - <<'PY'
import time
print(int(time.time() * 1000))
PY
)"
python3 - <<'PY' "steps/$STEP_ID/ours/round-result.json" "$STEP_ID" "$KIND" "$started_ms" "$finished_ms" "$exit_code"
import json, sys
path, step_id, kind, started_ms, finished_ms, exit_code = sys.argv[1:]
json.dump(
{
"stepId": step_id,
"kind": kind,
"durationMs": int(finished_ms) - int(started_ms),
"exitCode": int(exit_code),
},
open(path, "w"),
indent=2,
)
PY
}
run_client() {
cd state/rpki-client
python3 - <<'PY' "$START_EPOCH"
import sys, time
x = float(sys.argv[1])
d = x - time.time()
if d > 0:
time.sleep(d)
PY
started_ms="$(python3 - <<'PY'
import time
print(int(time.time() * 1000))
PY
)"
set +e
LD_LIBRARY_PATH="$REMOTE_ROOT/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" "$REMOTE_ROOT/bin/rpki-client" \
-vv \
-S rpki-client-skiplist \
"${CLIENT_TAL_ARGS[@]}" \
-d cache out \
> "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/run.log" 2>&1
exit_code=$?
set -e
cp out/rpki.ccr "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/result.ccr" 2>/dev/null || true
cp out/rpki.cir "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/result.cir" 2>/dev/null || true
cp out/openbgpd "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/openbgpd" 2>/dev/null || true
finished_ms="$(python3 - <<'PY'
import time
print(int(time.time() * 1000))
PY
)"
python3 - <<'PY' "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/round-result.json" "$STEP_ID" "$KIND" "$started_ms" "$finished_ms" "$exit_code"
import json, sys
path, step_id, kind, started_ms, finished_ms, exit_code = sys.argv[1:]
json.dump(
{
"stepId": step_id,
"kind": kind,
"durationMs": int(finished_ms) - int(started_ms),
"exitCode": int(exit_code),
},
open(path, "w"),
indent=2,
)
PY
}
if [[ "$RP_RUN_MODE" == "parallel" ]]; then
run_ours &
OURS_PID=$!
run_client &
CLIENT_PID=$!
wait "$OURS_PID"
wait "$CLIENT_PID"
else
run_ours
run_client
fi
EOS
for rel in result.ccr result.cir round-result.json run.log stage-timing.json; do
scp -C "$SSH_TARGET:$REMOTE_ROOT/steps/$step_id/ours/$rel" "$local_step/ours/"
done
for rel in result.ccr result.cir round-result.json run.log openbgpd; do
scp -C "$SSH_TARGET:$REMOTE_ROOT/steps/$step_id/rpki-client/$rel" "$local_step/rpki-client/" || true
done
if [[ "$COPY_RPKI_CLIENT_CACHE" == "1" ]]; then
mkdir -p "$local_step/rpki-client/cache"
rsync -a --delete "$SSH_TARGET:$REMOTE_ROOT/state/rpki-client/cache/" "$local_step/rpki-client/cache/"
fi
if [[ -f "$local_step/ours/result.cir" && -f "$local_step/rpki-client/result.cir" ]]; then
"$ROOT_DIR/scripts/periodic/compare_ccr_cir_round.sh" \
--ours-ccr "$local_step/ours/result.ccr" \
--rpki-client-ccr "$local_step/rpki-client/result.ccr" \
--ours-cir "$local_step/ours/result.cir" \
--rpki-client-cir "$local_step/rpki-client/result.cir" \
--out-dir "$local_step/compare" \
--trust-anchor unknown >/dev/null
if [[ "$PROBE_RPKI_CLIENT_CACHE" == "1" ]]; then
ssh "$SSH_TARGET" "set -e; mkdir -p '$REMOTE_ROOT/steps/$step_id/compare/cir'; '$REMOTE_ROOT/bin/cir_probe_rpki_client_cache' --ours-cir '$REMOTE_ROOT/steps/$step_id/ours/result.cir' --rpki-client-cir '$REMOTE_ROOT/steps/$step_id/rpki-client/result.cir' --cache-root '$REMOTE_ROOT/state/rpki-client/cache' --rpki-client-log '$REMOTE_ROOT/steps/$step_id/rpki-client/run.log' --out-json '$REMOTE_ROOT/steps/$step_id/compare/cir/rpki-client-cache-probe.json' --sample-limit 50 >/dev/null"
scp -C "$SSH_TARGET:$REMOTE_ROOT/steps/$step_id/compare/cir/rpki-client-cache-probe.json" "$local_step/compare/cir/"
fi
if [[ "$COPY_RPKI_CLIENT_CACHE" == "1" ]]; then
"$ROOT_DIR/target/release/cir_probe_rpki_client_cache" \
--ours-cir "$local_step/ours/result.cir" \
--rpki-client-cir "$local_step/rpki-client/result.cir" \
--cache-root "$local_step/rpki-client/cache" \
--rpki-client-log "$local_step/rpki-client/run.log" \
--out-json "$local_step/compare/cir/rpki-client-cache-probe.json" \
--sample-limit 50 >/dev/null
fi
else
"$ROOT_DIR/scripts/periodic/compare_ccr_round.sh" \
--ours-ccr "$local_step/ours/result.ccr" \
--rpki-client-ccr "$local_step/rpki-client/result.ccr" \
--out-dir "$local_step/compare" \
--trust-anchor unknown >/dev/null
fi
python3 - <<'PY' "$local_step/ours/round-result.json" "$local_step/rpki-client/round-result.json" "$local_step/ours/stage-timing.json" "$local_step/compare/summary.json" "$local_step/compare/compare-summary.json" "$local_step/step-summary.json" "$OURS_EXTRA_ARGS"
import json, sys
ours = json.load(open(sys.argv[1]))
client = json.load(open(sys.argv[2]))
stage = json.load(open(sys.argv[3]))
compare_path = sys.argv[4] if __import__('pathlib').Path(sys.argv[4]).exists() else sys.argv[5]
compare = json.load(open(compare_path))
ours_extra_args = sys.argv[7]
json.dump(
{
"stepId": ours["stepId"],
"kind": ours["kind"],
"oursExtraArgs": ours_extra_args,
"oursDurationMs": ours["durationMs"],
"rpkiClientDurationMs": client["durationMs"],
"oursExitCode": ours["exitCode"],
"rpkiClientExitCode": client["exitCode"],
"oursTotalMs": stage["total_ms"],
"oursRepoSyncMsTotal": stage["repo_sync_ms_total"],
"oursPublicationPointRepoSyncMsTotal": stage.get("publication_point_repo_sync_ms_total"),
"oursDownloadEventCount": stage.get("download_event_count"),
"oursRrdpDownloadMsTotal": stage.get("rrdp_download_ms_total"),
"oursRsyncDownloadMsTotal": stage.get("rsync_download_ms_total"),
"oursDownloadBytesTotal": stage.get("download_bytes_total"),
"oursVrps": compare["vrps"]["ours"],
"rpkiClientVrps": compare["vrps"]["rpkiClient"],
"oursVaps": compare["vaps"]["ours"],
"rpkiClientVaps": compare["vaps"]["rpkiClient"],
"vrpMatch": compare["vrps"]["match"],
"vapMatch": compare["vaps"]["match"],
"allMatch": compare["allMatch"],
"onlyInOurs": len(compare["vrps"]["onlyInOurs"]),
"onlyInRpkiClient": len(compare["vrps"]["onlyInRpkiClient"]),
},
open(sys.argv[6], "w"),
indent=2,
)
PY
}
run_step step-001 snapshot
run_step step-002 delta
python3 - <<'PY' "$RUN_ROOT/steps/step-001/step-summary.json" "$RUN_ROOT/steps/step-002/step-summary.json" "$RUN_ROOT/summary.json" "$RP_RUN_MODE" "$OURS_EXTRA_ARGS" "$RIR_SET" "$SCOPE_LABEL" "${RIRS[@]}"
import json, sys
steps = [json.load(open(p)) for p in sys.argv[1:3]]
summary = {
"workflowName": "性能对比测试快速版",
"scope": sys.argv[7],
"rpRunMode": sys.argv[4],
"oursExtraArgs": sys.argv[5],
"rirSet": sys.argv[6],
"rirs": sys.argv[8:],
"steps": steps,
}
json.dump(summary, open(sys.argv[3], "w"), indent=2, ensure_ascii=False)
print(json.dumps(summary, indent=2, ensure_ascii=False))
PY

1706
scripts/compare/run_three_rp_10run_benchmark.py
View File

File diff suppressed because it is too large Load Diff

104
scripts/coverage.sh
View File

@ -1,104 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Requires:
# rustup component add llvm-tools-preview
# cargo install cargo-llvm-cov --locked
# Optional:
# COVERAGE_FORCE_CLEAN=1 Force `cargo llvm-cov clean --workspace` before the run.
# Default behavior is to reuse existing llvm-cov build artifacts.
# RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS=1 Skip replay/matrix integration tests that
# spawn shell pipelines and can trigger separate release builds.
# coverage.sh enables this by default.
# RPKI_SKIP_HEAVY_BLACKBOX_TESTS=1 Skip slower blackbox CLI/script integration tests
# that provide low incremental coverage per wall-clock second.
# coverage.sh enables this by default.
# RPKI_SKIP_HEAVY_CRYPTO_TESTS=1 Skip slower OpenSSL-heavy certificate generation tests
# that provide low incremental coverage per wall-clock second.
# coverage.sh enables this by default.
run_out="$(mktemp)"
text_out="$(mktemp)"
html_out="$(mktemp)"
cleanup() {
rm -f "$run_out" "$text_out" "$html_out"
}
trap cleanup EXIT
IGNORE_REGEX='repository_view_stats\.rs|db_stats\.rs|rrdp_state_dump\.rs|ccr_dump\.rs|ccr_verify\.rs|ccr_to_routinator_csv\.rs|ccr_to_compare_views\.rs|cir_materialize\.rs|cir_extract_inputs\.rs|cir_drop_report\.rs|cir_ta_only_fixture\.rs|cir_dump_reject_list\.rs|rpki_object_parse\.rs|rpki_query_indexer\.rs|rpki_query_service\.rs|triage_ccr_cir_pair\.rs|rpki_artifact_metrics|rpki_inter_rp_metrics|rpki_daemon\.rs|sequence_triage_ccr_cir|ccr_state_compare\.rs|cir_state_compare\.rs|cir_probe_rpki_client_cache\.rs|ccr/compare_view\.rs|progress_log\.rs|cli\.rs|validation/run_tree_from_tal\.rs|validation/tree_parallel\.rs|validation/tree_runner|validation/from_tal\.rs|sync/store_projection\.rs|sync/repo\.rs|sync/rrdp|(^|/)storage(/|\.rs$)|cir/materialize\.rs'
# Preserve colored output even though we post-process output by running under a pseudo-TTY.
# We run tests only once, then generate both CLI text + HTML reports without rerunning tests.
set +e
if [ "${COVERAGE_FORCE_CLEAN:-0}" = "1" ]; then
cargo llvm-cov clean --workspace >/dev/null 2>&1
echo "coverage mode: clean build (COVERAGE_FORCE_CLEAN=1)"
else
echo "coverage mode: reuse existing llvm-cov artifacts (default)"
fi
export RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS="${RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS:-1}"
export RPKI_SKIP_HEAVY_BLACKBOX_TESTS="${RPKI_SKIP_HEAVY_BLACKBOX_TESTS:-1}"
export RPKI_SKIP_HEAVY_CRYPTO_TESTS="${RPKI_SKIP_HEAVY_CRYPTO_TESTS:-1}"
# 1) Run tests once to collect coverage data (no report).
script -q -e -c "CARGO_TERM_COLOR=always cargo llvm-cov --no-report" "$run_out" >/dev/null 2>&1
run_status="$?"
# 2) CLI summary report + fail-under gate (no test rerun).
script -q -e -c "CARGO_TERM_COLOR=always cargo llvm-cov report --fail-under-lines 90 --ignore-filename-regex '$IGNORE_REGEX'" "$text_out" >/dev/null 2>&1
text_status="$?"
# 3) HTML report (no test rerun).
script -q -e -c "CARGO_TERM_COLOR=always cargo llvm-cov report --html --ignore-filename-regex '$IGNORE_REGEX'" "$html_out" >/dev/null 2>&1
html_status="$?"
set -e
strip_script_noise() {
tr -d '\r' | sed '/^Script \(started\|done\) on /d'
}
strip_ansi_for_parse() {
awk '
{
line = $0
gsub(/\033\[[0-9;]*[A-Za-z]/, "", line) # CSI escapes
gsub(/\033\([A-Za-z]/, "", line) # charset escapes (e.g., ESC(B)
gsub(/\r/, "", line)
print line
}
'
}
cat "$run_out" | strip_script_noise
cat "$text_out" | strip_script_noise
cat "$html_out" | strip_script_noise
cat "$run_out" | strip_ansi_for_parse | awk '
BEGIN {
passed=0; failed=0; ignored=0; measured=0; filtered=0;
}
/^test result: / {
if (match($0, /([0-9]+) passed; ([0-9]+) failed; ([0-9]+) ignored; ([0-9]+) measured; ([0-9]+) filtered out;/, m)) {
passed += m[1]; failed += m[2]; ignored += m[3]; measured += m[4]; filtered += m[5];
}
}
END {
executed = passed + failed;
total = passed + failed + ignored + measured;
printf("\nTEST SUMMARY (all suites): passed=%d failed=%d ignored=%d measured=%d filtered_out=%d executed=%d total=%d\n",
passed, failed, ignored, measured, filtered, executed, total);
}
'
echo
echo "HTML report: target/llvm-cov/html/index.html"
status="$text_status"
if [ "$run_status" -ne 0 ]; then status="$run_status"; fi
if [ "$html_status" -ne 0 ]; then status="$html_status"; fi
exit "$status"

5
scripts/docker/build_arm64_installer_package.sh
View File

@ -1,5 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
exec "$SCRIPT_DIR/build_docker_installer_package.sh" --arch arm64 "$@"

5
scripts/docker/build_arm64_metrics_image.sh
View File

@ -1,5 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
exec "$SCRIPT_DIR/build_docker_metrics_image.sh" --arch arm64 "$@"

5
scripts/docker/build_arm64_runtime_image.sh
View File

@ -1,5 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
exec "$SCRIPT_DIR/build_docker_runtime_image.sh" --arch arm64 "$@"

146
scripts/docker/build_custom_tal_patch.sh
View File

@ -1,146 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$BASH_SOURCE")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
PATCH_SOURCE="$REPO_ROOT/deploy/docker-installer/custom-tal-patch"
BASE_COMPONENT_PACKAGE=""
OUT_DIR="$(printenv OUT_DIR 2>/dev/null || true)"
PREFIX="$(printenv PREFIX 2>/dev/null || true)"
ALLOW_DIRTY=0
[[ -n "$OUT_DIR" ]] || OUT_DIR="$REPO_ROOT/target/custom-tal-patch"
[[ -n "$PREFIX" ]] || PREFIX="ours-rp-custom-tal-patch"
usage() {
cat <<'USAGE'
Usage:
scripts/docker/build_custom_tal_patch.sh \
--base-component-package <ours-rp-installer-*.tar.gz> \
[--out-dir <dir>] [--prefix <name>] [--allow-dirty]
The patch targets the Arm64 component package from the customer stack. It
overlays the runner, Compose, fixture mount and test-service tools; it does not
rebuild or replace the runtime image.
USAGE
}
die() {
echo "error: $*" >&2
exit 2
}
while [[ $# -gt 0 ]]; do
case "$1" in
--base-component-package) BASE_COMPONENT_PACKAGE="$2"; shift 2 ;;
--out-dir) OUT_DIR="$2"; shift 2 ;;
--prefix) PREFIX="$2"; shift 2 ;;
--allow-dirty) ALLOW_DIRTY=1; shift ;;
-h|--help) usage; exit 0 ;;
*) die "unknown option: $1" ;;
esac
done
[[ -f "$BASE_COMPONENT_PACKAGE" ]] || die "missing base component package: $BASE_COMPONENT_PACKAGE"
[[ -d "$PATCH_SOURCE" ]] || die "missing patch source: $PATCH_SOURCE"
SOURCE_COMMIT="$(git -C "$REPO_ROOT" rev-parse HEAD)"
SOURCE_COMMIT_SHORT="$(git -C "$REPO_ROOT" rev-parse --short=8 HEAD)"
SOURCE_DIRTY=false
if [[ -n "$(git -C "$REPO_ROOT" status --short)" ]]; then
SOURCE_DIRTY=true
fi
if [[ "$SOURCE_DIRTY" == true && "$ALLOW_DIRTY" != 1 ]]; then
die "source worktree is dirty; use --allow-dirty for a development patch"
fi
package_root="$(tar -tzf "$BASE_COMPONENT_PACKAGE" | awk -F/ '!seen { print $1; seen = 1 }')"
[[ -n "$package_root" ]] || die "cannot determine component package root"
manifest_text="$(tar -xOf "$BASE_COMPONENT_PACKAGE" "$package_root/PACKAGE-MANIFEST.env")" \
|| die "base component package has no PACKAGE-MANIFEST.env"
read_manifest_value() {
printf '%s\n' "$manifest_text" | awk -F= -v key="$1" '$1 == key { print substr($0, index($0, "=") + 1); exit }'
}
base_source_commit="$(read_manifest_value source_commit)"
base_package_arch="$(read_manifest_value package_arch)"
base_runtime_image="$(read_manifest_value runtime_image)"
[[ "$base_source_commit" == 9f4f4cf069e9fa8f6f97ed0f1065adac38abe82f ]] \
|| die "base source commit mismatch: $base_source_commit"
[[ "$base_package_arch" == arm64 ]] || die "base package must be arm64: $base_package_arch"
timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
patch_name="$PREFIX-$SOURCE_COMMIT_SHORT"
if [[ "$SOURCE_DIRTY" == true ]]; then
patch_name="$patch_name-dirty"
fi
patch_name="$patch_name-$timestamp"
stage="$OUT_DIR/$patch_name"
tar_path="$OUT_DIR/$patch_name.tar.gz"
base_sha256="$(sha256sum "$BASE_COMPONENT_PACKAGE" | awk '{ print $1 }')"
rm -rf "$stage"
mkdir -p "$stage/payload/compose" "$stage/payload/scripts/soak" \
"$stage/payload/custom-fixtures" "$stage/payload/tools"
cp "$PATCH_SOURCE/apply_custom_tal_patch.sh" "$stage/"
cp "$PATCH_SOURCE/rollback_custom_tal_patch.sh" "$stage/"
cp "$PATCH_SOURCE/custom-tal.env.example" "$stage/"
cp "$REPO_ROOT/deploy/docker-installer/compose/docker-compose.yml" "$stage/payload/compose/"
cp "$REPO_ROOT/scripts/soak/run_soak.sh" "$stage/payload/scripts/soak/"
cp "$REPO_ROOT/deploy/docker-installer/custom-fixtures/README.md" "$stage/payload/custom-fixtures/"
cp "$PATCH_SOURCE/tools/generate_custom_fixture.py" "$stage/payload/tools/"
cp "$PATCH_SOURCE/tools/serve_https.py" "$stage/payload/tools/"
cp "$PATCH_SOURCE/tools/start_fixed_services.sh" "$stage/payload/tools/"
cp "$PATCH_SOURCE/tools/stop_fixed_services.sh" "$stage/payload/tools/"
chmod +x "$stage/apply_custom_tal_patch.sh" "$stage/rollback_custom_tal_patch.sh" \
"$stage/payload/scripts/soak/run_soak.sh" "$stage/payload/tools/"*.sh \
"$stage/payload/tools/"*.py
{
echo "patch_schema_version=1"
echo "patch_name=$patch_name"
echo "created_at_utc=$timestamp"
echo "source_commit=$SOURCE_COMMIT"
echo "source_commit_short=$SOURCE_COMMIT_SHORT"
echo "source_dirty=$SOURCE_DIRTY"
echo "base_component_archive=$(basename "$BASE_COMPONENT_PACKAGE")"
echo "base_component_archive_sha256=$base_sha256"
echo "base_component_root=$package_root"
echo "base_source_commit=$base_source_commit"
echo "base_package_arch=$base_package_arch"
echo "base_runtime_image=$base_runtime_image"
echo "overlay_runtime_image_unchanged=true"
echo "custom_mode=RIRS=custom,TAL_INPUT_MODE=custom-file-with-ta"
echo "rrdp_port=18443"
echo "rsync_port=1873"
echo "payload_root=payload"
while IFS= read -r relative_path; do
safe_name="$(printf '%s' "$relative_path" | tr '/.' '__')"
file_sha256="$(sha256sum "$stage/payload/$relative_path" | awk '{ print $1 }')"
echo "payload_"$safe_name"_sha256=$file_sha256"
done < <(cd "$stage/payload" && find . -type f -printf '%P\n' | sort)
} > "$stage/PATCH-MANIFEST.env"
cat > "$stage/PATCH-SUMMARY.txt" <<EOF
patch_name: $patch_name
base_component_archive: $(basename "$BASE_COMPONENT_PACKAGE")
base_component_archive_sha256: $base_sha256
base_source_commit: $base_source_commit
source_commit: $SOURCE_COMMIT
source_dirty: $SOURCE_DIRTY
runtime_image: $base_runtime_image
runtime_image_changed: false
custom_fixture_generator: payload/tools/generate_custom_fixture.py
fixed_services: payload/tools/start_fixed_services.sh
EOF
tar -C "$OUT_DIR" -czf "$tar_path" "$patch_name"
patch_sha256="$(sha256sum "$tar_path" | awk '{ print $1 }')"
{
echo "patch=$tar_path"
echo "patch_dir=$stage"
echo "patch_sha256=$patch_sha256"
echo "manifest=$stage/PATCH-MANIFEST.env"
echo "base_component_archive_sha256=$base_sha256"
echo "source_commit=$SOURCE_COMMIT"
echo "source_dirty=$SOURCE_DIRTY"
} > "$OUT_DIR/$patch_name.summary.env"
echo "patch built: $tar_path"

465
scripts/docker/build_docker_installer_package.sh
View File

@ -1,465 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
TARGET_ARCH="${TARGET_ARCH:-}"
IMAGE_TAG=""
IMAGE_TAR=""
METRICS_IMAGE=""
METRICS_IMAGE_TAR=""
PROMETHEUS_IMAGE="${PROMETHEUS_IMAGE:-prom/prometheus:v2.55.1}"
PROMETHEUS_IMAGE_TAR="${PROMETHEUS_IMAGE_TAR:-}"
GRAFANA_IMAGE="${GRAFANA_IMAGE:-grafana/grafana:11.3.1}"
GRAFANA_IMAGE_TAR="${GRAFANA_IMAGE_TAR:-}"
OUT_DIR="${OUT_DIR:-}"
PACKAGE_PREFIX="${PACKAGE_PREFIX:-}"
TEMPLATE_DIR="${TEMPLATE_DIR:-$REPO_ROOT/deploy/docker-installer}"
ALLOW_DIRTY=0
usage() {
cat <<'USAGE'
Usage:
scripts/docker/build_docker_installer_package.sh --arch amd64|arm64 [options]
Options:
--arch <arch> Target architecture: amd64|arm64
--prometheus-image <tag>
Prometheus image tag to record and package.
--prometheus-image-tar <path>
Existing Prometheus docker save tar/tar.gz to include.
--grafana-image <tag>
Grafana image tag to record and package.
--grafana-image-tar <path>
Existing Grafana docker save tar/tar.gz to include.
--out-dir <path> Output directory.
--prefix <name> Package directory/tar prefix.
--template-dir <path>
Package template directory.
--allow-dirty Allow a development build from a dirty worktree and mark
image tags, package name and manifest as dirty.
-h, --help Show help.
Every package rebuilds runtime and metrics images from the current HEAD. Their
tags are ours-rp-runtime-<arch>:<git8> and ours-rp-metrics-<arch>:<git8>.
External runtime/metrics image tags and tar archives are intentionally rejected.
USAGE
}
normalize_arch() {
case "$1" in
amd64|x86_64)
printf 'amd64\n'
;;
arm64|aarch64)
printf 'arm64\n'
;;
*)
return 1
;;
esac
}
platform_for_arch() {
printf 'linux/%s\n' "$1"
}
safe_tag_name() {
printf '%s' "$1" | tr '/:' '--'
}
runtime_image_tag() {
local arch="$1"
local revision="$2"
printf 'ours-rp-runtime-%s:%s\n' "$arch" "$revision"
}
metrics_image_tag() {
local arch="$1"
local revision="$2"
printf 'ours-rp-metrics-%s:%s\n' "$arch" "$revision"
}
default_package_prefix() {
printf 'ours-rp-installer-%s\n' "$1"
}
default_host_data_dir() {
printf '/var/lib/ours-rp-%s-installer\n' "$1"
}
default_compose_project() {
printf 'ours-rp-%s-installer\n' "$1"
}
default_metrics_instance() {
printf '%s-installer\n' "$1"
}
replace_or_append_env() {
local env_path="$1"
local key="$2"
local value="$3"
local tmp_env
tmp_env="${env_path}.tmp"
awk -v key="$key" -v value="$value" '
BEGIN { done=0 }
$0 ~ "^" key "=" { print key "=" value; done=1; next }
{ print }
END { if (!done) print key "=" value }
' "$env_path" > "$tmp_env"
mv "$tmp_env" "$env_path"
}
replace_text_in_file() {
local file_path="$1"
local old_text="$2"
local new_text="$3"
OLD_TEXT="$old_text" NEW_TEXT="$new_text" perl -0pi -e 's/\Q$ENV{OLD_TEXT}\E/$ENV{NEW_TEXT}/g' "$file_path"
}
ensure_image_for_platform() {
local image="$1"
local expected_platform="$2"
local role="$3"
local actual_platform
if ! docker image inspect "$image" >/dev/null 2>&1; then
echo "pulling $role image for $expected_platform: $image" >&2
docker pull --platform "$expected_platform" "$image" >&2
fi
actual_platform="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image" 2>/dev/null || echo unknown)"
if [[ "$actual_platform" != "$expected_platform" ]]; then
echo "re-pulling $role image for $expected_platform: $image (current=$actual_platform)" >&2
docker pull --platform "$expected_platform" "$image" >&2
actual_platform="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image" 2>/dev/null || echo unknown)"
fi
[[ "$actual_platform" == "$expected_platform" ]] || {
cat >&2 <<EOF
wrong platform for $role image: $image
expected: $expected_platform
actual: $actual_platform
EOF
exit 2
}
}
save_image_if_needed() {
local image="$1"
local existing_tar="$2"
local out_dir="$3"
local role="$4"
local expected_platform="$5"
if [[ -n "$existing_tar" ]]; then
[[ -f "$existing_tar" ]] || {
echo "missing $role image tar: $existing_tar" >&2
exit 2
}
printf '%s\n' "$existing_tar"
return 0
fi
ensure_image_for_platform "$image" "$expected_platform" "$role"
local tar_path="$out_dir/$(safe_tag_name "$image")-${TARGET_ARCH}.tar.gz"
echo "saving $role image to $tar_path" >&2
docker save "$image" | gzip -c > "$tar_path"
printf '%s\n' "$tar_path"
}
image_label() {
local image="$1"
local key="$2"
docker image inspect --format "{{ index .Config.Labels \"$key\" }}" "$image"
}
while [[ $# -gt 0 ]]; do
case "$1" in
--arch)
TARGET_ARCH="$(normalize_arch "$2")" || {
echo "unsupported target architecture: $2" >&2
exit 2
}
shift 2
;;
--image|--image-tar|--metrics-image|--metrics-image-tar)
echo "$1 is no longer supported: installer packages always rebuild runtime and metrics images from current HEAD" >&2
exit 2
;;
--prometheus-image)
PROMETHEUS_IMAGE="$2"
shift 2
;;
--prometheus-image-tar)
PROMETHEUS_IMAGE_TAR="$2"
shift 2
;;
--grafana-image)
GRAFANA_IMAGE="$2"
shift 2
;;
--grafana-image-tar)
GRAFANA_IMAGE_TAR="$2"
shift 2
;;
--out-dir)
OUT_DIR="$2"
shift 2
;;
--prefix)
PACKAGE_PREFIX="$2"
shift 2
;;
--template-dir)
TEMPLATE_DIR="$2"
shift 2
;;
--allow-dirty)
ALLOW_DIRTY=1
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo "unknown option: $1" >&2
usage >&2
exit 2
;;
esac
done
[[ -n "$TARGET_ARCH" ]] || {
echo "--arch is required" >&2
usage >&2
exit 2
}
[[ -d "$TEMPLATE_DIR" ]] || {
echo "missing template dir: $TEMPLATE_DIR" >&2
exit 2
}
SOURCE_COMMIT_FULL="$(git -C "$REPO_ROOT" rev-parse HEAD)"
SOURCE_COMMIT_SHORT="$(git -C "$REPO_ROOT" rev-parse --short=8 HEAD)"
SOURCE_DIRTY="false"
if [[ -n "$(git -C "$REPO_ROOT" status --short)" ]]; then
SOURCE_DIRTY="true"
fi
if [[ "$SOURCE_DIRTY" == "true" && "$ALLOW_DIRTY" != "1" ]]; then
cat >&2 <<EOF
refusing to build a release installer from a dirty worktree.
source_commit: $SOURCE_COMMIT_FULL
Use --allow-dirty only for development verification; the package and image tags
will be explicitly marked dirty.
EOF
exit 2
fi
TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)"
REVISION_TOKEN="$SOURCE_COMMIT_SHORT"
if [[ "$SOURCE_DIRTY" == "true" ]]; then
REVISION_TOKEN="${REVISION_TOKEN}-dirty"
fi
IMAGE_TAG="$(runtime_image_tag "$TARGET_ARCH" "$REVISION_TOKEN")"
METRICS_IMAGE="$(metrics_image_tag "$TARGET_ARCH" "$REVISION_TOKEN")"
if [[ -z "$OUT_DIR" ]]; then
OUT_DIR="$REPO_ROOT/target/${TARGET_ARCH}-installer"
fi
if [[ -z "$PACKAGE_PREFIX" ]]; then
PACKAGE_PREFIX="$(default_package_prefix "$TARGET_ARCH")"
fi
TARGET_PLATFORM="$(platform_for_arch "$TARGET_ARCH")"
HOST_DATA_DIR_DEFAULT="$(default_host_data_dir "$TARGET_ARCH")"
COMPOSE_PROJECT_DEFAULT="$(default_compose_project "$TARGET_ARCH")"
METRICS_INSTANCE_DEFAULT="$(default_metrics_instance "$TARGET_ARCH")"
IMAGE_OUT_DIR="$REPO_ROOT/target/${TARGET_ARCH}-docker"
mkdir -p "$OUT_DIR"
echo "rebuilding runtime image: $IMAGE_TAG"
"$SCRIPT_DIR/build_docker_runtime_image.sh" \
--arch "$TARGET_ARCH" \
--image "$IMAGE_TAG" \
--out-dir "$IMAGE_OUT_DIR"
echo "rebuilding metrics image: $METRICS_IMAGE"
"$SCRIPT_DIR/build_docker_metrics_image.sh" \
--arch "$TARGET_ARCH" \
--image "$METRICS_IMAGE" \
--out-dir "$IMAGE_OUT_DIR"
IMAGE_TAR="$IMAGE_OUT_DIR/$(safe_tag_name "$IMAGE_TAG").tar.gz"
METRICS_IMAGE_TAR="$IMAGE_OUT_DIR/$(safe_tag_name "$METRICS_IMAGE").tar.gz"
[[ -f "$IMAGE_TAR" ]] || { echo "runtime image build did not produce $IMAGE_TAR" >&2; exit 2; }
[[ -f "$METRICS_IMAGE_TAR" ]] || { echo "metrics image build did not produce $METRICS_IMAGE_TAR" >&2; exit 2; }
runtime_image_revision="$(image_label "$IMAGE_TAG" "org.opencontainers.image.revision")"
metrics_image_revision="$(image_label "$METRICS_IMAGE" "org.opencontainers.image.revision")"
runtime_image_dirty="$(image_label "$IMAGE_TAG" "org.opencontainers.image.source-dirty")"
metrics_image_dirty="$(image_label "$METRICS_IMAGE" "org.opencontainers.image.source-dirty")"
[[ "$runtime_image_revision" == "$SOURCE_COMMIT_FULL" ]] || { echo "runtime image revision mismatch: $runtime_image_revision != $SOURCE_COMMIT_FULL" >&2; exit 2; }
[[ "$metrics_image_revision" == "$SOURCE_COMMIT_FULL" ]] || { echo "metrics image revision mismatch: $metrics_image_revision != $SOURCE_COMMIT_FULL" >&2; exit 2; }
[[ "$runtime_image_dirty" == "$SOURCE_DIRTY" ]] || { echo "runtime image dirty flag mismatch" >&2; exit 2; }
[[ "$metrics_image_dirty" == "$SOURCE_DIRTY" ]] || { echo "metrics image dirty flag mismatch" >&2; exit 2; }
package_name="${PACKAGE_PREFIX}-${REVISION_TOKEN}-${TIMESTAMP}"
stage="$OUT_DIR/$package_name"
tar_path="$OUT_DIR/$package_name.tar.gz"
rm -rf "$stage"
rsync -a --delete "$TEMPLATE_DIR"/ "$stage"/
mkdir -p "$stage/images"
cp "$IMAGE_TAR" "$stage/images/"
cp "$METRICS_IMAGE_TAR" "$stage/images/"
monitor_image_stage="$OUT_DIR/.monitor-images-$TARGET_ARCH-$TIMESTAMP"
rm -rf "$monitor_image_stage"
mkdir -p "$monitor_image_stage"
prometheus_tar="$(save_image_if_needed "$PROMETHEUS_IMAGE" "$PROMETHEUS_IMAGE_TAR" "$monitor_image_stage" "prometheus" "$TARGET_PLATFORM")"
grafana_tar="$(save_image_if_needed "$GRAFANA_IMAGE" "$GRAFANA_IMAGE_TAR" "$monitor_image_stage" "grafana" "$TARGET_PLATFORM")"
cp "$prometheus_tar" "$stage/images/"
cp "$grafana_tar" "$stage/images/"
if [[ -f "$stage/.env.example" ]]; then
replace_or_append_env "$stage/.env.example" "PACKAGE_ARCH" "$TARGET_ARCH"
replace_or_append_env "$stage/.env.example" "PACKAGE_PLATFORM" "$TARGET_PLATFORM"
replace_or_append_env "$stage/.env.example" "COMPOSE_PROJECT_NAME" "$COMPOSE_PROJECT_DEFAULT"
replace_or_append_env "$stage/.env.example" "RPKI_IMAGE" "$IMAGE_TAG"
replace_or_append_env "$stage/.env.example" "RPKI_PLATFORM" "$TARGET_PLATFORM"
replace_or_append_env "$stage/.env.example" "METRICS_IMAGE" "$METRICS_IMAGE"
replace_or_append_env "$stage/.env.example" "METRICS_PLATFORM" "$TARGET_PLATFORM"
replace_or_append_env "$stage/.env.example" "HOST_DATA_DIR" "$HOST_DATA_DIR_DEFAULT"
replace_or_append_env "$stage/.env.example" "RTR_REPORT_DIR" "$HOST_DATA_DIR_DEFAULT/empty-rtr-report"
replace_or_append_env "$stage/.env.example" "RTR_REPORT_CONTAINER_DIR" "/var/lib/ours-rp/rtr-report"
replace_or_append_env "$stage/.env.example" "ALLOW_CROSS_ARCH" "0"
replace_or_append_env "$stage/.env.example" "METRICS_INSTANCE" "$METRICS_INSTANCE_DEFAULT"
replace_or_append_env "$stage/.env.example" "MONITOR_PLATFORM" "$TARGET_PLATFORM"
replace_or_append_env "$stage/.env.example" "PROMETHEUS_IMAGE" "$PROMETHEUS_IMAGE"
replace_or_append_env "$stage/.env.example" "GRAFANA_IMAGE" "$GRAFANA_IMAGE"
fi
replace_text_in_file "$stage/docs/README.zh-CN.md" "__PACKAGE_ARCH__" "$TARGET_ARCH"
replace_text_in_file "$stage/docs/README.en.md" "__PACKAGE_ARCH__" "$TARGET_ARCH"
replace_text_in_file "$stage/docs/operations.zh-CN.md" "__PACKAGE_ARCH__" "$TARGET_ARCH"
replace_text_in_file "$stage/docs/operations.en.md" "__PACKAGE_ARCH__" "$TARGET_ARCH"
replace_text_in_file "$stage/docs/troubleshooting.zh-CN.md" "__PACKAGE_ARCH__" "$TARGET_ARCH"
replace_text_in_file "$stage/docs/troubleshooting.en.md" "__PACKAGE_ARCH__" "$TARGET_ARCH"
replace_text_in_file "$stage/docs/README.zh-CN.md" "__PACKAGE_PLATFORM__" "$TARGET_PLATFORM"
replace_text_in_file "$stage/docs/README.en.md" "__PACKAGE_PLATFORM__" "$TARGET_PLATFORM"
replace_text_in_file "$stage/docs/troubleshooting.zh-CN.md" "__PACKAGE_PLATFORM__" "$TARGET_PLATFORM"
replace_text_in_file "$stage/docs/troubleshooting.en.md" "__PACKAGE_PLATFORM__" "$TARGET_PLATFORM"
replace_text_in_file "$stage/docs/README.zh-CN.md" "__HOST_DATA_DIR__" "$HOST_DATA_DIR_DEFAULT"
replace_text_in_file "$stage/docs/README.en.md" "__HOST_DATA_DIR__" "$HOST_DATA_DIR_DEFAULT"
replace_text_in_file "$stage/docs/operations.zh-CN.md" "__HOST_DATA_DIR__" "$HOST_DATA_DIR_DEFAULT"
replace_text_in_file "$stage/docs/operations.en.md" "__HOST_DATA_DIR__" "$HOST_DATA_DIR_DEFAULT"
replace_text_in_file "$stage/docs/troubleshooting.zh-CN.md" "__HOST_DATA_DIR__" "$HOST_DATA_DIR_DEFAULT"
replace_text_in_file "$stage/docs/troubleshooting.en.md" "__HOST_DATA_DIR__" "$HOST_DATA_DIR_DEFAULT"
replace_text_in_file "$stage/docs/README.zh-CN.md" "__RUNTIME_IMAGE__" "$IMAGE_TAG"
replace_text_in_file "$stage/docs/README.en.md" "__RUNTIME_IMAGE__" "$IMAGE_TAG"
replace_text_in_file "$stage/docs/README.zh-CN.md" "__METRICS_IMAGE__" "$METRICS_IMAGE"
replace_text_in_file "$stage/docs/README.en.md" "__METRICS_IMAGE__" "$METRICS_IMAGE"
runtime_tar_sha256="$(sha256sum "$IMAGE_TAR" | awk '{print $1}')"
metrics_tar_sha256="$(sha256sum "$METRICS_IMAGE_TAR" | awk '{print $1}')"
prometheus_tar_sha256="$(sha256sum "$prometheus_tar" | awk '{print $1}')"
grafana_tar_sha256="$(sha256sum "$grafana_tar" | awk '{print $1}')"
cat > "$stage/PACKAGE-MANIFEST.env" <<EOF
package_schema_version=2
package_name=$package_name
created_at_utc=$TIMESTAMP
build_timestamp_utc=$TIMESTAMP
source_commit=$SOURCE_COMMIT_FULL
source_commit_short=$SOURCE_COMMIT_SHORT
source_dirty=$SOURCE_DIRTY
git_commit=$SOURCE_COMMIT_FULL
git_status_count=$(git -C "$REPO_ROOT" status --short 2>/dev/null | wc -l | tr -d ' ')
PACKAGE_ARCH=$TARGET_ARCH
PACKAGE_PLATFORM=$TARGET_PLATFORM
package_arch=$TARGET_ARCH
package_platform=$TARGET_PLATFORM
cross_arch_default=deny
cross_arch_enable_var=ALLOW_CROSS_ARCH
RPKI_IMAGE=$IMAGE_TAG
RPKI_PLATFORM=$TARGET_PLATFORM
image_tag=$IMAGE_TAG
image_tar=$(basename "$IMAGE_TAR")
image_tar_size_bytes=$(wc -c < "$IMAGE_TAR")
image_tar_sha256=$runtime_tar_sha256
runtime_image=$IMAGE_TAG
runtime_image_revision=$runtime_image_revision
runtime_image_dirty=$runtime_image_dirty
rpki_image=$IMAGE_TAG
rpki_platform=$TARGET_PLATFORM
METRICS_IMAGE=$METRICS_IMAGE
METRICS_PLATFORM=$TARGET_PLATFORM
metrics_image=$METRICS_IMAGE
metrics_image_tar=$(basename "$METRICS_IMAGE_TAR")
metrics_image_tar_size_bytes=$(wc -c < "$METRICS_IMAGE_TAR")
metrics_image_tar_sha256=$metrics_tar_sha256
metrics_image_revision=$metrics_image_revision
metrics_image_dirty=$metrics_image_dirty
PROMETHEUS_IMAGE=$PROMETHEUS_IMAGE
prometheus_image=$PROMETHEUS_IMAGE
prometheus_image_tar=$(basename "$prometheus_tar")
prometheus_image_tar_size_bytes=$(wc -c < "$prometheus_tar")
prometheus_image_tar_sha256=$prometheus_tar_sha256
GRAFANA_IMAGE=$GRAFANA_IMAGE
grafana_image=$GRAFANA_IMAGE
grafana_image_tar=$(basename "$grafana_tar")
grafana_image_tar_size_bytes=$(wc -c < "$grafana_tar")
grafana_image_tar_sha256=$grafana_tar_sha256
target_platform=$TARGET_PLATFORM
MONITOR_PLATFORM=$TARGET_PLATFORM
rpki_image_tar=$(basename "$IMAGE_TAR")
monitor_platform=$TARGET_PLATFORM
metrics_platform=$TARGET_PLATFORM
default_host_data_dir=$HOST_DATA_DIR_DEFAULT
default_compose_project_name=$COMPOSE_PROJECT_DEFAULT
default_metrics_instance=$METRICS_INSTANCE_DEFAULT
EOF
cat > "$stage/PACKAGE-SUMMARY.txt" <<EOF
package_name: $package_name
source_commit: $SOURCE_COMMIT_FULL
source_commit_short: $SOURCE_COMMIT_SHORT
source_dirty: $SOURCE_DIRTY
build_timestamp_utc: $TIMESTAMP
package_arch: $TARGET_ARCH
package_platform: $TARGET_PLATFORM
runtime_image: $IMAGE_TAG
runtime_image_revision: $runtime_image_revision
metrics_image: $METRICS_IMAGE
metrics_image_revision: $metrics_image_revision
prometheus_image: $PROMETHEUS_IMAGE
grafana_image: $GRAFANA_IMAGE
default_host_data_dir: $HOST_DATA_DIR_DEFAULT
default_compose_project_name: $COMPOSE_PROJECT_DEFAULT
cross_arch_default: deny
cross_arch_enable_var: ALLOW_CROSS_ARCH
EOF
chmod +x "$stage/scripts"/*.sh
if find "$stage" -maxdepth 1 -type f -name '*.sh' -print -quit | grep -q .; then
echo "error: operational scripts must be installed under scripts/ only" >&2
exit 1
fi
tar -C "$OUT_DIR" -czf "$tar_path" "$package_name"
package_sha256="$(sha256sum "$tar_path" | awk '{print $1}')"
rm -rf "$monitor_image_stage"
{
echo "package=$tar_path"
echo "package_dir=$stage"
echo "package_size_bytes=$(wc -c < "$tar_path")"
echo "package_sha256=$package_sha256"
echo "source_commit=$SOURCE_COMMIT_FULL"
echo "source_commit_short=$SOURCE_COMMIT_SHORT"
echo "source_dirty=$SOURCE_DIRTY"
echo "build_timestamp_utc=$TIMESTAMP"
echo "package_arch=$TARGET_ARCH"
echo "package_platform=$TARGET_PLATFORM"
echo "manifest=$stage/PACKAGE-MANIFEST.env"
echo "summary=$stage/PACKAGE-SUMMARY.txt"
} > "$OUT_DIR/$package_name.summary.env"
echo "package built: $tar_path"

82
scripts/docker/build_docker_metrics_image.sh
View File

@ -1,82 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
normalize_arch() {
case "$1" in
amd64|x86_64)
printf 'amd64\n'
;;
arm64|aarch64)
printf 'arm64\n'
;;
*)
return 1
;;
esac
}
TARGET_ARCH="${TARGET_ARCH:-}"
IMAGE_TAG="${IMAGE_TAG:-}"
DOCKERFILE="${DOCKERFILE:-$REPO_ROOT/docker/ours-rp-metrics.Dockerfile}"
args=()
image_explicit=0
dockerfile_explicit=0
while [[ $# -gt 0 ]]; do
case "$1" in
--arch)
TARGET_ARCH="$(normalize_arch "$2")" || {
echo "unsupported target architecture: $2" >&2
exit 2
}
args+=("$1" "$TARGET_ARCH")
shift 2
;;
--image)
IMAGE_TAG="$2"
image_explicit=1
args+=("$1" "$2")
shift 2
;;
--dockerfile)
DOCKERFILE="$2"
dockerfile_explicit=1
args+=("$1" "$2")
shift 2
;;
*)
args+=("$1")
shift
;;
esac
done
if [[ -z "$TARGET_ARCH" ]]; then
cat >&2 <<'EOF'
missing target architecture.
Usage:
scripts/docker/build_docker_metrics_image.sh --arch amd64|arm64 [options]
EOF
exit 2
fi
if [[ -z "$IMAGE_TAG" ]]; then
SOURCE_COMMIT_SHORT="$(git -C "$REPO_ROOT" rev-parse --short=8 HEAD 2>/dev/null || echo unknown)"
if [[ -n "$(git -C "$REPO_ROOT" status --short 2>/dev/null)" ]]; then
SOURCE_COMMIT_SHORT="${SOURCE_COMMIT_SHORT}-dirty"
fi
IMAGE_TAG="ours-rp-metrics-${TARGET_ARCH}:${SOURCE_COMMIT_SHORT}"
fi
if [[ "$dockerfile_explicit" != "1" ]]; then
args=(--dockerfile "$DOCKERFILE" "${args[@]}")
fi
if [[ "$image_explicit" != "1" ]]; then
args=(--image "$IMAGE_TAG" "${args[@]}")
fi
exec "$SCRIPT_DIR/build_docker_runtime_image.sh" "${args[@]}"

295
scripts/docker/build_docker_runtime_image.sh
View File

@ -1,295 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
TARGET_ARCH="${TARGET_ARCH:-}"
IMAGE_TAG="${IMAGE_TAG:-}"
BUILDER_IMAGE="${BUILDER_IMAGE:-rust:1-bookworm}"
RUNTIME_IMAGE="${RUNTIME_IMAGE:-debian:bookworm-slim}"
OUT_DIR="${OUT_DIR:-}"
DOCKERFILE="${DOCKERFILE:-$REPO_ROOT/docker/ours-rp-runtime.Dockerfile}"
BUILDER_NAME="${BUILDER_NAME:-default}"
INSTALL_BINFMT="${INSTALL_BINFMT:-1}"
SAVE_IMAGE="${SAVE_IMAGE:-1}"
LOAD_IMAGE="${LOAD_IMAGE:-1}"
usage() {
cat <<'USAGE'
Usage:
scripts/docker/build_docker_runtime_image.sh --arch amd64|arm64 [options]
Options:
--arch <arch> Target architecture: amd64|arm64
--image <tag> Docker image tag (default: ours-rp-runtime-<arch>:<git8>)
--out-dir <path> Directory for docker save tar.gz (default: target/<arch>-docker)
--dockerfile <path> Dockerfile path
--builder <name> buildx builder name
--builder-image <tag>
Builder base image (default: rust:1-bookworm)
--runtime-image <tag>
Runtime base image (default: debian:bookworm-slim)
--no-binfmt Do not install binfmt/qemu for cross-architecture builds
--no-save Build image but do not docker save it
--no-load Use buildx output tar instead of --load
-h, --help Show this help
USAGE
}
normalize_arch() {
case "$1" in
amd64|x86_64)
printf 'amd64\n'
;;
arm64|aarch64)
printf 'arm64\n'
;;
*)
return 1
;;
esac
}
safe_tag_name() {
printf '%s' "$1" | tr '/:' '--'
}
local_base_image_tag() {
local role="$1"
local arch="$2"
local source_image="$3"
printf 'ours-rp-base-%s-%s:%s\n' "$role" "$arch" "$(safe_tag_name "$source_image")"
}
platform_for_arch() {
printf 'linux/%s\n' "$1"
}
default_image_tag() {
local arch="$1"
local commit_short="$2"
printf 'ours-rp-runtime-%s:%s\n' "$arch" "$commit_short"
}
require_command() {
command -v "$1" >/dev/null 2>&1 || {
echo "missing required command: $1" >&2
exit 2
}
}
host_arch() {
local raw_arch
raw_arch="$(uname -m)"
normalize_arch "$raw_arch" || {
echo "unsupported host architecture: $raw_arch" >&2
exit 2
}
}
while [[ $# -gt 0 ]]; do
case "$1" in
--arch)
TARGET_ARCH="$(normalize_arch "$2")" || {
echo "unsupported target architecture: $2" >&2
exit 2
}
shift 2
;;
--image)
IMAGE_TAG="$2"
shift 2
;;
--out-dir)
OUT_DIR="$2"
shift 2
;;
--dockerfile)
DOCKERFILE="$2"
shift 2
;;
--builder)
BUILDER_NAME="$2"
shift 2
;;
--builder-image)
BUILDER_IMAGE="$2"
shift 2
;;
--runtime-image)
RUNTIME_IMAGE="$2"
shift 2
;;
--no-binfmt)
INSTALL_BINFMT=0
shift
;;
--no-save)
SAVE_IMAGE=0
shift
;;
--no-load)
LOAD_IMAGE=0
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo "unknown option: $1" >&2
usage >&2
exit 2
;;
esac
done
[[ -n "$TARGET_ARCH" ]] || {
echo "--arch is required" >&2
usage >&2
exit 2
}
SOURCE_COMMIT_FULL="$(git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null || echo unknown)"
SOURCE_COMMIT_SHORT="$(git -C "$REPO_ROOT" rev-parse --short=8 HEAD 2>/dev/null || echo unknown)"
SOURCE_DIRTY="false"
if [[ -n "$(git -C "$REPO_ROOT" status --short 2>/dev/null)" ]]; then
SOURCE_DIRTY="true"
fi
BUILD_TIMESTAMP_UTC="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
if [[ -z "$IMAGE_TAG" ]]; then
default_revision="$SOURCE_COMMIT_SHORT"
if [[ "$SOURCE_DIRTY" == "true" ]]; then
default_revision="${default_revision}-dirty"
fi
IMAGE_TAG="$(default_image_tag "$TARGET_ARCH" "$default_revision")"
fi
if [[ -z "$OUT_DIR" ]]; then
OUT_DIR="$REPO_ROOT/target/${TARGET_ARCH}-docker"
fi
require_command docker
mkdir -p "$OUT_DIR"
target_platform="$(platform_for_arch "$TARGET_ARCH")"
local_host_arch="$(host_arch)"
builder_platform="$(platform_for_arch "$local_host_arch")"
builder_source_image="$BUILDER_IMAGE"
runtime_source_image="$RUNTIME_IMAGE"
builder_local_image="$(local_base_image_tag builder "$local_host_arch" "$builder_source_image")"
runtime_local_image="$(local_base_image_tag runtime "$TARGET_ARCH" "$runtime_source_image")"
if [[ "$INSTALL_BINFMT" == "1" && "$local_host_arch" != "$TARGET_ARCH" ]]; then
echo "installing binfmt/qemu for $TARGET_ARCH"
docker run --rm --privileged tonistiigi/binfmt --install "$TARGET_ARCH"
fi
# Pulling through the Docker daemon uses its configured proxy. Buildx then consumes
# these architecture-pinned local aliases without issuing separate registry metadata
# requests for the Dockerfile FROM instructions.
echo "pulling builder base image: $builder_source_image ($builder_platform)"
docker pull --platform "$builder_platform" "$builder_source_image"
docker tag "$builder_source_image" "$builder_local_image"
echo "pulling runtime base image: $runtime_source_image ($target_platform)"
docker pull --platform "$target_platform" "$runtime_source_image"
docker tag "$runtime_source_image" "$runtime_local_image"
BUILDER_IMAGE="$builder_local_image"
RUNTIME_IMAGE="$runtime_local_image"
if [[ "$BUILDER_NAME" != "default" ]] && ! docker buildx inspect "$BUILDER_NAME" >/dev/null 2>&1; then
docker buildx create --name "$BUILDER_NAME" --driver docker-container --use >/dev/null
else
docker buildx use "$BUILDER_NAME" >/dev/null
fi
docker buildx inspect --bootstrap >/dev/null
metadata_path="$OUT_DIR/$(safe_tag_name "$IMAGE_TAG").build-metadata.json"
tar_path="$OUT_DIR/$(safe_tag_name "$IMAGE_TAG").tar.gz"
build_log="$OUT_DIR/$(safe_tag_name "$IMAGE_TAG").build.log"
echo "building $target_platform image: $IMAGE_TAG"
echo "repo: $REPO_ROOT"
echo "dockerfile: $DOCKERFILE"
echo "builder_source_image: $builder_source_image"
echo "runtime_source_image: $runtime_source_image"
echo "builder_local_image: $BUILDER_IMAGE"
echo "runtime_local_image: $RUNTIME_IMAGE"
echo "source_commit: $SOURCE_COMMIT_FULL"
echo "source_dirty: $SOURCE_DIRTY"
echo "build_timestamp_utc: $BUILD_TIMESTAMP_UTC"
start_epoch="$(date +%s)"
if [[ "$LOAD_IMAGE" == "1" ]]; then
docker buildx build \
--platform "$target_platform" \
--builder "$BUILDER_NAME" \
--pull=false \
--load \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--build-arg "BUILDER_IMAGE=$BUILDER_IMAGE" \
--build-arg "RUNTIME_IMAGE=$RUNTIME_IMAGE" \
--build-arg "SOURCE_COMMIT=$SOURCE_COMMIT_FULL" \
--build-arg "SOURCE_DIRTY=$SOURCE_DIRTY" \
--build-arg "BUILD_TIMESTAMP_UTC=$BUILD_TIMESTAMP_UTC" \
--metadata-file "$metadata_path" \
-t "$IMAGE_TAG" \
-f "$DOCKERFILE" \
"$REPO_ROOT" 2>&1 | tee "$build_log"
else
raw_tar_path="$OUT_DIR/$(safe_tag_name "$IMAGE_TAG").tar"
docker buildx build \
--platform "$target_platform" \
--builder "$BUILDER_NAME" \
--pull=false \
--output "type=docker,dest=$raw_tar_path" \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--build-arg "BUILDER_IMAGE=$BUILDER_IMAGE" \
--build-arg "RUNTIME_IMAGE=$RUNTIME_IMAGE" \
--build-arg "SOURCE_COMMIT=$SOURCE_COMMIT_FULL" \
--build-arg "SOURCE_DIRTY=$SOURCE_DIRTY" \
--build-arg "BUILD_TIMESTAMP_UTC=$BUILD_TIMESTAMP_UTC" \
--metadata-file "$metadata_path" \
-t "$IMAGE_TAG" \
-f "$DOCKERFILE" \
"$REPO_ROOT" 2>&1 | tee "$build_log"
gzip -f "$raw_tar_path"
tar_path="${raw_tar_path}.gz"
fi
elapsed_secs=$(( $(date +%s) - start_epoch ))
if [[ "$SAVE_IMAGE" == "1" && "$LOAD_IMAGE" == "1" ]]; then
echo "saving image to $tar_path"
docker save "$IMAGE_TAG" | gzip -c > "$tar_path"
fi
tar_sha256=""
if [[ -f "$tar_path" ]]; then
tar_sha256="$(sha256sum "$tar_path" | awk '{print $1}')"
fi
{
echo "image=$IMAGE_TAG"
echo "platform=$target_platform"
echo "arch=$TARGET_ARCH"
echo "builder_image=$builder_source_image"
echo "runtime_image=$runtime_source_image"
echo "builder_local_image=$BUILDER_IMAGE"
echo "runtime_local_image=$RUNTIME_IMAGE"
echo "elapsed_secs=$elapsed_secs"
echo "metadata=$metadata_path"
echo "tar=$tar_path"
echo "tar_size_bytes=$(wc -c < "$tar_path" 2>/dev/null || echo 0)"
echo "tar_sha256=$tar_sha256"
echo "source_commit=$SOURCE_COMMIT_FULL"
echo "source_commit_short=$SOURCE_COMMIT_SHORT"
echo "source_dirty=$SOURCE_DIRTY"
echo "build_timestamp_utc=$BUILD_TIMESTAMP_UTC"
echo "git_commit=$SOURCE_COMMIT_FULL"
echo "git_status_count=$(git -C "$REPO_ROOT" status --short 2>/dev/null | wc -l | tr -d ' ')"
echo "built_at_utc=$BUILD_TIMESTAMP_UTC"
} > "$OUT_DIR/$(safe_tag_name "$IMAGE_TAG").build-summary.env"
echo "build complete: elapsed=${elapsed_secs}s tar=$tar_path"

Some files were not shown because too many files have changed in this diff Show More