20260723 新增 rpki-explorer 单容器部署资产并修复 parsed 投影渲染（#128）：deploy/rpki-explorer 全套独立 compose 部署资产（对接 231 在跑 soak、retain 10 runs、SSH 转发访问）；ProjectionView 按真实 API 结构解包 projection.object、证书走 resourceCertificate+extensions、CRL 读顶层字段并新增 IP/AS 资源格式化；e2e fixtures 对齐真实嵌套结构并新增证书/CRL parsed 覆盖

replay 对比，五个RIR 输出vrp与routinator一致

.dockerignore

@@ -0,0 +1,18 @@
+target/
+.git/
+.gitignore
+docker/
+perf.*
+**/* copy.excalidraw
+ui/rpki-explorer/node_modules/
+ui/rpki-explorer/dist/
+ui/rpki-explorer/playwright-report/
+ui/rpki-explorer/test-results/
+ui/rpki-explorer/.vite/
+deploy/arm64-compose/.env
+target/
+*.profraw
+*.profdata
+*.tar
+*.tar.gz
+*.zip

.gitignore

@@ -1,2 +1,9 @@
 target/
 Cargo.lock
+perf.*
+specs/* copy.excalidraw
+ui/rpki-explorer/node_modules/
+ui/rpki-explorer/dist/
+ui/rpki-explorer/playwright-report/
+ui/rpki-explorer/test-results/
+ui/rpki-explorer/.vite/

Cargo.toml

@@ -3,10 +3,35 @@ name = "rpki"
 version = "0.1.0"
 edition = "2024"
 
+[features]
+default = ["full"]
+# Full build used by the main RP implementation (includes RocksDB-backed storage).
+full = ["dep:rocksdb"]
+profile = ["dep:pprof", "dep:flate2"]
+
 [dependencies]
-der-parser = "10.0.0"
+asn1-rs = "0.7.1"
+der-parser = { version = "10.0.0", features = ["serialize"] }
 hex = "0.4.3"
+base64 = "0.22.1"
+sha2 = "0.10.8"
 thiserror = "2.0.18"
 time = "0.3.45"
+ring = "0.17.14"
 x509-parser = { version = "0.18.0", features = ["verify"] }
 url = "2.5.8"
+serde = { version = "1.0.218", features = ["derive"] }
+serde_json = { version = "1.0.140", features = ["raw_value"] }
+toml = "0.8.20"
+rocksdb = { version = "0.22.0", optional = true, default-features = false, features = ["lz4"] }
+serde_cbor = "0.11.2"
+memmap2 = "0.9.10"
+roxmltree = "0.20.0"
+quick-xml = "0.37.2"
+uuid = { version = "1.7.0", features = ["v4"] }
+reqwest = { version = "0.12.12", default-features = false, features = ["blocking", "rustls-tls", "gzip", "brotli", "deflate"] }
+pprof = { version = "0.14.1", optional = true, features = ["flamegraph", "prost-codec"] }
+flate2 = { version = "1.0.35", optional = true }
+tempfile = "3.16.0"
+
+[dev-dependencies]

README.md

@@ -9,3 +9,55 @@ cargo test
 cargo test -- --nocapture
 ```
 
+# 覆盖率（cargo-llvm-cov）
+
+安装工具：
+
+```
+rustup component add llvm-tools-preview
+cargo install cargo-llvm-cov --locked
+```
+
+统计行覆盖率并要求 >=90%：
+
+```
+./scripts/coverage.sh
+# 或
+cargo llvm-cov --fail-under-lines 90
+```
+
+默认会复用现有插桩产物，不会先 clean。需要强制全量重编译时：
+
+```
+COVERAGE_FORCE_CLEAN=1 ./scripts/coverage.sh
+```
+
+说明：
+- 默认行为适合本地重复确认覆盖率，避免每次都重编译整套插桩目标；
+- 默认还会设置 `RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS=1`，跳过会拉起 shell replay pipeline 的重型集成测试，避免 coverage 期间额外触发 `target/release` 构建；
+- 默认还会设置 `RPKI_SKIP_HEAVY_BLACKBOX_TESTS=1`，跳过更慢的 blackbox CLI / CIR record 脚本测试，进一步降低日常 coverage 成本；
+- 默认还会设置 `RPKI_SKIP_HEAVY_CRYPTO_TESTS=1`，跳过需要大量 OpenSSL 生成证书/CRL 的重型密码学测试，进一步压缩日常 coverage 时长；
+- 如需把这批脚本回放测试也纳入 coverage，可显式关闭该开关：
+
+```
+RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS=0 ./scripts/coverage.sh
+```
+
+如需连同第二批 blackbox 测试一起跑：
+
+```
+RPKI_SKIP_HEAVY_BLACKBOX_TESTS=0 ./scripts/coverage.sh
+```
+
+如需连同重型 OpenSSL 证书路径测试一起跑：
+
+```
+RPKI_SKIP_HEAVY_CRYPTO_TESTS=0 ./scripts/coverage.sh
+```
+
+- replay 脚本现在也支持通过环境变量注入现成二进制，避免找不到二进制时自动 `cargo build --release`：
+  - `RPKI_BIN`
+  - `CIR_MATERIALIZE_BIN`
+  - `CIR_EXTRACT_INPUTS_BIN`
+  - `CCR_TO_COMPARE_VIEWS_BIN`
+- `COVERAGE_FORCE_CLEAN=1` 适合需要完全从零重建插桩目标时使用。

benchmark/ours_manifest_bench/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "ours-manifest-bench"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+rpki = { path = "../..", default-features = false }
+

benchmark/ours_manifest_bench/src/main.rs

@@ -0,0 +1,145 @@
+use rpki::data_model::manifest::ManifestObject;
+use std::hint::black_box;
+use std::path::PathBuf;
+use std::time::Instant;
+
+#[derive(Debug, Clone)]
+struct Config {
+    sample: Option<String>,
+    manifest_path: Option<PathBuf>,
+    iterations: u64,
+    warmup_iterations: u64,
+    repeats: u32,
+}
+
+fn usage_and_exit() -> ! {
+    eprintln!(
+        "Usage:\n  ours-manifest-bench (--sample <name> | --manifest <path>) [--iterations N] [--warmup-iterations N] [--repeats N]\n\nExamples:\n  cargo run --release -- --sample small-01 --iterations 20000 --warmup-iterations 2000 --repeats 3\n  cargo run --release -- --manifest ../../tests/benchmark/selected_der/small-01.mft"
+    );
+    std::process::exit(2);
+}
+
+fn parse_args() -> Config {
+    let mut sample: Option<String> = None;
+    let mut manifest_path: Option<PathBuf> = None;
+    let mut iterations: u64 = 20_000;
+    let mut warmup_iterations: u64 = 2_000;
+    let mut repeats: u32 = 3;
+
+    let mut args = std::env::args().skip(1);
+    while let Some(arg) = args.next() {
+        match arg.as_str() {
+            "--sample" => sample = Some(args.next().unwrap_or_else(|| usage_and_exit())),
+            "--manifest" => {
+                manifest_path = Some(PathBuf::from(args.next().unwrap_or_else(|| usage_and_exit())))
+            }
+            "--iterations" => {
+                iterations = args
+                    .next()
+                    .unwrap_or_else(|| usage_and_exit())
+                    .parse()
+                    .unwrap_or_else(|_| usage_and_exit())
+            }
+            "--warmup-iterations" => {
+                warmup_iterations = args
+                    .next()
+                    .unwrap_or_else(|| usage_and_exit())
+                    .parse()
+                    .unwrap_or_else(|_| usage_and_exit())
+            }
+            "--repeats" => {
+                repeats = args
+                    .next()
+                    .unwrap_or_else(|| usage_and_exit())
+                    .parse()
+                    .unwrap_or_else(|_| usage_and_exit())
+            }
+            "-h" | "--help" => usage_and_exit(),
+            _ => usage_and_exit(),
+        }
+    }
+
+    if sample.is_none() && manifest_path.is_none() {
+        usage_and_exit();
+    }
+    if sample.is_some() && manifest_path.is_some() {
+        usage_and_exit();
+    }
+
+    Config {
+        sample,
+        manifest_path,
+        iterations,
+        warmup_iterations,
+        repeats,
+    }
+}
+
+fn derive_manifest_path(sample: &str) -> PathBuf {
+    // Assumes current working directory is `rpki/benchmark/ours_manifest_bench`.
+    PathBuf::from(format!("../../tests/benchmark/selected_der/{sample}.mft"))
+}
+
+fn main() {
+    let cfg = parse_args();
+    let manifest_path = cfg
+        .manifest_path
+        .clone()
+        .unwrap_or_else(|| derive_manifest_path(cfg.sample.as_deref().unwrap()));
+
+    let bytes = std::fs::read(&manifest_path).unwrap_or_else(|e| {
+        eprintln!("read manifest fixture failed: {e}; path={}", manifest_path.display());
+        std::process::exit(1);
+    });
+
+    let decoded_once = ManifestObject::decode_der(&bytes).unwrap_or_else(|e| {
+        eprintln!("decode failed: {e}; path={}", manifest_path.display());
+        std::process::exit(1);
+    });
+    let file_count = decoded_once.manifest.file_count();
+
+    let mut round_ns_per_op: Vec<f64> = Vec::with_capacity(cfg.repeats as usize);
+    let mut round_ops_per_s: Vec<f64> = Vec::with_capacity(cfg.repeats as usize);
+
+    for _round in 0..cfg.repeats {
+        for _ in 0..cfg.warmup_iterations {
+            let obj = ManifestObject::decode_der(black_box(&bytes)).expect("warmup decode");
+            black_box(obj);
+        }
+
+        let start = Instant::now();
+        for _ in 0..cfg.iterations {
+            let obj = ManifestObject::decode_der(black_box(&bytes)).expect("timed decode");
+            black_box(obj);
+        }
+        let elapsed = start.elapsed();
+
+        let ns_per_op = (elapsed.as_secs_f64() * 1e9) / (cfg.iterations as f64);
+        let ops_per_s = (cfg.iterations as f64) / elapsed.as_secs_f64();
+        round_ns_per_op.push(ns_per_op);
+        round_ops_per_s.push(ops_per_s);
+    }
+
+    let avg_ns_per_op = round_ns_per_op.iter().sum::<f64>() / (round_ns_per_op.len() as f64);
+    let avg_ops_per_s = round_ops_per_s.iter().sum::<f64>() / (round_ops_per_s.len() as f64);
+
+    let sample_name = cfg.sample.clone().unwrap_or_else(|| {
+        manifest_path
+            .file_name()
+            .map(|s| s.to_string_lossy().to_string())
+            .unwrap_or_else(|| manifest_path.display().to_string())
+    });
+    let sample_name = sample_name
+        .strip_suffix(".mft")
+        .unwrap_or(&sample_name)
+        .to_string();
+
+    println!("fixture: {}", manifest_path.display());
+    println!();
+    println!("| sample | avg ns/op | ops/s | file count |");
+    println!("|---|---:|---:|---:|");
+    println!(
+        "| {} | {:.2} | {:.2} | {} |",
+        sample_name, avg_ns_per_op, avg_ops_per_s, file_count
+    );
+}

benchmark/routinator_object_bench/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "routinator-object-bench"
+version = "0.1.0"
+edition = "2024"
+publish = false
+
+[dependencies]
+rpki = { version = "=0.19.1", features = ["repository"] }

benchmark/routinator_object_bench/src/main.rs

@@ -0,0 +1,552 @@
+use rpki::repository::cert::Cert;
+use rpki::repository::crl::Crl;
+use rpki::repository::manifest::Manifest;
+use rpki::repository::roa::Roa;
+use rpki::repository::aspa::Aspa;
+use rpki::repository::resources::{AsResources, IpResources};
+
+use std::hint::black_box;
+use std::path::{Path, PathBuf};
+use std::time::Instant;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+enum ObjType {
+    Cer,
+    Crl,
+    Manifest,
+    Roa,
+    Aspa,
+}
+
+impl ObjType {
+    fn parse(s: &str) -> Result<Self, String> {
+        match s {
+            "cer" => Ok(Self::Cer),
+            "crl" => Ok(Self::Crl),
+            "manifest" => Ok(Self::Manifest),
+            "roa" => Ok(Self::Roa),
+            "aspa" => Ok(Self::Aspa),
+            _ => Err("type must be one of: cer, crl, manifest, roa, aspa".into()),
+        }
+    }
+
+    fn as_str(self) -> &'static str {
+        match self {
+            ObjType::Cer => "cer",
+            ObjType::Crl => "crl",
+            ObjType::Manifest => "manifest",
+            ObjType::Roa => "roa",
+            ObjType::Aspa => "aspa",
+        }
+    }
+
+    fn ext(self) -> &'static str {
+        match self {
+            ObjType::Cer => "cer",
+            ObjType::Crl => "crl",
+            ObjType::Manifest => "mft",
+            ObjType::Roa => "roa",
+            ObjType::Aspa => "asa",
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+struct Sample {
+    obj_type: ObjType,
+    name: String,
+    path: PathBuf,
+}
+
+#[derive(Clone, Debug)]
+struct Config {
+    dir: PathBuf,
+    type_filter: Option<ObjType>,
+    sample_filter: Option<String>,
+    fixed_iters: Option<u64>,
+    warmup_iters: u64,
+    rounds: u64,
+    min_round_ms: u64,
+    max_adaptive_iters: u64,
+    strict: bool,
+    cert_inspect: bool,
+    out_csv: Option<PathBuf>,
+    out_md: Option<PathBuf>,
+}
+
+fn usage_and_exit(err: Option<&str>) -> ! {
+    if let Some(err) = err {
+        eprintln!("error: {err}");
+        eprintln!();
+    }
+    eprintln!(
+        "Usage:\n\
+  cargo run --release --manifest-path rpki/benchmark/routinator_object_bench/Cargo.toml -- [OPTIONS]\n\
+\n\
+Options:\n\
+  --dir <PATH>            Fixtures root dir (default: ../../tests/benchmark/selected_der_v2)\n\
+  --type <cer|crl|manifest|roa|aspa>    Filter by type\n\
+  --sample <NAME>         Filter by sample name (e.g. p50)\n\
+  --iters <N>             Fixed iterations per round (optional; otherwise adaptive)\n\
+  --warmup-iters <N>      Warmup iterations (default: 50)\n\
+  --rounds <N>            Rounds (default: 5)\n\
+  --min-round-ms <MS>     Adaptive: minimum round time (default: 200)\n\
+  --max-iters <N>         Adaptive: maximum iters (default: 1_000_000)\n\
+  --strict <true|false>   Strict DER where applicable (default: true)\n\
+  --cert-inspect          Also run Cert::inspect_ca/inspect_ee where applicable (default: false)\n\
+  --out-csv <PATH>        Write CSV output\n\
+  --out-md <PATH>         Write Markdown output\n\
+"
+    );
+    std::process::exit(2);
+}
+
+fn parse_bool(s: &str, name: &str) -> bool {
+    match s {
+        "1" | "true" | "TRUE" | "yes" | "YES" => true,
+        "0" | "false" | "FALSE" | "no" | "NO" => false,
+        _ => usage_and_exit(Some(&format!("{name} must be true/false"))),
+    }
+}
+
+fn parse_u64(s: &str, name: &str) -> u64 {
+    s.parse::<u64>()
+        .unwrap_or_else(|_| usage_and_exit(Some(&format!("{name} must be an integer"))))
+}
+
+fn default_samples_dir() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/benchmark/selected_der_v2")
+}
+
+fn parse_args() -> Config {
+    let mut dir: PathBuf = default_samples_dir();
+    let mut type_filter: Option<ObjType> = None;
+    let mut sample_filter: Option<String> = None;
+    let mut fixed_iters: Option<u64> = None;
+    let mut warmup_iters: u64 = 50;
+    let mut rounds: u64 = 5;
+    let mut min_round_ms: u64 = 200;
+    let mut max_adaptive_iters: u64 = 1_000_000;
+    let mut strict: bool = true;
+    let mut cert_inspect: bool = false;
+    let mut out_csv: Option<PathBuf> = None;
+    let mut out_md: Option<PathBuf> = None;
+
+    let mut args = std::env::args().skip(1);
+    while let Some(arg) = args.next() {
+        match arg.as_str() {
+            "--dir" => dir = PathBuf::from(args.next().unwrap_or_else(|| usage_and_exit(None))),
+            "--type" => {
+                type_filter = Some(ObjType::parse(
+                    &args.next().unwrap_or_else(|| usage_and_exit(None)),
+                )
+                .unwrap_or_else(|e| usage_and_exit(Some(&e))))
+            }
+            "--sample" => {
+                sample_filter = Some(args.next().unwrap_or_else(|| usage_and_exit(None)))
+            }
+            "--iters" => {
+                fixed_iters = Some(parse_u64(
+                    &args.next().unwrap_or_else(|| usage_and_exit(None)),
+                    "--iters",
+                ))
+            }
+            "--warmup-iters" => {
+                warmup_iters = parse_u64(
+                    &args.next().unwrap_or_else(|| usage_and_exit(None)),
+                    "--warmup-iters",
+                )
+            }
+            "--rounds" => {
+                rounds = parse_u64(&args.next().unwrap_or_else(|| usage_and_exit(None)), "--rounds")
+            }
+            "--min-round-ms" => {
+                min_round_ms = parse_u64(
+                    &args.next().unwrap_or_else(|| usage_and_exit(None)),
+                    "--min-round-ms",
+                )
+            }
+            "--max-iters" => {
+                max_adaptive_iters = parse_u64(
+                    &args.next().unwrap_or_else(|| usage_and_exit(None)),
+                    "--max-iters",
+                )
+            }
+            "--strict" => {
+                strict = parse_bool(
+                    &args.next().unwrap_or_else(|| usage_and_exit(None)),
+                    "--strict",
+                )
+            }
+            "--cert-inspect" => cert_inspect = true,
+            "--out-csv" => out_csv = Some(PathBuf::from(args.next().unwrap_or_else(|| usage_and_exit(None)))),
+            "--out-md" => out_md = Some(PathBuf::from(args.next().unwrap_or_else(|| usage_and_exit(None)))),
+            "-h" | "--help" => usage_and_exit(None),
+            _ => usage_and_exit(Some(&format!("unknown argument: {arg}"))),
+        }
+    }
+
+    if warmup_iters == 0 {
+        usage_and_exit(Some("--warmup-iters must be > 0"));
+    }
+    if rounds == 0 {
+        usage_and_exit(Some("--rounds must be > 0"));
+    }
+    if min_round_ms == 0 {
+        usage_and_exit(Some("--min-round-ms must be > 0"));
+    }
+    if max_adaptive_iters == 0 {
+        usage_and_exit(Some("--max-iters must be > 0"));
+    }
+    if let Some(n) = fixed_iters {
+        if n == 0 {
+            usage_and_exit(Some("--iters must be > 0"));
+        }
+    }
+
+    Config {
+        dir,
+        type_filter,
+        sample_filter,
+        fixed_iters,
+        warmup_iters,
+        rounds,
+        min_round_ms,
+        max_adaptive_iters,
+        strict,
+        cert_inspect,
+        out_csv,
+        out_md,
+    }
+}
+
+fn read_samples(root: &Path) -> Vec<Sample> {
+    let mut out = Vec::new();
+    for obj_type in [
+        ObjType::Cer,
+        ObjType::Crl,
+        ObjType::Manifest,
+        ObjType::Roa,
+        ObjType::Aspa,
+    ] {
+        let dir = root.join(obj_type.as_str());
+        let rd = match std::fs::read_dir(&dir) {
+            Ok(rd) => rd,
+            Err(_) => continue,
+        };
+        for ent in rd.flatten() {
+            let path = ent.path();
+            if path.extension().and_then(|s| s.to_str()) != Some(obj_type.ext()) {
+                continue;
+            }
+            let name = path
+                .file_stem()
+                .and_then(|s| s.to_str())
+                .unwrap_or("unknown")
+                .to_string();
+            out.push(Sample { obj_type, name, path });
+        }
+    }
+    out.sort_by(|a, b| a.obj_type.cmp(&b.obj_type).then_with(|| a.name.cmp(&b.name)));
+    out
+}
+
+fn choose_iters_adaptive<F: FnMut()>(mut op: F, min_round_ms: u64, max_iters: u64) -> u64 {
+    let min_secs = (min_round_ms as f64) / 1e3;
+    let mut iters: u64 = 1;
+    loop {
+        let start = Instant::now();
+        for _ in 0..iters {
+            op();
+        }
+        let elapsed = start.elapsed().as_secs_f64();
+        if elapsed >= min_secs {
+            return iters;
+        }
+        if iters >= max_iters {
+            return iters;
+        }
+        iters = (iters.saturating_mul(2)).min(max_iters);
+    }
+}
+
+fn count_ip(res: &IpResources) -> u64 {
+    if res.is_inherited() {
+        return 1;
+    }
+    let Ok(blocks) = res.to_blocks() else {
+        return 0;
+    };
+    blocks.iter().count() as u64
+}
+
+fn count_as(res: &AsResources) -> u64 {
+    if res.is_inherited() {
+        return 1;
+    }
+    let Ok(blocks) = res.to_blocks() else {
+        return 0;
+    };
+    blocks.iter().count() as u64
+}
+
+fn complexity(obj_type: ObjType, bytes: &[u8], strict: bool, cert_inspect: bool) -> u64 {
+    match obj_type {
+        ObjType::Cer => {
+            let cert = Cert::decode(bytes).expect("decode cert");
+            if cert_inspect {
+                if cert.is_ca() {
+                    cert.inspect_ca(strict).expect("inspect ca");
+                } else {
+                    cert.inspect_ee(strict).expect("inspect ee");
+                }
+            }
+            count_ip(cert.v4_resources())
+                .saturating_add(count_ip(cert.v6_resources()))
+                .saturating_add(count_as(cert.as_resources()))
+        }
+        ObjType::Crl => {
+            let crl = Crl::decode(bytes).expect("decode crl");
+            crl.revoked_certs().iter().count() as u64
+        }
+        ObjType::Manifest => {
+            let mft = Manifest::decode(bytes, strict).expect("decode manifest");
+            if cert_inspect {
+                mft.cert().inspect_ee(strict).expect("inspect ee");
+            }
+            mft.content().len() as u64
+        }
+        ObjType::Roa => {
+            let roa = Roa::decode(bytes, strict).expect("decode roa");
+            if cert_inspect {
+                roa.cert().inspect_ee(strict).expect("inspect ee");
+            }
+            roa.content().iter().count() as u64
+        }
+        ObjType::Aspa => {
+            let asa = Aspa::decode(bytes, strict).expect("decode aspa");
+            if cert_inspect {
+                asa.cert().inspect_ee(strict).expect("inspect ee");
+            }
+            asa.content().provider_as_set().len() as u64
+        }
+    }
+}
+
+fn decode_profile(obj_type: ObjType, bytes: &[u8], strict: bool, cert_inspect: bool) {
+    match obj_type {
+        ObjType::Cer => {
+            let cert = Cert::decode(black_box(bytes)).expect("decode cert");
+            if cert_inspect {
+                if cert.is_ca() {
+                    cert.inspect_ca(strict).expect("inspect ca");
+                } else {
+                    cert.inspect_ee(strict).expect("inspect ee");
+                }
+            }
+            black_box(cert);
+        }
+        ObjType::Crl => {
+            let crl = Crl::decode(black_box(bytes)).expect("decode crl");
+            black_box(crl);
+        }
+        ObjType::Manifest => {
+            let mft = Manifest::decode(black_box(bytes), strict).expect("decode manifest");
+            if cert_inspect {
+                mft.cert().inspect_ee(strict).expect("inspect ee");
+            }
+            black_box(mft);
+        }
+        ObjType::Roa => {
+            let roa = Roa::decode(black_box(bytes), strict).expect("decode roa");
+            if cert_inspect {
+                roa.cert().inspect_ee(strict).expect("inspect ee");
+            }
+            black_box(roa);
+        }
+        ObjType::Aspa => {
+            let asa = Aspa::decode(black_box(bytes), strict).expect("decode aspa");
+            if cert_inspect {
+                asa.cert().inspect_ee(strict).expect("inspect ee");
+            }
+            black_box(asa);
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+struct ResultRow {
+    obj_type: String,
+    sample: String,
+    size_bytes: usize,
+    complexity: u64,
+    avg_ns_per_op: f64,
+    ops_per_sec: f64,
+}
+
+fn render_markdown(title: &str, rows: &[ResultRow]) -> String {
+    let mut out = String::new();
+    out.push_str(&format!("# {title}\n\n"));
+    out.push_str("| type | sample | size_bytes | complexity | avg ns/op | ops/s |\n");
+    out.push_str("|---|---|---:|---:|---:|---:|\n");
+    for r in rows {
+        out.push_str(&format!(
+            "| {} | {} | {} | {} | {:.2} | {:.2} |\n",
+            r.obj_type, r.sample, r.size_bytes, r.complexity, r.avg_ns_per_op, r.ops_per_sec
+        ));
+    }
+    out
+}
+
+fn render_csv(rows: &[ResultRow]) -> String {
+    let mut out = String::new();
+    out.push_str("type,sample,size_bytes,complexity,avg_ns_per_op,ops_per_sec\n");
+    for r in rows {
+        let sample = r.sample.replace('"', "\"\"");
+        out.push_str(&format!(
+            "{},{},{},{},{:.6},{:.6}\n",
+            r.obj_type,
+            format!("\"{}\"", sample),
+            r.size_bytes,
+            r.complexity,
+            r.avg_ns_per_op,
+            r.ops_per_sec
+        ));
+    }
+    out
+}
+
+fn create_parent_dirs(path: &Path) {
+    if let Some(parent) = path.parent() {
+        std::fs::create_dir_all(parent).unwrap_or_else(|e| {
+            panic!("create_dir_all {}: {e}", parent.display());
+        });
+    }
+}
+
+fn write_text_file(path: &Path, content: &str) {
+    create_parent_dirs(path);
+    std::fs::write(path, content).unwrap_or_else(|e| panic!("write {}: {e}", path.display()));
+}
+
+fn main() {
+    let cfg = parse_args();
+    let mut samples = read_samples(&cfg.dir);
+    if samples.is_empty() {
+        usage_and_exit(Some(&format!(
+            "no samples found under: {}",
+            cfg.dir.display()
+        )));
+    }
+
+    if let Some(t) = cfg.type_filter {
+        samples.retain(|s| s.obj_type == t);
+        if samples.is_empty() {
+            usage_and_exit(Some(&format!("no sample matched --type {}", t.as_str())));
+        }
+    }
+    if let Some(filter) = cfg.sample_filter.as_deref() {
+        samples.retain(|s| s.name == filter);
+        if samples.is_empty() {
+            usage_and_exit(Some(&format!("no sample matched --sample {filter}")));
+        }
+    }
+
+    println!("# Routinator baseline (rpki crate) decode benchmark (selected_der_v2)");
+    println!();
+    println!("- dir: {}", cfg.dir.display());
+    println!("- strict: {}", cfg.strict);
+    println!("- cert_inspect: {}", cfg.cert_inspect);
+    if let Some(t) = cfg.type_filter {
+        println!("- type: {}", t.as_str());
+    }
+    if let Some(s) = cfg.sample_filter.as_deref() {
+        println!("- sample: {}", s);
+    }
+    if let Some(n) = cfg.fixed_iters {
+        println!("- iters: {} (fixed)", n);
+    } else {
+        println!(
+            "- warmup: {} iters, rounds: {}, min_round: {}ms (adaptive iters, max {})",
+            cfg.warmup_iters, cfg.rounds, cfg.min_round_ms, cfg.max_adaptive_iters
+        );
+    }
+    if let Some(p) = cfg.out_csv.as_ref() {
+        println!("- out_csv: {}", p.display());
+    }
+    if let Some(p) = cfg.out_md.as_ref() {
+        println!("- out_md: {}", p.display());
+    }
+    println!();
+
+    println!("| type | sample | size_bytes | complexity | avg ns/op | ops/s |");
+    println!("|---|---|---:|---:|---:|---:|");
+
+    let mut rows: Vec<ResultRow> = Vec::with_capacity(samples.len());
+    for sample in &samples {
+        let bytes = std::fs::read(&sample.path)
+            .unwrap_or_else(|e| panic!("read {}: {e}", sample.path.display()));
+        let size_bytes = bytes.len();
+        let complexity = complexity(sample.obj_type, bytes.as_slice(), cfg.strict, cfg.cert_inspect);
+
+        for _ in 0..cfg.warmup_iters {
+            decode_profile(sample.obj_type, bytes.as_slice(), cfg.strict, cfg.cert_inspect);
+        }
+
+        let mut per_round_ns_per_op = Vec::with_capacity(cfg.rounds as usize);
+        for _round in 0..cfg.rounds {
+            let iters = if let Some(n) = cfg.fixed_iters {
+                n
+            } else {
+                choose_iters_adaptive(
+                    || decode_profile(sample.obj_type, bytes.as_slice(), cfg.strict, cfg.cert_inspect),
+                    cfg.min_round_ms,
+                    cfg.max_adaptive_iters,
+                )
+            };
+            let start = Instant::now();
+            for _ in 0..iters {
+                decode_profile(sample.obj_type, bytes.as_slice(), cfg.strict, cfg.cert_inspect);
+            }
+            let elapsed = start.elapsed();
+            let total_ns = elapsed.as_secs_f64() * 1e9;
+            per_round_ns_per_op.push(total_ns / (iters as f64));
+        }
+
+        let avg_ns = per_round_ns_per_op.iter().sum::<f64>() / (per_round_ns_per_op.len() as f64);
+        let ops_per_sec = 1e9_f64 / avg_ns;
+
+        println!(
+            "| {} | {} | {} | {} | {:.2} | {:.2} |",
+            sample.obj_type.as_str(),
+            sample.name,
+            size_bytes,
+            complexity,
+            avg_ns,
+            ops_per_sec
+        );
+
+        rows.push(ResultRow {
+            obj_type: sample.obj_type.as_str().to_string(),
+            sample: sample.name.clone(),
+            size_bytes,
+            complexity,
+            avg_ns_per_op: avg_ns,
+            ops_per_sec,
+        });
+    }
+
+    if let Some(path) = cfg.out_md.as_ref() {
+        let md = render_markdown(
+            "Routinator baseline (rpki crate) decode+inspect (selected_der_v2)",
+            &rows,
+        );
+        write_text_file(path, &md);
+        eprintln!("Wrote {}", path.display());
+    }
+    if let Some(path) = cfg.out_csv.as_ref() {
+        let csv = render_csv(&rows);
+        write_text_file(path, &csv);
+        eprintln!("Wrote {}", path.display());
+    }
+}

deploy/docker-installer/.env.example

@@ -0,0 +1,116 @@
+# ours RP Docker installer configuration
+# `build_docker_installer_package.sh` rewrites package-specific placeholders
+# when producing amd64/arm64 release packages.
+# 中文说明见 docs/README.zh-CN.md。English guide: docs/README.en.md
+# Every assignment is a required non-empty configuration key with a package
+# default. During upgrade: .env.example < reused old .env < command environment.
+
+# Package metadata and architecture guardrails.
+PACKAGE_ARCH=__PACKAGE_ARCH__
+PACKAGE_PLATFORM=__PACKAGE_PLATFORM__
+ALLOW_CROSS_ARCH=0
+
+# Compose project name.
+COMPOSE_PROJECT_NAME=ours-rp-__PACKAGE_ARCH__-installer
+
+# Runtime image rebuilt from the current source commit while packaging.
+RPKI_IMAGE=__RUNTIME_IMAGE__
+RPKI_PLATFORM=__PACKAGE_PLATFORM__
+
+# Metrics image rebuilt from the current source commit while packaging.
+METRICS_IMAGE=__METRICS_IMAGE__
+METRICS_PLATFORM=__PACKAGE_PLATFORM__
+
+# Restart policy for the soak container. Production default keeps the daemon alive.
+# For finite acceptance tests such as MAX_RUNS=3, set SOAK_RESTART_POLICY=no to avoid an extra restarted run.
+SOAK_RESTART_POLICY=unless-stopped
+
+# Host-side persistent data directory. All state/runs/logs/monitoring data are bind-mounted here.
+HOST_DATA_DIR=__HOST_DATA_DIR__
+
+# RIR list. Options: afrinic,apnic,arin,lacnic,ripe
+RIRS=afrinic,apnic,arin,lacnic,ripe
+
+# Negative MAX_RUNS means keep running forever. Default production interval is 10 minutes.
+MAX_RUNS=-1
+INTERVAL_SECS=600
+RETAIN_RUNS=100
+
+# TAL/TA input mode:
+# file-with-ta: use packaged fixture TAL + TA only.
+# file-live-ta: use packaged fixture TAL; snapshot waits for live TA refresh, delta refreshes TA in background.
+# url: pass TAL URL to child process.
+TAL_INPUT_MODE=file-live-ta
+LIVE_TA_REFRESH_BEFORE_SNAPSHOT=1
+LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS=15
+LIVE_TA_REFRESH_MAX_TIME_SECS=120
+
+# Sync and runtime behavior.
+RSYNC_SCOPE=module-root
+DISABLE_COMPETING_RPS=0
+RUN_ROOT=/var/lib/ours-rp
+DB_DIR=/var/lib/ours-rp/state/db
+RSYNC_MIRROR_ROOT=/var/lib/ours-rp/state/rsync-mirror
+# 每轮完成并复制出正式 run 产物后清理 tmp/daemon-run_*，避免长跑占满磁盘。
+CLEAN_TMP_AFTER_RUN=1
+OUTPUT_COMPACT_REPORT=1
+ALLOW_RSYNC_MIRROR_REUSE=1
+FAILURE_SNAPSHOT_RESET=1
+# Periodic snapshot reset of active state DB.
+# 0: keep existing behavior.
+# 1: after one successful snapshot, allow at most N successful delta runs;
+#    the next run is forced to snapshot and active state/db is rebuilt from empty.
+# Lifecycle run state is persisted independently at:
+#   ${HOST_DATA_DIR}/state/run-lifecycle-state.json
+# It is not affected by run retention or state/db reset.
+PERIODIC_SNAPSHOT_RESET=0
+PERIODIC_SNAPSHOT_MAX_DELTAS=100
+DB_STATS_EXACT_EVERY=0
+
+# Resource certificate validation mode passed to the rpki child process.
+# Options: validation-update-03 | rfc6487
+RESOURCE_VALIDATION_MODE=validation-update-03
+
+# Validation and performance options aligned with current optimized soak defaults.
+ENABLE_CHILD_CERTIFICATE_VALIDATION_CACHE=1
+RPKI_ANALYZE=1
+RPKI_EXTRA_ARGS="--enable-transport-request-prefetch --enable-publication-point-validation-cache --enable-roa-validation-cache --parallel-max-repo-sync-workers-global 4 --parallel-phase2-object-workers 4 --memory-trim-after-validation"
+
+# Progress logs.
+RPKI_PROGRESS_LOG=1
+RPKI_PROGRESS_SLOW_SECS=20
+RPKI_PROGRESS_STAGE_FRESH_SLOW_MS=2000
+RPKI_PROGRESS_PP_CONTROL_SLOW_MS=200
+RPKI_PROGRESS_PP_CACHE_SLOW_MS=100
+RPKI_PROGRESS_CONTROL_LOOP_SLOW_MS=2000
+
+# Metrics sidecar.
+METRICS_INSTANCE=__PACKAGE_ARCH__-installer
+METRICS_PORT=9556
+METRICS_POLL_SECS=10
+# CCR rpki-client format check: the artifact-metrics container runs
+# `rpki-client -f result.ccr` once for every newly completed run and exports
+# ours_rp_ccr_format_check_* metrics. The binary is packaged inside the
+# metrics image; adjust only when deliberately testing another binary.
+CCR_CHECK_BIN=/opt/ours-rp/bin/rpki-client
+CCR_CHECK_TIMEOUT_SECS=120
+# Optional external RTR report directory produced by a separately deployed RTR service.
+# Default points to an installer-managed empty fallback directory. To enable real RTR
+# metrics, set this to the host-side report directory, for example:
+#   RTR_REPORT_DIR=/root/rpki/report
+RTR_REPORT_DIR=__HOST_DATA_DIR__/empty-rtr-report
+RTR_REPORT_CONTAINER_DIR=/var/lib/ours-rp/rtr-report
+
+# Prometheus / Grafana.
+# Monitor images are packaged as docker-save archives and loaded by install.sh.
+MONITOR_PLATFORM=__PACKAGE_PLATFORM__
+PROMETHEUS_IMAGE=prom/prometheus:v2.55.1
+GRAFANA_IMAGE=grafana/grafana:11.3.1
+PROMETHEUS_PORT=9090
+PROMETHEUS_RETENTION=7d
+GRAFANA_PORT=3000
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=admin
+
+# First snapshot waiting timeout used by start.sh.
+FIRST_RUN_WAIT_TIMEOUT_SECS=7200

deploy/docker-installer/compose/docker-compose.yml

@@ -0,0 +1,97 @@
+services:
+  ours-rp-soak:
+    image: ${RPKI_IMAGE:?RPKI_IMAGE is required}
+    platform: ${RPKI_PLATFORM:?RPKI_PLATFORM is required}
+    container_name: ${COMPOSE_PROJECT_NAME:-ours-rp-package-installer}-soak
+    env_file:
+      - ../.env
+    environment:
+      PACKAGE_ROOT: /opt/ours-rp
+      ENV_FILE: /opt/ours-rp/.env
+      RUN_ROOT: /var/lib/ours-rp
+      BIN_DIR: /opt/ours-rp/bin
+      FIXTURE_DIR: /opt/ours-rp/fixtures
+    volumes:
+      - ../.env:/opt/ours-rp/.env:ro
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/state:/var/lib/ours-rp/state
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/runs:/var/lib/ours-rp/runs
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/logs:/var/lib/ours-rp/logs
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/tmp:/var/lib/ours-rp/tmp
+    restart: ${SOAK_RESTART_POLICY:-unless-stopped}
+    profiles:
+      - core
+
+  artifact-metrics:
+    image: ${METRICS_IMAGE:?METRICS_IMAGE is required}
+    platform: ${METRICS_PLATFORM:?METRICS_PLATFORM is required}
+    container_name: ${COMPOSE_PROJECT_NAME:-ours-rp-package-installer}-artifact-metrics
+    env_file:
+      - ../.env
+    environment:
+      RPKI_METRICS_RTR_REPORT_DIR: ${RTR_REPORT_CONTAINER_DIR:-/var/lib/ours-rp/rtr-report}
+    command:
+      - /opt/ours-rp/bin/rpki_artifact_metrics
+      - --run-root
+      - /var/lib/ours-rp
+      - --listen
+      - 0.0.0.0:9556
+      - --poll-secs
+      - ${METRICS_POLL_SECS:-10}
+      - --instance
+      - ${METRICS_INSTANCE:-package-installer}
+      - --ccr-check-bin
+      - ${CCR_CHECK_BIN:-/opt/ours-rp/bin/rpki-client}
+      - --ccr-check-timeout-secs
+      - ${CCR_CHECK_TIMEOUT_SECS:-120}
+    ports:
+      - "${METRICS_PORT:-9556}:9556"
+    volumes:
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/state:/var/lib/ours-rp/state:ro
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/runs:/var/lib/ours-rp/runs:ro
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/logs:/var/lib/ours-rp/logs:ro
+      - ${RTR_REPORT_DIR:-/var/lib/ours-rp-package-installer/empty-rtr-report}:${RTR_REPORT_CONTAINER_DIR:-/var/lib/ours-rp/rtr-report}:ro
+    restart: unless-stopped
+    profiles:
+      - sidecar
+
+  prometheus:
+    image: ${PROMETHEUS_IMAGE:-prom/prometheus:v2.55.1}
+    platform: ${MONITOR_PLATFORM:?MONITOR_PLATFORM is required}
+    container_name: ${COMPOSE_PROJECT_NAME:-ours-rp-package-installer}-prometheus
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-7d}
+      - --web.enable-lifecycle
+    depends_on:
+      - artifact-metrics
+    user: "0:0"
+    ports:
+      - "${PROMETHEUS_PORT:-9090}:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/prometheus:/prometheus
+    restart: unless-stopped
+    profiles:
+      - monitor
+
+  grafana:
+    image: ${GRAFANA_IMAGE:-grafana/grafana:11.3.1}
+    platform: ${MONITOR_PLATFORM:?MONITOR_PLATFORM is required}
+    container_name: ${COMPOSE_PROJECT_NAME:-ours-rp-package-installer}-grafana
+    depends_on:
+      - prometheus
+    user: "0:0"
+    ports:
+      - "${GRAFANA_PORT:-3000}:3000"
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+    volumes:
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/grafana:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+    restart: unless-stopped
+    profiles:
+      - monitor

deploy/docker-installer/compose/grafana/dashboards/ours-rp-repo-sync.json

@@ -0,0 +1,761 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_publication_points",
+          "legendFormat": "publication points",
+          "refId": "A"
+        }
+      ],
+      "title": "Publication Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_ok\"}",
+          "legendFormat": "rrdp ok",
+          "refId": "A"
+        }
+      ],
+      "title": "RRDP OK Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "sum(ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}) or vector(0)",
+          "legendFormat": "fallback",
+          "refId": "A"
+        }
+      ],
+      "title": "Rsync Fallback Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache",
+          "refId": "A"
+        }
+      ],
+      "title": "Failed No Cache Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 4
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"repo_sync_total\"}",
+          "legendFormat": "repo sync total",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"rrdp_download_total\"}",
+          "legendFormat": "rrdp download",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"rsync_download_total\"}",
+          "legendFormat": "rsync download",
+          "refId": "C"
+        }
+      ],
+      "title": "Repo Sync Download Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 12,
+        "w": 12,
+        "h": 8
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count",
+          "legendFormat": "{{phase}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repo Sync Phase Counts",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 12,
+        "w": 12,
+        "h": 8
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}",
+          "legendFormat": "rrdp failed, rsync ok",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_failed\"}",
+          "legendFormat": "rrdp failed, rsync failed",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache",
+          "refId": "C"
+        },
+        {
+          "expr": "ours_rp_tree_instances{state=\"failed\"}",
+          "legendFormat": "tree failed",
+          "refId": "D"
+        }
+      ],
+      "title": "Repo Failure / Fallback Counts",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 20,
+        "w": 12,
+        "h": 8
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_ok\"}",
+          "legendFormat": "rsync fallback duration",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_failed\"}",
+          "legendFormat": "failed duration",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_repo_terminal_state_duration_seconds_total{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache duration",
+          "refId": "C"
+        }
+      ],
+      "title": "Repo Failure / Fallback Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 29,
+        "w": 12,
+        "h": 9
+      },
+      "id": 9,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_rrdp_rsync_failed_repository_duration_seconds",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "RRDP + Rsync Failed Repositories",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "terminal_state": true,
+              "rank": true,
+              "transport": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "phase": 2,
+              "uri": 3,
+              "Value": 4
+            },
+            "renameByName": {
+              "Value": "duration"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 29,
+        "w": 12,
+        "h": 9
+      },
+      "id": 11,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "topk(20, ours_rp_top_repository_sync_duration_seconds)",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Top 20 Repositories by Sync Duration",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "terminal_state": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "phase": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "rank": 2,
+              "transport": 3,
+              "uri": 4,
+              "Value": 5
+            },
+            "renameByName": {
+              "Value": "value"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 38,
+        "w": 24,
+        "h": 9
+      },
+      "id": 10,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "topk(20, ours_rp_top_publication_point_object_count)",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Top Publication Points by Objects",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "phase": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "rank": 2,
+              "terminal_state": 3,
+              "transport": 4,
+              "uri": 5,
+              "Value": 6
+            },
+            "renameByName": {}
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository sync success in the latest successful run; 1 means successful, 0 means failed or failed_no_cache.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bool"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 47
+      },
+      "id": 12,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_sync_success",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Sync Success by Repo",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository total sync duration aggregated from publication point repo_sync_duration_ms.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 55
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_sync_duration_seconds{stat=\"sum\"}",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Sync Duration by Repo",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository downloaded bytes attributed from report.json downloads events.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 63
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_download_bytes",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Download Bytes by Repo",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": [
+    "ours-rp",
+    "rpki",
+    "soak",
+    "repo-sync"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Ours RP Repo Sync",
+  "uid": "ours-rp-repo-sync",
+  "version": 3,
+  "weekStart": ""
+}

deploy/docker-installer/compose/grafana/dashboards/ours-rp-rtr-overview.json

@@ -0,0 +1,666 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "id": 1,
+      "title": "RTR Metrics Enabled",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 0,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(ours_rp_rtr_metrics_enabled)",
+          "legendFormat": "enabled",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 2,
+      "title": "Refresh Success",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 5,
+        "x": 4,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(ours_rp_rtr_source_refresh_status{status=\"success\"})",
+          "legendFormat": "success",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 3,
+      "title": "Consecutive Failures",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 5,
+        "x": 9,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(ours_rp_rtr_source_refresh_consecutive_failures)",
+          "legendFormat": "failures",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 4,
+      "title": "Last Success Age",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 5,
+        "x": 14,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "decimals": 0,
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(ours_rp_rtr_source_last_success_age_seconds)",
+          "legendFormat": "age",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 5,
+      "title": "RTR RSS",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 5,
+        "x": 19,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "decimals": 0,
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(ours_rp_rtr_process_rss_bytes)",
+          "legendFormat": "rss",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 6,
+      "title": "Data Quality Totals",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_rtr_data_quality_items{stage=~\"ccr_input|before_slurm|after_slurm\",type=\"total\"}",
+          "legendFormat": "{{stage}} total",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_rtr_data_quality_items{stage=\"after_slurm\",type=\"vrp\"}",
+          "legendFormat": "after_slurm vrp",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_rtr_data_quality_items{stage=\"after_slurm\",type=\"aspa\"}",
+          "legendFormat": "after_slurm aspa",
+          "refId": "C"
+        }
+      ]
+    },
+    {
+      "id": 7,
+      "title": "SLURM Filters / Assertions",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_rtr_slurm_filters",
+          "legendFormat": "filter {{type}}",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_rtr_slurm_assertions",
+          "legendFormat": "assert {{type}}",
+          "refId": "B"
+        }
+      ]
+    },
+    {
+      "id": 8,
+      "title": "Cache Ready / Delta Window",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 12
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_rtr_cache_ready",
+          "legendFormat": "ready",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_rtr_cache_delta_window_length",
+          "legendFormat": "v{{version}} length",
+          "refId": "B"
+        }
+      ]
+    },
+    {
+      "id": 9,
+      "title": "Cache Snapshot Items",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 12
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_rtr_cache_snapshot_items",
+          "legendFormat": "v{{version}} {{type}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 10,
+      "title": "Latest Delta Items",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 12
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_rtr_cache_delta_items",
+          "legendFormat": "v{{version}} {{direction}} {{type}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 11,
+      "title": "Connections",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 20
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_rtr_active_connections",
+          "legendFormat": "active",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_rtr_connections",
+          "legendFormat": "{{transport}}",
+          "refId": "B"
+        }
+      ]
+    },
+    {
+      "id": 12,
+      "title": "Connection Utilization / Max",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 20
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": ".*utilization.*"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "percentunit"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": ".*max.*"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "short"
+              }
+            ]
+          }
+        ]
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_rtr_connection_utilization",
+          "legendFormat": "utilization",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_rtr_max_connections",
+          "legendFormat": "max",
+          "refId": "B"
+        }
+      ]
+    },
+    {
+      "id": 13,
+      "title": "Report Age",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 20
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_rtr_source_report_age_seconds",
+          "legendFormat": "source",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_rtr_runtime_report_age_seconds",
+          "legendFormat": "runtime",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_rtr_clients_report_age_seconds",
+          "legendFormat": "clients",
+          "refId": "C"
+        }
+      ]
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 40,
+  "tags": [
+    "rpki",
+    "inter-rp",
+    "routinator"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timezone": "browser",
+  "title": "RTR Service Overview",
+  "uid": "ours-rp-rtr-overview",
+  "version": 1
+}

deploy/docker-installer/compose/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: ours-rp-docker-installer
+    orgId: 1
+    folder: Ours RP Docker Installer
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards

deploy/docker-installer/compose/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    uid: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: true

deploy/docker-installer/compose/prometheus.yml

@@ -0,0 +1,13 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+
+scrape_configs:
+  - job_name: ours-rp-artifact-metrics
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - artifact-metrics:9556
+        labels:
+          rp: ours-rp
+          source: docker-installer-artifact-sidecar

deploy/docker-installer/docs/README.en.md

@@ -0,0 +1,200 @@
+# ours RP Docker Installer Guide (`__PACKAGE_ARCH__` / `__PACKAGE_PLATFORM__`)
+
+## Goal
+
+This package deploys ours RP on Linux servers whose host architecture matches the packaged installer metadata, using Docker Compose and continuous all-five RIR validation.
+
+The package includes four architecture-matched images: ours RP runtime, artifact metrics, Prometheus, and Grafana. Deployment does not need to pull application images on the target host. Runtime state, run artifacts, logs, Prometheus data and Grafana data are persisted through host bind mounts.
+
+Every installer build rebuilds the ours RP runtime and artifact metrics images. The deliverable name is `ours-rp-installer-{arch}-{git8}-{YYYYMMDDTHHMMSSZ}.tar.gz`, and both ours RP image tags use the current source commit. `PACKAGE-MANIFEST.env` records the full commit, UTC build time, dirty state, and image archive hashes; `./scripts/status.sh --brief` displays that provenance after installation.
+
+By default the host architecture must match `PACKAGE_ARCH` from `PACKAGE-MANIFEST.env`. If you intentionally want to run this package on a different host architecture through QEMU/binfmt, set:
+
+```bash
+ALLOW_CROSS_ARCH=1
+```
+
+## Compose Template Source
+
+`compose/` is the single production Docker Compose template source. The Docker installer build script copies this directory directly. Do not maintain a second copy of the Compose, Prometheus, or Grafana dashboard files elsewhere.
+
+The legacy direct Compose deployment entry point has been removed. For either amd64 or arm64, build the matching installer package with `scripts/docker/build_docker_installer_package.sh`, then use `scripts/install.sh`, `scripts/start.sh`, and `scripts/status.sh` as described here.
+
+## Quick Start
+
+```bash
+tar -xzf ours-rp-installer-__PACKAGE_ARCH__-*.tar.gz
+cd ours-rp-installer-__PACKAGE_ARCH__-*
+
+./scripts/install.sh
+cp .env.example .env   # install.sh creates .env automatically if missing
+vim .env
+./scripts/start.sh
+./scripts/status.sh
+```
+
+Defaults:
+
+- `PACKAGE_ARCH=__PACKAGE_ARCH__`
+- `PACKAGE_PLATFORM=__PACKAGE_PLATFORM__`
+- `RIRS=afrinic,apnic,arin,lacnic,ripe`
+- `MAX_RUNS=-1`
+- `INTERVAL_SECS=600`
+- `TAL_INPUT_MODE=file-live-ta`
+- `RESOURCE_VALIDATION_MODE=validation-update-03`
+- `LIVE_TA_REFRESH_BEFORE_SNAPSHOT=1`
+- `PERIODIC_SNAPSHOT_RESET=0`
+- `PERIODIC_SNAPSHOT_MAX_DELTAS=100`
+- `HOST_DATA_DIR=__HOST_DATA_DIR__`
+- `SOAK_RESTART_POLICY=unless-stopped`
+- `RPKI_IMAGE=__RUNTIME_IMAGE__`
+- `METRICS_IMAGE=__METRICS_IMAGE__`
+- `METRICS_PLATFORM=__PACKAGE_PLATFORM__`
+- `MONITOR_PLATFORM=__PACKAGE_PLATFORM__`
+- `ALLOW_CROSS_ARCH=0`
+- `RTR_REPORT_DIR=__HOST_DATA_DIR__/empty-rtr-report`, which mounts an empty fallback directory by default; after deploying a separate RTR service, register its same-host report directory through `scripts/register_rtr_monitor.sh`
+
+## First Start Semantics
+
+If there is no successful run under `HOST_DATA_DIR/runs`, `start.sh` starts the core `ours-rp-soak` service first and waits for the first snapshot to succeed before starting metrics, Prometheus and Grafana.
+
+Container/image mapping:
+
+- `ours-rp-soak` uses `RPKI_IMAGE`
+- `artifact-metrics` uses `METRICS_IMAGE`
+- `prometheus` / `grafana` use their monitor images
+
+The first snapshot refreshes live TA certificates before starting the RP process.
+
+## Resource Validation Mode
+
+New knob:
+
+```bash
+RESOURCE_VALIDATION_MODE=validation-update-03
+```
+
+Supported values:
+
+- `validation-update-03`: default behavior, using the current validation-update draft semantics;
+- `rfc6487`: switch back to the original RFC 6487 resource containment behavior.
+
+The installer / soak runner passes `--resource-validation-mode` to every `rpki` child start.
+When an upgraded deployment reuses an older `.env` that does not contain this key, the new package `.env.example` default `validation-update-03` remains in the final `.env`; an empty final value fails before Docker operations.
+
+## Architecture Guardrails
+
+The key scripts read:
+
+- `PACKAGE_ARCH` / `PACKAGE_PLATFORM` from `.env`
+- `PACKAGE-MANIFEST.env`
+- the current host `uname -m`
+
+Default behavior:
+
+1. matching host and package architectures: run natively;
+2. mismatched host and package architectures: fail with an explicit error;
+3. only when `ALLOW_CROSS_ARCH=1` is set do the scripts attempt to enable matching `binfmt/qemu`.
+
+That means an `arm64` package on an `x86_64` host does not silently switch to emulation by default.
+
+## Ports
+
+Default ports:
+
+- metrics: `http://<host>:9556/metrics`
+- Prometheus: `http://<host>:9090`
+- Grafana: `http://<host>:3000`
+
+Grafana credentials come from `.env`:
+
+```bash
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=admin
+```
+
+Change the password and restrict public access for production deployments.
+
+## Optional RTR report metrics input
+
+If a separately deployed RTR service on the same host continuously writes `rtr-source-*`, `rtr-runtime-*`, and `rtr-clients-*` JSON reports, run:
+
+```bash
+./scripts/register_rtr_monitor.sh --report-dir /root/rpki/report
+```
+
+The command accepts only an absolute path on the same host, validates the candidate Compose mount, and atomically updates `.env`. It recreates only `artifact-metrics`, Prometheus, and Grafana; it does not restart or recreate `ours-rp-soak`. The `artifact-metrics` container mounts the directory read-only and reads it through `RPKI_METRICS_RTR_REPORT_DIR`, producing `ours_rp_rtr_*` metrics. By default, `RTR_REPORT_DIR` points at an empty fallback directory under the installer data root, so the metrics container also starts cleanly without an RTR service.
+
+## CCR artifact format check metrics
+
+The metrics image ships the official `rpki-client 9.8`. For every newly completed run, `artifact-metrics` runs `rpki-client -f result.ccr` once and exports `ours_rp_ccr_format_check_*` metrics (cumulative pass/ccr_parse_error/tool_error counters, latest check result, check duration, consecutive failures). The Grafana `Ours RP Soak Overview` dashboard shows them in a dedicated panel row. On service start only the latest existing run is checked; older history is not backfilled.
+
+Related environment variables (normally no need to change): `CCR_CHECK_BIN` (defaults to the in-image `/opt/ours-rp/bin/rpki-client`) and `CCR_CHECK_TIMEOUT_SECS` (default 120 seconds).
+
+## Data Directory
+
+Default host directory:
+
+```text
+__HOST_DATA_DIR__/
+  state/
+  runs/
+  logs/
+  tmp/
+  prometheus/
+  grafana/
+```
+
+Each `runs/run_XXXX/` directory contains `report.json`, `result.ccr`, `input.cir`, `vrps.csv`, `vaps.csv`, `stage-timing.json`, logs and metadata.
+
+## Periodic Snapshot Reset
+
+New knobs:
+
+```bash
+PERIODIC_SNAPSHOT_RESET=0
+PERIODIC_SNAPSHOT_MAX_DELTAS=100
+```
+
+Semantics:
+
+- disabled by default, keeping previous behavior unchanged;
+- when enabled, one successful snapshot is followed by at most `N` successful delta runs;
+- after the threshold is reached, the next run is forced to snapshot;
+- before that forced snapshot, only the active `state/db` is reset, while `runs/`, `logs/`, `state/rsync-mirror`, `.env`, and Prometheus/Grafana data are preserved;
+- the counter is persisted independently in `HOST_DATA_DIR/state/run-lifecycle-state.json`, so it does not depend on retained `runs/` history;
+- if that lifecycle file is corrupt, it is backed up as `run-lifecycle-state.json.corrupt.<timestamp>.<pid>` before best-effort bootstrap from retained runs;
+- after a successful forced snapshot, the old DB staging is deleted so disk usage does not keep growing elsewhere.
+
+Check the latest `run-meta.json` for:
+
+- `sync_mode`
+- `snapshot_reason`
+- `periodic_snapshot_delta_count`
+- `periodic_snapshot_forced`
+- `reset_db_cleanup_status`
+
+## Common Commands
+
+```bash
+./scripts/status.sh
+./scripts/logs.sh ours-rp-soak --tail 200
+./scripts/restart.sh
+./scripts/stop.sh
+./scripts/cleanup.sh --keep-runs 100 --execute
+./scripts/uninstall.sh
+```
+
+For finite acceptance tests, for example `MAX_RUNS=3`, also set:
+
+```bash
+SOAK_RESTART_POLICY=no
+```
+
+Otherwise Compose `unless-stopped` will restart the container after it exits successfully.
+
+`uninstall.sh` keeps data by default. Use the following only when you really want to delete `HOST_DATA_DIR`:
+
+```bash
+./scripts/uninstall.sh --purge-data
+```

deploy/docker-installer/docs/README.zh-CN.md

@@ -0,0 +1,202 @@
+# ours RP Docker 安装包使用说明（`__PACKAGE_ARCH__` / `__PACKAGE_PLATFORM__`）
+
+## 目标
+
+本安装包用于在目标架构与包元数据匹配的 Linux 服务器上，通过 Docker Compose 部署 ours RP，并持续运行 all5 RIR 同步验证任务。
+
+安装包内置与包架构一致的四类镜像：ours RP runtime、artifact metrics、Prometheus、Grafana。部署时不需要现场拉取应用镜像。运行产物、状态数据库、日志、Prometheus 和 Grafana 数据均通过宿主机目录挂载保存。
+
+每次构建安装包都会重新构建 ours RP runtime 与 artifact metrics 镜像。交付包命名为 `ours-rp-installer-{arch}-{git8}-{YYYYMMDDTHHMMSSZ}.tar.gz`，两类 ours RP 镜像分别使用当前源码 commit 的 tag。完整 commit、UTC 构建时间、dirty 状态和镜像归档 hash 保存在 `PACKAGE-MANIFEST.env`，可通过 `./scripts/status.sh --brief` 查看。
+
+默认情况下，宿主架构必须与 `PACKAGE-MANIFEST.env` 中的 `PACKAGE_ARCH` 匹配；若确需在异构主机上通过 QEMU/binfmt 运行，必须显式设置：
+
+```bash
+ALLOW_CROSS_ARCH=1
+```
+
+## Compose 模板来源
+
+`compose/` 是唯一的生产 Docker Compose 模板来源，Docker 安装包构建脚本会直接复制该目录。不要在其他目录维护第二份 compose、Prometheus 或 Grafana dashboard 文件。
+
+旧的直接 Compose 部署入口已经移除。需要部署 amd64 或 arm64 时，统一先用 `scripts/docker/build_docker_installer_package.sh` 构建对应架构安装包，再按本说明执行 `scripts/install.sh`、`scripts/start.sh` 和 `scripts/status.sh`。
+
+## 快速开始
+
+```bash
+tar -xzf ours-rp-installer-__PACKAGE_ARCH__-*.tar.gz
+cd ours-rp-installer-__PACKAGE_ARCH__-*
+
+./scripts/install.sh
+cp .env.example .env   # 如 install.sh 已自动创建，可直接编辑现有 .env
+vim .env
+./scripts/start.sh
+./scripts/status.sh
+```
+
+默认配置：
+
+- `PACKAGE_ARCH=__PACKAGE_ARCH__`
+- `PACKAGE_PLATFORM=__PACKAGE_PLATFORM__`
+- `RIRS=afrinic,apnic,arin,lacnic,ripe`
+- `MAX_RUNS=-1`
+- `INTERVAL_SECS=600`
+- `TAL_INPUT_MODE=file-live-ta`
+- `RESOURCE_VALIDATION_MODE=validation-update-03`
+- `LIVE_TA_REFRESH_BEFORE_SNAPSHOT=1`
+- `PERIODIC_SNAPSHOT_RESET=0`
+- `PERIODIC_SNAPSHOT_MAX_DELTAS=100`
+- `HOST_DATA_DIR=__HOST_DATA_DIR__`
+- `SOAK_RESTART_POLICY=unless-stopped`
+- `RPKI_IMAGE=__RUNTIME_IMAGE__`
+- `METRICS_IMAGE=__METRICS_IMAGE__`
+- `METRICS_PLATFORM=__PACKAGE_PLATFORM__`
+- `MONITOR_PLATFORM=__PACKAGE_PLATFORM__`
+- `ALLOW_CROSS_ARCH=0`
+- `RTR_REPORT_DIR=__HOST_DATA_DIR__/empty-rtr-report`，默认挂载空目录；独立 RTR 服务部署完成后，使用 `scripts/register_rtr_monitor.sh` 注册同机 report 目录
+
+## 首次启动语义
+
+如果 `HOST_DATA_DIR/runs` 下没有成功 run，`start.sh` 会先启动核心 `ours-rp-soak`，等待第一轮 snapshot 成功后再启动 metrics、Prometheus 和 Grafana。
+
+容器镜像分工：
+
+- `ours-rp-soak` 使用 `RPKI_IMAGE`
+- `artifact-metrics` 使用 `METRICS_IMAGE`
+- `prometheus` / `grafana` 使用各自 monitor image
+
+第一轮 snapshot 会先拉取 live TA，避免 clean state 使用旧 fixture TA。
+
+## 资源验证模式
+
+新增配置：
+
+```bash
+RESOURCE_VALIDATION_MODE=validation-update-03
+```
+
+可选值：
+
+- `validation-update-03`：默认，按当前 validation update draft 语义运行；
+- `rfc6487`：切换为 RFC 6487 原始资源包含判定语义。
+
+installer / soak runner 每次启动 `rpki` 子进程时都会显式传入 `--resource-validation-mode`。
+如果升级时复用旧 `.env` 且缺少该变量，新包 `.env.example` 中的默认值 `validation-update-03` 会保留在最终 `.env` 中；最终值为空时升级会在 Docker 操作前失败。
+
+## 架构检查
+
+关键脚本会读取：
+
+- `.env` 中的 `PACKAGE_ARCH` / `PACKAGE_PLATFORM`
+- `PACKAGE-MANIFEST.env`
+- 当前宿主 `uname -m`
+
+默认行为：
+
+1. 宿主与包架构匹配：直接运行；
+2. 宿主与包架构不匹配：明确报错并停止；
+3. 仅当 `ALLOW_CROSS_ARCH=1` 时，脚本才会尝试启用对应架构的 `binfmt/qemu`。
+
+因此在 `x86_64` 主机上运行 `arm64` 包，不会默认静默进入模拟执行。
+
+## 访问端口
+
+默认端口：
+
+- metrics: `http://<host>:9556/metrics`
+- Prometheus: `http://<host>:9090`
+- Grafana: `http://<host>:3000`
+
+Grafana 默认账号密码来自 `.env`：
+
+```bash
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=admin
+```
+
+生产部署时应修改密码并限制外部访问。
+
+## 可选 RTR report 监控接入
+
+如果同一台机器上已经有独立部署的 RTR 服务，并且它持续输出 `rtr-source-*`、`rtr-runtime-*`、`rtr-clients-*` JSON report，执行：
+
+```bash
+./scripts/register_rtr_monitor.sh --report-dir /root/rpki/report
+```
+
+该命令只接受本机绝对路径，会先校验候选 Compose 挂载，再原子更新 `.env`。随后只重建 `artifact-metrics`、Prometheus 和 Grafana，不重启或重建 `ours-rp-soak`。`artifact-metrics` 以只读方式挂载该目录，并通过 `RPKI_METRICS_RTR_REPORT_DIR` 读取 report，输出 `ours_rp_rtr_*` 指标。默认 `RTR_REPORT_DIR` 指向安装包数据目录下的空 fallback 目录，因此未接入 RTR 服务时 metrics 容器也能稳定启动。
+
+## CCR 产物格式检查监控
+
+metrics 镜像内置官方 `rpki-client 9.8`。`artifact-metrics` 会对每个新完成 run 的 `result.ccr` 执行一次 `rpki-client -f` 格式检查，并输出 `ours_rp_ccr_format_check_*` 指标（累计 pass/ccr_parse_error/tool_error、最近检查结果、检查耗时、连续失败数）。Grafana `Ours RP Soak Overview` 底部有对应面板。服务启动时只补查最新的一个历史 run，不回扫全部历史。
+
+相关环境变量（通常无需修改）：`CCR_CHECK_BIN`（默认为镜像内 `/opt/ours-rp/bin/rpki-client`）、`CCR_CHECK_TIMEOUT_SECS`（默认 120 秒）。
+
+## 数据目录
+
+默认宿主机目录：
+
+```text
+__HOST_DATA_DIR__/
+  state/
+  runs/
+  logs/
+  tmp/
+  prometheus/
+  grafana/
+```
+
+`runs/run_XXXX/` 中包含每轮 `report.json`、`result.ccr`、`input.cir`、`vrps.csv`、`vaps.csv`、`stage-timing.json`、日志和元数据。
+
+## 定期 snapshot reset
+
+新增配置：
+
+```bash
+PERIODIC_SNAPSHOT_RESET=0
+PERIODIC_SNAPSHOT_MAX_DELTAS=100
+```
+
+语义：
+
+- 默认关闭，行为与旧版本一致；
+- 开启后，一次成功 snapshot 后最多连续执行 `N` 个成功 delta；
+- 达到阈值后，下一轮强制跑 snapshot；
+- 强制 snapshot 前只重置 active `state/db`，保留 `runs/`、`logs/`、`state/rsync-mirror`、`.env`、Prometheus/Grafana 数据；
+- 周期计数保存在独立 lifecycle 文件 `HOST_DATA_DIR/state/run-lifecycle-state.json`，不依赖 `runs/` 保留窗口；
+- lifecycle 文件损坏时会先备份为 `run-lifecycle-state.json.corrupt.<timestamp>.<pid>`，再从当前保留 run 尽力 bootstrap；
+- 强制 snapshot 成功后旧 DB staging 会被删除，避免磁盘只是换目录继续增长。
+
+可通过最新 `run-meta.json` 中的以下字段确认：
+
+- `sync_mode`
+- `snapshot_reason`
+- `periodic_snapshot_delta_count`
+- `periodic_snapshot_forced`
+- `reset_db_cleanup_status`
+
+## 常用命令
+
+```bash
+./scripts/status.sh
+./scripts/logs.sh ours-rp-soak --tail 200
+./scripts/restart.sh
+./scripts/stop.sh
+./scripts/cleanup.sh --keep-runs 100 --execute
+./scripts/uninstall.sh
+```
+
+如果做有限轮次验收，例如 `MAX_RUNS=3`，建议同时设置：
+
+```bash
+SOAK_RESTART_POLICY=no
+```
+
+否则 Compose 的 `unless-stopped` 策略会在容器正常退出后再次拉起下一轮。
+
+`uninstall.sh` 默认不删除数据。只有显式执行：
+
+```bash
+./scripts/uninstall.sh --purge-data
+```
+
+才会删除 `HOST_DATA_DIR`。

deploy/docker-installer/docs/operations.en.md

@@ -0,0 +1,154 @@
+# Operations Guide (`__PACKAGE_ARCH__`)
+
+## Install
+
+```bash
+./scripts/install.sh
+```
+
+The installer is idempotent:
+
+- existing `.env` is kept;
+- existing Docker/Compose installation is reused;
+- repeated loading of packaged runtime, metrics, Prometheus and Grafana images for the package architecture is safe;
+- existing data directory is reused;
+- matching host/package architecture is required by default.
+
+## Start
+
+```bash
+./scripts/start.sh
+```
+
+Start without waiting for the first snapshot:
+
+```bash
+./scripts/start.sh --no-wait-first-run
+```
+
+If you intentionally want to run this package on a different host architecture through QEMU/binfmt, first set:
+
+```bash
+ALLOW_CROSS_ARCH=1
+```
+
+Then rerun `./scripts/install.sh` or `./scripts/self-check.sh`, and only then will the scripts attempt to enable the matching binfmt handler.
+
+## Stop and Restart
+
+```bash
+./scripts/stop.sh
+./scripts/restart.sh
+```
+
+## Status Checks
+
+```bash
+./scripts/status.sh
+./scripts/self-check.sh
+```
+
+Important checks:
+
+- Docker/Compose availability;
+- runtime, metrics, Prometheus and Grafana images exist;
+- `HOST_DATA_DIR` is writable;
+- Compose config is valid;
+- latest run status;
+- metrics, Prometheus and Grafana endpoints;
+- whether `host_arch`, `package_arch`, and `arch_mode` match expectations.
+
+`status.sh` also prints:
+
+- `package_arch`
+- `package_platform`
+- `runtime_image`
+- `metrics_image`
+- `allow_cross_arch`
+- `periodic_snapshot_reset`
+- `periodic_snapshot_max_deltas`
+- `rtr_report_dir`
+- `rtr_report_container_dir`
+
+To verify RTR report ingestion:
+
+```bash
+./scripts/register_rtr_monitor.sh --report-dir /root/rpki/report --dry-run
+./scripts/register_rtr_monitor.sh --report-dir /root/rpki/report
+./scripts/status.sh
+curl -s "http://127.0.0.1:${METRICS_PORT:-9556}/metrics" | grep '^ours_rp_rtr_' | head
+```
+
+Registration accepts only an absolute directory on the same host. The command retains a timestamped `.env` backup and updates only the metrics, Prometheus, and Grafana sidecars; the `ours-rp-soak` container ID must remain unchanged.
+
+## Upgrade
+
+Extract the new package into a new directory and explicitly reuse the existing `.env` through the upgrade script:
+
+```bash
+./scripts/upgrade.sh --reuse-env-from /path/to/old-installer/.env
+```
+
+Configuration is merged in this order, with a key on the right overriding the same key on the left:
+
+```text
+new .env.example < old .env < environment variables on this upgrade command
+```
+
+For example, override the next-run interval and Grafana password for this upgrade:
+
+```bash
+INTERVAL_SECS=300 GRAFANA_ADMIN_PASSWORD='new-password' \
+  ./scripts/upgrade.sh --reuse-env-from /path/to/old-installer/.env
+```
+
+Before any image is loaded or container is started, upgrade verifies that every key declared by the new template is present and non-empty in the final `.env`. An explicit empty assignment fails, for example `INTERVAL_SECS= ./scripts/upgrade.sh ...`. Removed keys from an old `.env` are not copied, and the script reports key names only, never password-like values.
+
+The installer reads `.env` as Compose-compatible dotenv assignments. Do not put shell command substitutions or environment references such as `$HOME` in the file; use concrete values or command-level overrides instead.
+
+If the new package directory already has a `.env`, the upgrade script keeps it. In that case `--reuse-env-from` is not imported again, but environment variables from the current command still override the existing `.env`.
+
+When an older `.env` lacks a key introduced by the new package, the new `.env.example` default remains in the final `.env`. Upgrade fails before Docker operations if any final declared value is empty.
+
+Upgrade does not delete:
+
+- `runs/`
+- `logs/`
+- `state/rsync-mirror`
+- runtime configuration referenced by `.env`
+- Prometheus / Grafana data
+
+To validate periodic forced snapshot behavior, temporarily set:
+
+```bash
+PERIODIC_SNAPSHOT_RESET=1
+PERIODIC_SNAPSHOT_MAX_DELTAS=2
+```
+
+Then confirm the latest `run-meta.json` contains:
+
+```bash
+snapshot_reason=periodic_snapshot_delta_limit
+```
+
+And inspect the independent lifecycle state:
+
+```bash
+jq '{last_run,last_success_snapshot,successful_deltas_since_snapshot,state_health}' \
+  "${HOST_DATA_DIR}/state/run-lifecycle-state.json"
+```
+
+After validation, restore:
+
+```bash
+PERIODIC_SNAPSHOT_MAX_DELTAS=100
+```
+
+## Cleanup
+
+```bash
+./scripts/cleanup.sh --keep-runs 100
+./scripts/cleanup.sh --keep-runs 100 --execute
+```
+
+Cleanup is dry-run by default. Add `--execute` after reviewing the output.

deploy/docker-installer/docs/operations.zh-CN.md

@@ -0,0 +1,154 @@
+# 运维手册（`__PACKAGE_ARCH__`）
+
+## 安装
+
+```bash
+./scripts/install.sh
+```
+
+安装脚本是幂等的：
+
+- 已有 `.env` 不覆盖；
+- 已安装 Docker/Compose 则跳过；
+- 包内 runtime、metrics、Prometheus、Grafana 对应架构镜像重复加载是安全的；
+- 数据目录已存在则复用；
+- 默认要求宿主与包架构匹配。
+
+## 启动
+
+```bash
+./scripts/start.sh
+```
+
+如需后台启动后不等待首轮 snapshot：
+
+```bash
+./scripts/start.sh --no-wait-first-run
+```
+
+如需显式允许异构主机通过 QEMU/binfmt 跑包，先在 `.env` 中设置：
+
+```bash
+ALLOW_CROSS_ARCH=1
+```
+
+然后重新执行 `./scripts/install.sh` 或 `./scripts/self-check.sh`，脚本才会尝试启用对应 binfmt。
+
+## 停止和重启
+
+```bash
+./scripts/stop.sh
+./scripts/restart.sh
+```
+
+## 状态检查
+
+```bash
+./scripts/status.sh
+./scripts/self-check.sh
+```
+
+重点检查项：
+
+- Docker/Compose 可用；
+- runtime、metrics、Prometheus、Grafana 镜像存在；
+- `HOST_DATA_DIR` 可写；
+- Compose 配置合法；
+- 最新 run 状态；
+- metrics、Prometheus、Grafana endpoint；
+- `host_arch`、`package_arch`、`arch_mode` 是否符合预期。
+
+`status.sh` 还会显示：
+
+- `package_arch`
+- `package_platform`
+- `runtime_image`
+- `metrics_image`
+- `allow_cross_arch`
+- `periodic_snapshot_reset`
+- `periodic_snapshot_max_deltas`
+- `rtr_report_dir`
+- `rtr_report_container_dir`
+
+如需确认 RTR report 已接入：
+
+```bash
+./scripts/register_rtr_monitor.sh --report-dir /root/rpki/report --dry-run
+./scripts/register_rtr_monitor.sh --report-dir /root/rpki/report
+./scripts/status.sh
+curl -s "http://127.0.0.1:${METRICS_PORT:-9556}/metrics" | grep '^ours_rp_rtr_' | head
+```
+
+注册只支持同机的绝对目录。注册命令会保留一份带时间戳的 `.env` 备份，并且只更新 metrics、Prometheus、Grafana 三个旁路容器；`ours-rp-soak` 的容器 ID 应保持不变。
+
+## 升级
+
+把新安装包解压到新目录后，推荐通过升级脚本显式复用旧 `.env`：
+
+```bash
+./scripts/upgrade.sh --reuse-env-from /path/to/old-installer/.env
+```
+
+配置按以下优先级合并，右侧同名键覆盖左侧：
+
+```text
+新包 .env.example < 旧 .env < 本次 upgrade 命令的环境变量
+```
+
+例如，临时调整下轮触发间隔和 Grafana 密码：
+
+```bash
+INTERVAL_SECS=300 GRAFANA_ADMIN_PASSWORD='new-password' \
+  ./scripts/upgrade.sh --reuse-env-from /path/to/old-installer/.env
+```
+
+升级会在加载镜像或启动容器前检查：新模板声明的所有变量在最终 `.env` 中都必须存在且非空。显式传入空值会失败，例如 `INTERVAL_SECS= ./scripts/upgrade.sh ...`。旧 `.env` 中已不在新模板内的键不会复制，脚本只输出相关键名而不会输出密码等值。
+
+installer 按与 Compose 一致的 dotenv 赋值格式读取 `.env`。不要在文件中使用 Shell 命令替换或 `$HOME` 一类环境变量引用；请使用确定的值，或在执行 upgrade 时通过命令环境变量覆盖。
+
+如果新目录已经存在 `.env`，升级脚本会保留它，不覆盖；此时 `--reuse-env-from` 不会再次导入旧配置，但本次命令的环境变量仍会覆盖已存在的 `.env`。
+
+旧 `.env` 缺少新版本新增的键时，新包 `.env.example` 的默认值会保留在最终 `.env` 中；任一最终值为空则升级会在 Docker 操作前失败。
+
+升级不会删除以下数据：
+
+- `runs/`
+- `logs/`
+- `state/rsync-mirror`
+- `.env` 对应的运行配置
+- Prometheus / Grafana 数据
+
+验证定期 forced snapshot 时，可临时设置：
+
+```bash
+PERIODIC_SNAPSHOT_RESET=1
+PERIODIC_SNAPSHOT_MAX_DELTAS=2
+```
+
+然后检查最新 `run-meta.json` 应出现：
+
+```bash
+snapshot_reason=periodic_snapshot_delta_limit
+```
+
+并检查独立 lifecycle 状态：
+
+```bash
+jq '{last_run,last_success_snapshot,successful_deltas_since_snapshot,state_health}' \
+  "${HOST_DATA_DIR}/state/run-lifecycle-state.json"
+```
+
+验证完成后恢复：
+
+```bash
+PERIODIC_SNAPSHOT_MAX_DELTAS=100
+```
+
+## 清理
+
+```bash
+./scripts/cleanup.sh --keep-runs 100
+./scripts/cleanup.sh --keep-runs 100 --execute
+```
+
+默认 dry-run，确认后加 `--execute`。

deploy/docker-installer/docs/troubleshooting.en.md

@@ -0,0 +1,142 @@
+# Troubleshooting (`__PACKAGE_ARCH__` / `__PACKAGE_PLATFORM__`)
+
+## Docker or Compose Is Unavailable
+
+Run:
+
+```bash
+docker version
+docker compose version
+```
+
+If missing, run:
+
+```bash
+./scripts/install.sh
+```
+
+## Package Architecture Mismatch
+
+If `status.sh`, `install.sh`, or `self-check.sh` reports:
+
+```text
+package architecture mismatch
+```
+
+the host `uname -m` does not match the packaged `PACKAGE_ARCH=__PACKAGE_ARCH__`. The default behavior is to fail clearly rather than silently switching to emulation.
+
+Only if you intentionally accept QEMU/binfmt cross-architecture execution should you set:
+
+```bash
+ALLOW_CROSS_ARCH=1
+```
+
+Then rerun install or self-check.
+
+## `__PACKAGE_ARCH__` Image Cannot Run
+
+Running `__PACKAGE_PLATFORM__` images on a different host architecture requires binfmt/qemu. The installer only attempts this automatically when `ALLOW_CROSS_ARCH=1`; you can also enable it manually:
+
+```bash
+docker run --rm --privileged tonistiigi/binfmt --install __PACKAGE_ARCH__
+docker run --rm --platform __PACKAGE_PLATFORM__ debian:bookworm-slim uname -m
+```
+
+For `arm64` packages the expected output is `aarch64`; for `amd64` packages the expected output is `x86_64`.
+
+To confirm the metrics image is also loaded correctly:
+
+```bash
+docker image inspect "$(grep '^METRICS_IMAGE=' .env | cut -d= -f2-)"
+docker run --rm --platform __PACKAGE_PLATFORM__ \
+  "$(grep '^METRICS_IMAGE=' .env | cut -d= -f2-)" \
+  /opt/ours-rp/bin/rpki_artifact_metrics --help
+```
+
+## RTR report metrics are empty
+
+First check registration and the report directory:
+
+```bash
+./scripts/status.sh
+./scripts/register_rtr_monitor.sh --report-dir /root/rpki/report --dry-run
+```
+
+Check the metrics container mount:
+
+```bash
+docker inspect "$(grep '^COMPOSE_PROJECT_NAME=' .env | cut -d= -f2-)-artifact-metrics" \
+  | jq '.[0].Mounts[] | select(.Destination=="/var/lib/ours-rp/rtr-report")'
+```
+
+## First Snapshot Times Out
+
+All-five snapshot can be slow, especially under QEMU. Increase timeout:
+
+```bash
+./scripts/start.sh --timeout-secs 14400
+```
+
+## Output Counts Are Too Low
+
+Check:
+
+```bash
+grep LIVE_TA_REFRESH_BEFORE_SNAPSHOT .env
+ls -l __HOST_DATA_DIR__/state/live-ta
+tail -100 __HOST_DATA_DIR__/logs/live-ta-refresh-*.log
+```
+
+In `file-live-ta` mode, snapshot should wait until live TA refresh succeeds.
+
+## Grafana Login Fails
+
+Check `.env`:
+
+```bash
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=admin
+```
+
+If Grafana has already started, changing `.env` may not reset the existing Grafana database. Stop services and back up/clean `${HOST_DATA_DIR}/grafana` if needed.
+
+## A Finite Acceptance Test Starts an Extra Run
+
+If `.env` sets a finite `MAX_RUNS=3` while `SOAK_RESTART_POLICY=unless-stopped`, Docker Compose restarts the soak container after it exits successfully.
+
+For finite tests, set:
+
+```bash
+SOAK_RESTART_POLICY=no
+```
+
+## How to Confirm a Periodic Forced Snapshot
+
+Check the latest run metadata:
+
+```bash
+latest="$(find ${HOST_DATA_DIR}/runs -maxdepth 1 -type d -name 'run_*' | sort | tail -1)"
+jq '{run_id,sync_mode,snapshot_reason,periodic_snapshot_delta_count,periodic_snapshot_forced,reset_db_cleanup_status}' "$latest/run-meta.json"
+```
+
+For a threshold-triggered reset you should see:
+
+- `sync_mode: "snapshot"`
+- `snapshot_reason: "periodic_snapshot_delta_limit"`
+- `periodic_snapshot_forced: true`
+
+To confirm delta counting still advances after retained runs are pruned, also inspect:
+
+```bash
+jq '{last_success_snapshot,successful_deltas_since_snapshot,recent_runs_count:(.recent_runs|length)}' \
+  "${HOST_DATA_DIR}/state/run-lifecycle-state.json"
+```
+
+## Lifecycle State File Is Corrupt
+
+The script backs up the corrupt file before best-effort bootstrap from retained runs:
+
+```bash
+ls -1 "${HOST_DATA_DIR}/state"/run-lifecycle-state.json.corrupt.*
+jq . "${HOST_DATA_DIR}/state/run-lifecycle-state.json"
+```

deploy/docker-installer/docs/troubleshooting.zh-CN.md

@@ -0,0 +1,142 @@
+# 故障排查（`__PACKAGE_ARCH__` / `__PACKAGE_PLATFORM__`）
+
+## Docker 或 Compose 不可用
+
+执行：
+
+```bash
+docker version
+docker compose version
+```
+
+如果缺失，重新执行：
+
+```bash
+./scripts/install.sh
+```
+
+## 架构不匹配报错
+
+当 `status.sh` / `install.sh` / `self-check.sh` 报出：
+
+```text
+package architecture mismatch
+```
+
+说明宿主 `uname -m` 与包的 `PACKAGE_ARCH=__PACKAGE_ARCH__` 不一致。默认行为就是失败而不是静默模拟运行。
+
+只有在你明确接受 QEMU/binfmt 跨架构运行时，才设置：
+
+```bash
+ALLOW_CROSS_ARCH=1
+```
+
+然后重新执行安装或自检。
+
+## `__PACKAGE_ARCH__` 镜像无法运行
+
+在异构主机上运行 `__PACKAGE_PLATFORM__` 镜像需要 binfmt/qemu。安装脚本只有在 `ALLOW_CROSS_ARCH=1` 时才会自动尝试启用；也可以手动执行：
+
+```bash
+docker run --rm --privileged tonistiigi/binfmt --install __PACKAGE_ARCH__
+docker run --rm --platform __PACKAGE_PLATFORM__ debian:bookworm-slim uname -m
+```
+
+对于 `arm64` 包，预期输出 `aarch64`；对于 `amd64` 包，预期输出 `x86_64`。
+
+如需进一步确认 metrics 镜像也已正确加载：
+
+```bash
+docker image inspect "$(grep '^METRICS_IMAGE=' .env | cut -d= -f2-)"
+docker run --rm --platform __PACKAGE_PLATFORM__ \
+  "$(grep '^METRICS_IMAGE=' .env | cut -d= -f2-)" \
+  /opt/ours-rp/bin/rpki_artifact_metrics --help
+```
+
+## RTR report 指标为空
+
+先检查注册状态和 report 目录：
+
+```bash
+./scripts/status.sh
+./scripts/register_rtr_monitor.sh --report-dir /root/rpki/report --dry-run
+```
+
+检查 metrics 容器是否挂载该目录：
+
+```bash
+docker inspect "$(grep '^COMPOSE_PROJECT_NAME=' .env | cut -d= -f2-)-artifact-metrics" \
+  | jq '.[0].Mounts[] | select(.Destination=="/var/lib/ours-rp/rtr-report")'
+```
+
+## 首轮 snapshot 超时
+
+all5 snapshot 可能很慢，尤其在 QEMU 环境。可以提高超时：
+
+```bash
+./scripts/start.sh --timeout-secs 14400
+```
+
+## 产物数量异常偏低
+
+检查：
+
+```bash
+grep LIVE_TA_REFRESH_BEFORE_SNAPSHOT .env
+ls -l __HOST_DATA_DIR__/state/live-ta
+tail -100 __HOST_DATA_DIR__/logs/live-ta-refresh-*.log
+```
+
+`file-live-ta` 模式下，snapshot 应等待 live TA 成功刷新。
+
+## Grafana 无法登录
+
+确认 `.env` 中：
+
+```bash
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=admin
+```
+
+如果曾经启动过 Grafana，修改 `.env` 不一定重置已有 Grafana 数据库账号。可以停止服务后按需备份并清理 `${HOST_DATA_DIR}/grafana`。
+
+## 有限轮次验收后又多跑了一轮
+
+如果 `.env` 中设置了 `MAX_RUNS=3` 这类有限轮次，同时 `SOAK_RESTART_POLICY=unless-stopped`，Docker Compose 会在 soak 容器正常退出后重新启动容器。
+
+有限验收建议设置：
+
+```bash
+SOAK_RESTART_POLICY=no
+```
+
+## 如何确认触发了定期 forced snapshot
+
+检查最新 run metadata：
+
+```bash
+latest="$(find ${HOST_DATA_DIR}/runs -maxdepth 1 -type d -name 'run_*' | sort | tail -1)"
+jq '{run_id,sync_mode,snapshot_reason,periodic_snapshot_delta_count,periodic_snapshot_forced,reset_db_cleanup_status}' "$latest/run-meta.json"
+```
+
+阈值触发时应看到：
+
+- `sync_mode: "snapshot"`
+- `snapshot_reason: "periodic_snapshot_delta_limit"`
+- `periodic_snapshot_forced: true`
+
+如需确认 retain 裁剪后仍在累计 delta，可继续检查：
+
+```bash
+jq '{last_success_snapshot,successful_deltas_since_snapshot,recent_runs_count:(.recent_runs|length)}' \
+  "${HOST_DATA_DIR}/state/run-lifecycle-state.json"
+```
+
+## Lifecycle 状态文件损坏
+
+脚本会先备份损坏文件，再从当前保留的 `runs/` 尽力 bootstrap：
+
+```bash
+ls -1 "${HOST_DATA_DIR}/state"/run-lifecycle-state.json.corrupt.*
+jq . "${HOST_DATA_DIR}/state/run-lifecycle-state.json"
+```

deploy/docker-installer/scripts/cleanup.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=common.sh
+source "$SCRIPT_DIR/common.sh"
+
+DRY_RUN=1
+KEEP_RUNS=""
+usage() {
+  cat <<'USAGE'
+Usage: ./scripts/cleanup.sh [--execute] [--keep-runs N]
+
+By default this is a dry-run. It removes old run_* directories beyond KEEP_RUNS
+and clears tmp contents.
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --execute)
+      DRY_RUN=0
+      shift
+      ;;
+    --keep-runs)
+      KEEP_RUNS="$2"
+      shift 2
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+load_env
+keep="${KEEP_RUNS:-${RETAIN_RUNS:-100}}"
+mapfile -t runs < <(find "$HOST_DATA_DIR/runs" -maxdepth 1 -type d -name 'run_*' 2>/dev/null | sort)
+delete_count=$(( ${#runs[@]} - keep ))
+if (( delete_count > 0 )); then
+  for ((i=0; i<delete_count; i++)); do
+    if [[ "$DRY_RUN" == "1" ]]; then
+      echo "DRY-RUN rm -rf ${runs[$i]}"
+    else
+      rm -rf "${runs[$i]}"
+    fi
+  done
+fi
+if [[ "$DRY_RUN" == "1" ]]; then
+  echo "DRY-RUN rm -rf $HOST_DATA_DIR/tmp/*"
+else
+  find "$HOST_DATA_DIR/tmp" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
+fi
+df -h "$HOST_DATA_DIR" 2>/dev/null || true

deploy/docker-installer/scripts/common.sh

@@ -0,0 +1,819 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+INSTALLER_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+ENV_FILE="${ENV_FILE:-$INSTALLER_ROOT/.env}"
+ENV_EXAMPLE="$INSTALLER_ROOT/.env.example"
+COMPOSE_FILE="$INSTALLER_ROOT/compose/docker-compose.yml"
+MANIFEST_FILE="${MANIFEST_FILE:-$INSTALLER_ROOT/PACKAGE-MANIFEST.env}"
+declare -a ENV_TEMPLATE_KEYS=()
+declare -A ENV_TEMPLATE_KEY_SET=()
+declare -A INVOCATION_ENV_OVERRIDES=()
+declare -A REUSED_ENV_KEYS=()
+declare -A ENV_PARSED_VALUES=()
+
+log() {
+  printf '[ours-rp-installer] %s\n' "$*"
+}
+
+warn() {
+  printf '[ours-rp-installer][WARN] %s\n' "$*" >&2
+}
+
+die() {
+  printf '[ours-rp-installer][ERROR] %s\n' "$*" >&2
+  exit 1
+}
+
+require_cmd() {
+  command -v "$1" >/dev/null 2>&1 || die "missing command: $1"
+}
+
+normalize_arch() {
+  case "$1" in
+    amd64|x86_64|linux/amd64)
+      printf 'amd64\n'
+      ;;
+    arm64|aarch64|linux/arm64)
+      printf 'arm64\n'
+      ;;
+    *)
+      return 1
+      ;;
+  esac
+}
+
+platform_for_arch() {
+  printf 'linux/%s\n' "$1"
+}
+
+env_file_has_key() {
+  local env_path="$1"
+  local key="$2"
+  [[ -f "$env_path" ]] && grep -Eq "^${key}=" "$env_path"
+}
+
+load_env_template_keys() {
+  local env_path="$1"
+  local line
+  local key
+
+  [[ -f "$env_path" ]] || die "missing environment template: $env_path"
+  ENV_TEMPLATE_KEYS=()
+  ENV_TEMPLATE_KEY_SET=()
+
+  while IFS= read -r line || [[ -n "$line" ]]; do
+    line="${line%$'\r'}"
+    [[ "$line" =~ ^[[:space:]]*$ || "$line" =~ ^[[:space:]]*# ]] && continue
+    [[ "$line" =~ ^([A-Za-z_][A-Za-z0-9_]*)= ]] || die "invalid environment assignment in $env_path: $line"
+    key="${BASH_REMATCH[1]}"
+    [[ -z "${ENV_TEMPLATE_KEY_SET[$key]+x}" ]] || die "duplicate environment key in $env_path: $key"
+    ENV_TEMPLATE_KEYS+=("$key")
+    ENV_TEMPLATE_KEY_SET["$key"]=1
+  done < "$env_path"
+
+  (( ${#ENV_TEMPLATE_KEYS[@]} > 0 )) || die "no environment assignments found in $env_path"
+}
+
+template_has_env_key() {
+  local key="$1"
+  [[ -n "${ENV_TEMPLATE_KEY_SET[$key]+x}" ]]
+}
+
+env_file_assignment_keys() {
+  local env_path="$1"
+  local line
+
+  [[ -f "$env_path" ]] || die "missing environment file: $env_path"
+  while IFS= read -r line || [[ -n "$line" ]]; do
+    line="${line%$'\r'}"
+    [[ "$line" =~ ^([A-Za-z_][A-Za-z0-9_]*)= ]] || continue
+    printf '%s\n' "${BASH_REMATCH[1]}"
+  done < "$env_path"
+}
+
+dotenv_decode_value() {
+  local raw_value="$1"
+  local quote
+  local body
+  local decoded=""
+  local character
+  local next_character
+  local index=0
+
+  [[ -n "$raw_value" ]] || {
+    printf '\n'
+    return 0
+  }
+
+  quote="${raw_value:0:1}"
+  case "$quote" in
+    "'")
+      [[ "${raw_value: -1}" == "'" && ${#raw_value} -ge 2 ]] || return 1
+      body="${raw_value:1:${#raw_value}-2}"
+      while (( index < ${#body} )); do
+        character="${body:index:1}"
+        if [[ "$character" == "\\" && $((index + 1)) -lt ${#body} ]]; then
+          next_character="${body:index + 1:1}"
+          case "$next_character" in
+            "'")
+              decoded+="$next_character"
+              ((index += 2))
+              continue
+              ;;
+          esac
+        fi
+        decoded+="$character"
+        ((index += 1))
+      done
+      ;;
+    '"')
+      [[ "${raw_value: -1}" == '"' && ${#raw_value} -ge 2 ]] || return 1
+      body="${raw_value:1:${#raw_value}-2}"
+      while (( index < ${#body} )); do
+        character="${body:index:1}"
+        if [[ "$character" == '$' && $((index + 1)) -lt ${#body} && "${body:index + 1:1}" == '$' ]]; then
+          decoded+='$'
+          ((index += 2))
+          continue
+        fi
+        if [[ "$character" == "\\" && $((index + 1)) -lt ${#body} ]]; then
+          next_character="${body:index + 1:1}"
+          case "$next_character" in
+            '"'|\\|'$'|'`')
+              decoded+="$next_character"
+              ((index += 2))
+              continue
+              ;;
+            n)
+              decoded+=$'\n'
+              ((index += 2))
+              continue
+              ;;
+            r)
+              decoded+=$'\r'
+              ((index += 2))
+              continue
+              ;;
+            t)
+              decoded+=$'\t'
+              ((index += 2))
+              continue
+              ;;
+          esac
+        fi
+        decoded+="$character"
+        ((index += 1))
+      done
+      ;;
+    *)
+      decoded="$raw_value"
+      ;;
+  esac
+
+  printf '%s' "$decoded"
+}
+
+parse_env_file() {
+  local env_path="$1"
+  local line
+  local parsed_key
+  local raw_value
+  local parsed_value
+
+  [[ -f "$env_path" ]] || die "missing environment file: $env_path"
+  ENV_PARSED_VALUES=()
+  while IFS= read -r line || [[ -n "$line" ]]; do
+    line="${line%$'\r'}"
+    [[ "$line" =~ ^[[:space:]]*$ || "$line" =~ ^[[:space:]]*# ]] && continue
+    [[ "$line" =~ ^([A-Za-z_][A-Za-z0-9_]*)=(.*)$ ]] || return 2
+    parsed_key="${BASH_REMATCH[1]}"
+    raw_value="${BASH_REMATCH[2]}"
+    parsed_value="$(dotenv_decode_value "$raw_value")" || return 2
+    ENV_PARSED_VALUES["$parsed_key"]="$parsed_value"
+  done < "$env_path"
+}
+
+env_file_value() {
+  local env_path="$1"
+  local key="$2"
+
+  parse_env_file "$env_path"
+
+  [[ -n "${ENV_PARSED_VALUES[$key]+x}" ]] || return 3
+  printf '%s' "${ENV_PARSED_VALUES[$key]}"
+}
+
+env_file_set_value() {
+  local env_path="$1"
+  local key="$2"
+  local value="$3"
+  local tmp_env
+  local encoded_value
+
+  [[ "$value" != *$'\n'* && "$value" != *$'\r'* ]] || die "environment value for $key contains an unsupported newline"
+  encoded_value="$(env_file_encode_value "$value")"
+  if ! env_file_has_key "$env_path" "$key"; then
+    printf '%s=%s\n' "$key" "$encoded_value" >> "$env_path"
+    return 0
+  fi
+  tmp_env="$(mktemp)"
+  ENV_FILE_REPLACEMENT="$key=$encoded_value" awk -v key="$key" '
+    BEGIN { done=0 }
+    $0 ~ "^" key "=" { print ENVIRON["ENV_FILE_REPLACEMENT"]; done=1; next }
+    { print }
+    END { if (!done) print ENVIRON["ENV_FILE_REPLACEMENT"] }
+  ' "$env_path" > "$tmp_env"
+  mv "$tmp_env" "$env_path"
+}
+
+env_file_encode_value() {
+  local value="$1"
+  local escaped="$value"
+
+  [[ "$value" != *$'\n'* && "$value" != *$'\r'* ]] || die "environment value contains an unsupported newline"
+  escaped="${escaped//\\/\\\\}"
+  escaped="${escaped//\"/\\\"}"
+  escaped="${escaped//\$/\$\$}"
+  printf '"%s"' "$escaped"
+}
+
+load_env_file() {
+  local env_path="$1"
+  local key
+
+  parse_env_file "$env_path" || die "unable to parse environment file: $env_path"
+  for key in "${!ENV_PARSED_VALUES[@]}"; do
+    printf -v "$key" '%s' "${ENV_PARSED_VALUES[$key]}"
+    export "$key"
+  done
+}
+
+validate_env_file_contract() {
+  local env_path="$1"
+  local invalid_keys=""
+  local key
+
+  [[ -f "$env_path" ]] || die "missing environment file: $env_path"
+  (( ${#ENV_TEMPLATE_KEYS[@]} > 0 )) || die "environment template keys were not loaded"
+
+  if ! parse_env_file "$env_path"; then
+    die "unable to parse environment file: $env_path"
+  fi
+
+  for key in "${ENV_TEMPLATE_KEYS[@]}"; do
+    if [[ -z "${ENV_PARSED_VALUES[$key]+x}" || -z "${ENV_PARSED_VALUES[$key]}" ]]; then
+      invalid_keys+="$key"$'\n'
+    fi
+  done
+
+  [[ -z "$invalid_keys" ]] || die "environment contract requires non-empty values in $env_path: ${invalid_keys//$'\n'/,}"
+}
+
+capture_invocation_env_overrides() {
+  local key
+
+  (( ${#ENV_TEMPLATE_KEYS[@]} > 0 )) || die "environment template keys were not loaded"
+  INVOCATION_ENV_OVERRIDES=()
+  for key in "${ENV_TEMPLATE_KEYS[@]}"; do
+    if [[ -v "$key" ]]; then
+      INVOCATION_ENV_OVERRIDES["$key"]="${!key}"
+    fi
+  done
+}
+
+log_env_key_group() {
+  local group="$1"
+  shift
+  local rendered="none"
+  if (( $# > 0 )); then
+    local IFS=,
+    rendered="$*"
+  fi
+  log "$group=$rendered"
+}
+
+env_flag_enabled() {
+  case "${1:-0}" in
+    1|true|TRUE|yes|YES|on|ON)
+      return 0
+      ;;
+    *)
+      return 1
+      ;;
+  esac
+}
+
+detect_host_arch() {
+  local raw_arch
+  raw_arch="$(uname -m)"
+  normalize_arch "$raw_arch" || die "unsupported host architecture: $raw_arch"
+}
+
+load_manifest() {
+  MANIFEST_PACKAGE_ARCH_RAW=""
+  MANIFEST_PACKAGE_PLATFORM_RAW=""
+  if [[ -f "$MANIFEST_FILE" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$MANIFEST_FILE"
+    set +a
+    MANIFEST_PACKAGE_ARCH_RAW="${PACKAGE_ARCH:-${package_arch:-}}"
+    MANIFEST_PACKAGE_PLATFORM_RAW="${PACKAGE_PLATFORM:-${package_platform:-}}"
+  fi
+}
+
+effective_package_arch() {
+  if [[ -n "${PACKAGE_ARCH:-}" ]]; then
+    normalize_arch "$PACKAGE_ARCH" || die "unsupported PACKAGE_ARCH=${PACKAGE_ARCH}"
+    return 0
+  fi
+  if [[ -n "${PACKAGE_PLATFORM:-}" ]]; then
+    normalize_arch "$PACKAGE_PLATFORM" || die "unsupported PACKAGE_PLATFORM=${PACKAGE_PLATFORM}"
+    return 0
+  fi
+  if [[ -n "${RPKI_PLATFORM:-}" ]]; then
+    normalize_arch "$RPKI_PLATFORM" || die "unsupported RPKI_PLATFORM=${RPKI_PLATFORM}"
+    return 0
+  fi
+  if [[ -n "${METRICS_PLATFORM:-}" ]]; then
+    normalize_arch "$METRICS_PLATFORM" || die "unsupported METRICS_PLATFORM=${METRICS_PLATFORM}"
+    return 0
+  fi
+  if [[ -n "${MONITOR_PLATFORM:-}" ]]; then
+    normalize_arch "$MONITOR_PLATFORM" || die "unsupported MONITOR_PLATFORM=${MONITOR_PLATFORM}"
+    return 0
+  fi
+  die "unable to determine package architecture from manifest or env"
+}
+
+arch_mode() {
+  local host_arch package_arch
+  host_arch="$(detect_host_arch)"
+  package_arch="$(effective_package_arch)"
+  if [[ "$host_arch" == "$package_arch" ]]; then
+    printf 'native\n'
+    return 0
+  fi
+  if env_flag_enabled "${ALLOW_CROSS_ARCH:-0}"; then
+    printf 'cross-arch-enabled\n'
+    return 0
+  fi
+  printf 'mismatch-blocked\n'
+}
+
+assert_arch_compatibility() {
+  local host_arch package_arch package_platform mode
+  host_arch="$(detect_host_arch)"
+  package_arch="$(effective_package_arch)"
+  package_platform="${PACKAGE_PLATFORM:-${RPKI_PLATFORM:-$(platform_for_arch "$package_arch")}}"
+  mode="$(arch_mode)"
+  case "$mode" in
+    native)
+      return 0
+      ;;
+    cross-arch-enabled)
+      log "cross-arch execution explicitly enabled: host_arch=$host_arch package_arch=$package_arch package_platform=$package_platform"
+      return 0
+      ;;
+    mismatch-blocked)
+      die "package architecture mismatch: host_arch=$host_arch package_arch=$package_arch package_platform=$package_platform. Set ALLOW_CROSS_ARCH=1 in $ENV_FILE to allow explicit QEMU/binfmt emulation."
+      ;;
+    *)
+      die "unknown arch compatibility mode: $mode"
+      ;;
+  esac
+}
+
+load_env() {
+  load_manifest
+  [[ -f "$ENV_EXAMPLE" ]] || die "missing $ENV_EXAMPLE"
+  load_env_template_keys "$ENV_EXAMPLE"
+  validate_env_file_contract "$ENV_EXAMPLE"
+  if [[ ! -f "$ENV_FILE" ]]; then
+    cp "$ENV_EXAMPLE" "$ENV_FILE"
+    log "created .env from .env.example"
+  fi
+  validate_env_file_contract "$ENV_FILE"
+  load_env_file "$ENV_FILE"
+
+  if [[ -n "${MANIFEST_PACKAGE_ARCH_RAW:-}" ]]; then
+    manifest_package_arch="$(normalize_arch "$MANIFEST_PACKAGE_ARCH_RAW")" || die "unsupported manifest package arch: $MANIFEST_PACKAGE_ARCH_RAW"
+    if [[ -n "${PACKAGE_ARCH:-}" ]]; then
+      env_package_arch="$(normalize_arch "$PACKAGE_ARCH")" || die "unsupported env package arch: $PACKAGE_ARCH"
+      [[ "$env_package_arch" == "$manifest_package_arch" ]] || die "env PACKAGE_ARCH=$env_package_arch mismatches manifest PACKAGE_ARCH=$manifest_package_arch"
+    fi
+    PACKAGE_ARCH="$manifest_package_arch"
+  else
+    if [[ -z "${PACKAGE_ARCH:-}" ]]; then
+      PACKAGE_ARCH="$(effective_package_arch)"
+    else
+      PACKAGE_ARCH="$(normalize_arch "$PACKAGE_ARCH")"
+    fi
+  fi
+  if [[ -n "${MANIFEST_PACKAGE_PLATFORM_RAW:-}" ]]; then
+    manifest_package_platform="$MANIFEST_PACKAGE_PLATFORM_RAW"
+    if [[ -n "${PACKAGE_PLATFORM:-}" && "$PACKAGE_PLATFORM" != "$manifest_package_platform" ]]; then
+      die "env PACKAGE_PLATFORM=$PACKAGE_PLATFORM mismatches manifest PACKAGE_PLATFORM=$manifest_package_platform"
+    fi
+    PACKAGE_PLATFORM="$manifest_package_platform"
+  else
+    PACKAGE_PLATFORM="${PACKAGE_PLATFORM:-$(platform_for_arch "$PACKAGE_ARCH")}"
+  fi
+
+}
+
+compose_cmd() {
+  docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" -p "$COMPOSE_PROJECT_NAME" "$@"
+}
+
+validate_rtr_report_dir() {
+  load_env
+  [[ -n "${RTR_REPORT_DIR:-}" ]] || return 0
+  [[ "$RTR_REPORT_DIR" = /* ]] || die "RTR_REPORT_DIR must be an absolute host path: $RTR_REPORT_DIR"
+  [[ -d "$RTR_REPORT_DIR" ]] || die "RTR_REPORT_DIR does not exist or is not a directory: $RTR_REPORT_DIR"
+}
+
+validate_rtr_registration_dir() {
+  local report_dir="$1"
+
+  [[ -n "$report_dir" ]] || die "--report-dir is required"
+  [[ "$report_dir" = /* ]] || die "--report-dir must be an absolute host path: $report_dir"
+  [[ "$report_dir" =~ ^/[A-Za-z0-9._/@:+,-]+$ ]] || die "--report-dir contains unsupported characters: $report_dir"
+  [[ -d "$report_dir" ]] || die "--report-dir does not exist: $report_dir"
+  [[ -r "$report_dir" ]] || die "--report-dir is not readable: $report_dir"
+}
+
+rtr_report_file_count() {
+  local report_dir="$1"
+  {
+    find "$report_dir" -maxdepth 1 -type f \( \
+      -name 'rtr-source-*.json' -o \
+      -name 'rtr-runtime-*.json' -o \
+      -name 'rtr-clients-*.json' \
+    \) -print 2>/dev/null || true
+  } | wc -l | tr -d '[:space:]'
+}
+
+latest_rtr_report_file() {
+  local report_dir="$1"
+  {
+    find "$report_dir" -maxdepth 1 -type f \( \
+      -name 'rtr-source-*.json' -o \
+      -name 'rtr-runtime-*.json' -o \
+      -name 'rtr-clients-*.json' \
+    \) -printf '%T@ %p\n' 2>/dev/null || true
+  } | sort -nr | awk 'NR == 1 { sub(/^[^ ]+ /, ""); print }'
+}
+
+replace_env_key_file() {
+  local env_path="$1"
+  local key="$2"
+  local value="$3"
+  local temp_path
+
+  [[ "$key" =~ ^[A-Z][A-Z0-9_]*$ ]] || die "invalid environment key: $key"
+  [[ "$value" != *$'\n'* && "$value" != *$'\r'* ]] || die "environment value must be single-line"
+
+  temp_path="$(mktemp "${env_path}.tmp.XXXXXX")"
+  awk -v key="$key" -v value="$value" '
+    BEGIN { replaced = 0 }
+    $0 ~ "^" key "=" {
+      print key "=" value
+      replaced = 1
+      next
+    }
+    { print }
+    END {
+      if (!replaced) {
+        print key "=" value
+      }
+    }
+  ' "$env_path" > "$temp_path"
+  chmod --reference="$env_path" "$temp_path" 2>/dev/null || true
+  mv "$temp_path" "$env_path"
+}
+
+wait_for_endpoint() {
+  local url="$1"
+  local timeout_secs="$2"
+  local elapsed=0
+
+  while (( elapsed < timeout_secs )); do
+    if endpoint_ok "$url"; then
+      return 0
+    fi
+    sleep 2
+    elapsed=$((elapsed + 2))
+  done
+
+  return 1
+}
+
+create_data_dirs() {
+  load_env
+  mkdir -p \
+    "$HOST_DATA_DIR/state" \
+    "$HOST_DATA_DIR/runs" \
+    "$HOST_DATA_DIR/logs" \
+    "$HOST_DATA_DIR/tmp" \
+    "$HOST_DATA_DIR/empty-rtr-report" \
+    "$HOST_DATA_DIR/prometheus" \
+    "$HOST_DATA_DIR/grafana"
+  validate_rtr_report_dir
+  chmod 755 "$HOST_DATA_DIR" "$HOST_DATA_DIR/state" "$HOST_DATA_DIR/runs" "$HOST_DATA_DIR/logs" "$HOST_DATA_DIR/tmp" || true
+  chmod 777 "$HOST_DATA_DIR/prometheus" "$HOST_DATA_DIR/grafana" || true
+}
+
+latest_run_dir() {
+  load_env
+  find "$HOST_DATA_DIR/runs" -maxdepth 1 -mindepth 1 -type d -name 'run_*' 2>/dev/null | sort | tail -1
+}
+
+latest_success_run_dir() {
+  load_env
+  find "$HOST_DATA_DIR/runs" -maxdepth 2 -type f -path '*/run-summary.json' 2>/dev/null \
+    | while read -r summary; do
+        if jq -e '.status == "success"' "$summary" >/dev/null 2>&1; then
+          dirname "$summary"
+        fi
+      done | sort | tail -1
+}
+
+has_success_run() {
+  [[ -n "$(latest_success_run_dir)" ]]
+}
+
+print_run_summary() {
+  local run_dir="$1"
+  local summary="$run_dir/run-summary.json"
+  local meta="$run_dir/run-meta.json"
+  local timing="$run_dir/stage-timing.json"
+  local process_time="$run_dir/process-time.txt"
+  local vrps_file="$run_dir/vrps.csv"
+  local vaps_file="$run_dir/vaps.csv"
+  local status="unknown"
+  local sync_mode="unknown"
+  local wall_ms="null"
+  local validation_ms="null"
+  local repo_sync_ms="null"
+  local max_rss_kb="null"
+  local publication_points="null"
+  local vrps="null"
+  local vaps="null"
+  local warnings="null"
+  [[ -f "$summary" ]] || {
+    warn "missing run-summary.json in $run_dir"
+    return 1
+  }
+  status="$(jq -r '.status // "unknown"' "$summary" 2>/dev/null || echo unknown)"
+  wall_ms="$(jq -r '.wallMs // .wall_ms // "null"' "$summary" 2>/dev/null || echo null)"
+  warnings="$(jq -r '.warningCount // .warnings // "null"' "$summary" 2>/dev/null || echo null)"
+  if [[ -f "$meta" ]]; then
+    sync_mode="$(jq -r '.sync_mode // .syncMode // "unknown"' "$meta" 2>/dev/null || echo unknown)"
+    status="$(jq -r --arg fallback "$status" '.status // $fallback' "$meta" 2>/dev/null || echo "$status")"
+  fi
+  if [[ -f "$timing" ]]; then
+    validation_ms="$(jq -r '.validation_ms // "null"' "$timing" 2>/dev/null || echo null)"
+    repo_sync_ms="$(jq -r '.repo_sync_ms_total // "null"' "$timing" 2>/dev/null || echo null)"
+    publication_points="$(jq -r '.publication_points // "null"' "$timing" 2>/dev/null || echo null)"
+  fi
+  if [[ -f "$process_time" ]]; then
+    max_rss_kb="$(awk -F': ' '/Maximum resident set size/ {print $2; found=1} END {if (!found) print "null"}' "$process_time")"
+  fi
+  if [[ -f "$vrps_file" ]]; then
+    vrps="$(( $(wc -l < "$vrps_file") > 0 ? $(wc -l < "$vrps_file") - 1 : 0 ))"
+  fi
+  if [[ -f "$vaps_file" ]]; then
+    vaps="$(( $(wc -l < "$vaps_file") > 0 ? $(wc -l < "$vaps_file") - 1 : 0 ))"
+  fi
+  jq -n \
+    --arg run "$(basename "$run_dir")" \
+    --arg status "$status" \
+    --arg syncMode "$sync_mode" \
+    --argjson wallMs "$wall_ms" \
+    --argjson validationMs "$validation_ms" \
+    --argjson repoSyncMs "$repo_sync_ms" \
+    --argjson maxRssKb "$max_rss_kb" \
+    --argjson vrps "$vrps" \
+    --argjson vaps "$vaps" \
+    --argjson publicationPoints "$publication_points" \
+    --argjson warnings "$warnings" \
+    '{run:$run,status:$status,syncMode:$syncMode,wallMs:$wallMs,validationMs:$validationMs,repoSyncMs:$repoSyncMs,maxRssKb:$maxRssKb,vrps:$vrps,vaps:$vaps,publicationPoints:$publicationPoints,warnings:$warnings}'
+}
+
+wait_for_new_success_run() {
+  local before_latest="$1"
+  local timeout_secs="$2"
+  local start_epoch now run_dir summary meta status meta_status
+  start_epoch="$(date +%s)"
+  while true; do
+    run_dir="$(latest_run_dir || true)"
+    if [[ -n "$run_dir" && "$run_dir" != "$before_latest" ]]; then
+      summary="$run_dir/run-summary.json"
+      meta="$run_dir/run-meta.json"
+      if [[ -f "$summary" ]]; then
+        status="$(jq -r '.status // "unknown"' "$summary" 2>/dev/null || echo unknown)"
+        if [[ "$status" == "success" ]]; then
+          meta_status="unknown"
+          if [[ -f "$meta" ]]; then
+            meta_status="$(jq -r '.status // "unknown"' "$meta" 2>/dev/null || echo unknown)"
+          fi
+          if [[ "$meta_status" == "success" ]]; then
+            print_run_summary "$run_dir" || true
+            return 0
+          fi
+        fi
+        if [[ "$status" == "failed" || "$status" == "error" ]]; then
+          print_run_summary "$run_dir" || true
+          die "run failed: $run_dir"
+        fi
+      fi
+    fi
+    now="$(date +%s)"
+    if (( now - start_epoch > timeout_secs )); then
+      die "timed out waiting for first successful run after ${timeout_secs}s"
+    fi
+    sleep 10
+  done
+}
+
+docker_compose_available() {
+  docker compose version >/dev/null 2>&1
+}
+
+install_docker_if_missing() {
+  if command -v docker >/dev/null 2>&1 && docker_compose_available && command -v jq >/dev/null 2>&1 && command -v rsync >/dev/null 2>&1 && command -v curl >/dev/null 2>&1; then
+    log "docker and docker compose are already installed"
+    return 0
+  fi
+  if [[ "${SKIP_DEP_INSTALL:-0}" == "1" ]]; then
+    die "docker/docker compose missing and SKIP_DEP_INSTALL=1"
+  fi
+  if ! command -v apt-get >/dev/null 2>&1; then
+    die "docker/docker compose missing; automatic install currently supports apt-get only"
+  fi
+  log "installing missing runtime packages via apt"
+  apt-get update
+  DEBIAN_FRONTEND=noninteractive apt-get install -y ca-certificates curl jq rsync gzip tar docker.io
+  if ! docker_compose_available; then
+    if apt-cache show docker-compose-v2 >/dev/null 2>&1; then
+      DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose-v2
+    elif apt-cache show docker-compose-plugin >/dev/null 2>&1; then
+      DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose-plugin
+    elif apt-cache show docker-compose >/dev/null 2>&1; then
+      DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose
+    fi
+  fi
+  systemctl enable --now docker >/dev/null 2>&1 || true
+  docker_compose_available || die "docker compose is still unavailable after install"
+}
+
+load_installer_images() {
+  load_env
+  require_cmd docker
+  shopt -s nullglob
+  local image load_output desired_tag
+  local found=0
+  for image in "$INSTALLER_ROOT"/images/*.tar "$INSTALLER_ROOT"/images/*.tar.gz; do
+    found=1
+    log "loading docker image: $image"
+    if [[ "$image" == *.gz ]]; then
+      load_output="$(gzip -dc "$image" | docker load)"
+    else
+      load_output="$(docker load -i "$image")"
+    fi
+    printf '%s\n' "$load_output"
+    desired_tag=""
+    case "$(basename "$image")" in
+      "${image_tar:-}")
+        desired_tag="${image_tag:-$RPKI_IMAGE}"
+        ;;
+      "${metrics_image_tar:-}")
+        desired_tag="${metrics_image:-$METRICS_IMAGE}"
+        ;;
+      "${prometheus_image_tar:-}")
+        desired_tag="${prometheus_image:-$PROMETHEUS_IMAGE}"
+        ;;
+      "${grafana_image_tar:-}")
+        desired_tag="${grafana_image:-$GRAFANA_IMAGE}"
+        ;;
+    esac
+    if [[ -n "$desired_tag" ]]; then
+      ensure_loaded_image_tag "$desired_tag" "$load_output" "$image"
+    fi
+  done
+  shopt -u nullglob
+  (( found == 1 )) || warn "no image tar found under $INSTALLER_ROOT/images"
+}
+
+ensure_loaded_image_tag() {
+  local desired_tag="$1"
+  local load_output="$2"
+  local image_path="$3"
+  local loaded_ref=""
+  local loaded_id=""
+  local desired_id=""
+  loaded_ref="$(printf '%s\n' "$load_output" | awk -F'Loaded image: ' '/Loaded image: / {print $2; exit}')"
+  if [[ -z "$loaded_ref" ]]; then
+    loaded_ref="$(printf '%s\n' "$load_output" | awk -F'Loaded image ID: ' '/Loaded image ID: / {print $2; exit}')"
+  fi
+  [[ -n "$loaded_ref" ]] || die "unable to determine loaded image reference for $image_path"
+  loaded_id="$(docker image inspect --format '{{.Id}}' "$loaded_ref" 2>/dev/null || true)"
+  desired_id="$(docker image inspect --format '{{.Id}}' "$desired_tag" 2>/dev/null || true)"
+  if [[ -n "$loaded_id" && -n "$desired_id" && "$loaded_id" == "$desired_id" ]]; then
+    return 0
+  fi
+  log "tagging loaded image for package use: source=$loaded_ref target=$desired_tag"
+  docker tag "$loaded_ref" "$desired_tag"
+}
+
+ensure_binfmt_if_needed() {
+  require_cmd docker
+  load_env
+  local host_arch package_arch
+  host_arch="$(detect_host_arch)"
+  package_arch="$(effective_package_arch)"
+  if [[ "$host_arch" == "$package_arch" ]]; then
+    return 0
+  fi
+  assert_arch_compatibility
+  log "host arch is $host_arch; enabling binfmt/qemu for package arch $package_arch"
+  docker run --rm --privileged tonistiigi/binfmt --install "$package_arch"
+}
+
+verify_runtime_image() {
+  load_env
+  require_cmd docker
+  log "verifying runtime image $RPKI_IMAGE on $RPKI_PLATFORM"
+  verify_image_platform "$RPKI_IMAGE" "$RPKI_PLATFORM" "runtime"
+  verify_image_usage_output "$RPKI_IMAGE" "$RPKI_PLATFORM" "runtime" /opt/ours-rp/bin/rpki --help
+}
+
+verify_metrics_image() {
+  load_env
+  require_cmd docker
+  log "verifying metrics image $METRICS_IMAGE on $METRICS_PLATFORM"
+  verify_image_platform "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics"
+  verify_image_usage_output "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics" /opt/ours-rp/bin/rpki_artifact_metrics --help
+  verify_metrics_image_rpki_client
+}
+
+verify_metrics_image_rpki_client() {
+  local output
+  local status
+  set +e
+  output="$(docker run --rm --platform "$METRICS_PLATFORM" "$METRICS_IMAGE" /opt/ours-rp/bin/rpki-client -V 2>&1)"
+  status=$?
+  set -e
+  [[ "$status" == "0" ]] || die "metrics image rpki-client -V failed: image=$METRICS_IMAGE exit=$status output=$output"
+  grep -q "rpki-client" <<<"$output" || die "metrics image rpki-client -V unexpected output: $output"
+}
+
+verify_image_platform() {
+  local image="$1"
+  local expected_platform="$2"
+  local role="$3"
+  local actual_platform
+  docker image inspect "$image" >/dev/null
+  actual_platform="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image" 2>/dev/null || echo unknown)"
+  [[ "$actual_platform" == "$expected_platform" ]] || die "$role image platform mismatch: image=$image expected=$expected_platform actual=$actual_platform"
+}
+
+verify_image_usage_output() {
+  local image="$1"
+  local expected_platform="$2"
+  local role="$3"
+  shift 3
+  local help_path
+  local status
+  help_path="$(mktemp "${TMPDIR:-/tmp}/ours-rp-image-help.XXXXXX")"
+  set +e
+  docker run --rm --platform "$expected_platform" "$image" "$@" >"$help_path" 2>&1
+  status=$?
+  set -e
+  if [[ "$status" != "0" && "$status" != "1" ]]; then
+    cat "$help_path" >&2 || true
+    rm -f "$help_path"
+    die "$role image command failed: image=$image platform=$expected_platform exit=$status"
+  fi
+  grep -q '^Usage' "$help_path" || {
+    cat "$help_path" >&2 || true
+    rm -f "$help_path"
+    die "$role image did not emit usage output: image=$image platform=$expected_platform exit=$status"
+  }
+  head -5 "$help_path" || true
+  rm -f "$help_path"
+}
+
+verify_monitor_images() {
+  load_env
+  require_cmd docker
+  verify_image_platform "$PROMETHEUS_IMAGE" "$MONITOR_PLATFORM" "prometheus"
+  verify_image_platform "$GRAFANA_IMAGE" "$MONITOR_PLATFORM" "grafana"
+}
+
+endpoint_ok() {
+  local url="$1"
+  curl -fsS --max-time 5 "$url" >/dev/null 2>&1
+}

deploy/docker-installer/scripts/install.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=common.sh
+source "$SCRIPT_DIR/common.sh"
+
+usage() {
+  cat <<'USAGE'
+Usage: ./scripts/install.sh [--skip-dep-install]
+
+Install or update the ours RP Docker installer package idempotently.
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --skip-dep-install)
+      export SKIP_DEP_INSTALL=1
+      shift
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+load_env
+assert_arch_compatibility
+install_docker_if_missing
+require_cmd curl
+require_cmd jq
+require_cmd rsync
+require_cmd gzip
+require_cmd tar
+create_data_dirs
+load_installer_images
+ensure_binfmt_if_needed
+verify_runtime_image
+verify_metrics_image
+verify_monitor_images
+compose_config_path="$(mktemp "${TMPDIR:-/tmp}/ours-rp-compose-config.XXXXXX.yml")"
+compose_cmd --profile core --profile sidecar --profile monitor config >"$compose_config_path"
+"$SCRIPT_DIR/self-check.sh" --quick
+rm -f "$compose_config_path"
+log "install complete"

deploy/docker-installer/scripts/logs.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=common.sh
+source "$SCRIPT_DIR/common.sh"
+load_env
+compose_cmd --profile core --profile sidecar --profile monitor logs "$@"

deploy/docker-installer/scripts/register_rtr_monitor.sh

@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/common.sh"
+
+REPORT_DIR=""
+DRY_RUN=0
+CANDIDATE_ENV=""
+CONFIG_FILE=""
+ORIGINAL_ENV_FILE=""
+ENV_BACKUP=""
+REGISTRATION_APPLIED=0
+
+usage() {
+  cat <<'EOF'
+Usage: ./scripts/register_rtr_monitor.sh --report-dir <absolute-host-path> [--dry-run]
+
+Registers one local RTR report directory for the artifact metrics sidecar.
+The directory must be on the same host as this installer and may contain
+rtr-source-*.json, rtr-runtime-*.json, and rtr-clients-*.json report files.
+
+This command recreates only artifact-metrics, Prometheus, and Grafana. It does
+not restart or recreate the ours-rp-soak container.
+EOF
+}
+
+cleanup() {
+  [[ -n "$CANDIDATE_ENV" && -f "$CANDIDATE_ENV" ]] && rm -f "$CANDIDATE_ENV"
+  [[ -n "$CONFIG_FILE" && -f "$CONFIG_FILE" ]] && rm -f "$CONFIG_FILE"
+  return 0
+}
+trap cleanup EXIT
+
+restore_failed_registration() {
+  if (( REGISTRATION_APPLIED == 0 )) || [[ -z "$ENV_BACKUP" || ! -f "$ENV_BACKUP" ]]; then
+    return 0
+  fi
+
+  warn "registration failed after .env replacement; restoring previous metrics configuration"
+  cp -p "$ENV_BACKUP" "$ORIGINAL_ENV_FILE"
+  ENV_FILE="$ORIGINAL_ENV_FILE"
+  load_env || warn "unable to reload restored .env"
+  compose_cmd --profile sidecar up -d --force-recreate artifact-metrics \
+    || warn "unable to recreate artifact-metrics with restored configuration"
+}
+
+fail_after_apply() {
+  restore_failed_registration
+  die "$*"
+}
+
+while (( $# > 0 )); do
+  case "$1" in
+    --report-dir)
+      [[ $# -ge 2 ]] || die "--report-dir requires a value"
+      REPORT_DIR="$2"
+      shift 2
+      ;;
+    --dry-run)
+      DRY_RUN=1
+      shift
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      die "unknown argument: $1"
+      ;;
+  esac
+done
+
+require_cmd docker
+require_cmd curl
+require_cmd grep
+require_cmd jq
+validate_rtr_registration_dir "$REPORT_DIR"
+load_env
+assert_arch_compatibility
+
+ORIGINAL_ENV_FILE="$ENV_FILE"
+REPORT_FILE_COUNT="$(rtr_report_file_count "$REPORT_DIR")"
+LATEST_REPORT_FILE="$(latest_rtr_report_file "$REPORT_DIR")"
+if (( REPORT_FILE_COUNT == 0 )); then
+  warn "no recognized RTR report JSON files are currently present in $REPORT_DIR"
+fi
+
+CANDIDATE_ENV="$(mktemp "$INSTALLER_ROOT/.env.rtr-register.XXXXXX")"
+cp -p "$ORIGINAL_ENV_FILE" "$CANDIDATE_ENV"
+replace_env_key_file "$CANDIDATE_ENV" "RTR_REPORT_DIR" "$REPORT_DIR"
+
+ENV_FILE="$CANDIDATE_ENV"
+load_env
+CONFIG_FILE="$(mktemp "$INSTALLER_ROOT/.rtr-compose-config.XXXXXX")"
+compose_cmd --profile sidecar --profile monitor config --format json > "$CONFIG_FILE"
+jq -e --arg source "$REPORT_DIR" --arg target "$RTR_REPORT_CONTAINER_DIR" '
+  .services["artifact-metrics"].volumes
+  | any(.[]; .type == "bind" and .source == $source and .target == $target and .read_only == true)
+' "$CONFIG_FILE" >/dev/null || die "candidate compose configuration is missing expected read-only RTR bind mount"
+
+if (( DRY_RUN == 1 )); then
+  printf 'dry_run=ok\n'
+  printf 'report_dir=%s\n' "$REPORT_DIR"
+  printf 'recognized_report_files=%s\n' "$REPORT_FILE_COUNT"
+  printf 'latest_report=%s\n' "${LATEST_REPORT_FILE:--}"
+  printf 'expected_mount_source=%s\n' "$REPORT_DIR"
+  printf 'expected_mount_target=%s\n' "$RTR_REPORT_CONTAINER_DIR"
+  exit 0
+fi
+
+ENV_FILE="$ORIGINAL_ENV_FILE"
+load_env
+soak_container_before="$(compose_cmd ps -q ours-rp-soak || true)"
+ENV_BACKUP="${ORIGINAL_ENV_FILE}.rtr-register-$(date -u +%Y%m%dT%H%M%SZ).bak"
+cp -p "$ORIGINAL_ENV_FILE" "$ENV_BACKUP"
+mv "$CANDIDATE_ENV" "$ORIGINAL_ENV_FILE"
+CANDIDATE_ENV=""
+REGISTRATION_APPLIED=1
+
+load_env
+compose_cmd --profile sidecar up -d --force-recreate artifact-metrics \
+  || fail_after_apply "unable to recreate artifact-metrics"
+compose_cmd --profile sidecar --profile monitor up -d prometheus grafana \
+  || fail_after_apply "unable to start Prometheus and Grafana"
+
+soak_container_after="$(compose_cmd ps -q ours-rp-soak || true)"
+if [[ -n "$soak_container_before" && "$soak_container_before" != "$soak_container_after" ]]; then
+  fail_after_apply "ours-rp-soak container changed during RTR registration; investigate before continuing"
+fi
+
+wait_for_endpoint "http://127.0.0.1:${METRICS_PORT:-9556}/metrics" 60 \
+  || fail_after_apply "artifact-metrics did not become ready"
+wait_for_endpoint "http://127.0.0.1:${PROMETHEUS_PORT:-9090}/-/ready" 60 \
+  || fail_after_apply "Prometheus did not become ready"
+wait_for_endpoint "http://127.0.0.1:${GRAFANA_PORT:-3000}/api/health" 60 \
+  || fail_after_apply "Grafana did not become ready"
+
+if (( REPORT_FILE_COUNT > 0 )); then
+  curl -fsS "http://127.0.0.1:${METRICS_PORT:-9556}/metrics" | grep -q '^ours_rp_rtr_' \
+    || fail_after_apply "artifact-metrics exposes no RTR metrics after registration"
+fi
+
+printf 'registration=ok\n'
+printf 'report_dir=%s\n' "$REPORT_DIR"
+printf 'recognized_report_files=%s\n' "$REPORT_FILE_COUNT"
+printf 'latest_report=%s\n' "${LATEST_REPORT_FILE:--}"
+printf 'env_backup=%s\n' "$ENV_BACKUP"
+printf 'ours_rp_soak_container_unchanged=%s\n' "${soak_container_before:-not-running}"

deploy/docker-installer/scripts/restart.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+"$SCRIPT_DIR/stop.sh" || true
+"$SCRIPT_DIR/start.sh" "$@"

deploy/docker-installer/scripts/self-check.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=common.sh
+source "$SCRIPT_DIR/common.sh"
+
+QUICK=0
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --quick)
+      QUICK=1
+      shift
+      ;;
+    -h|--help)
+      echo "Usage: ./scripts/self-check.sh [--quick]"
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+load_env
+assert_arch_compatibility
+require_cmd docker
+require_cmd jq
+docker compose version >/dev/null
+[[ -f "$COMPOSE_FILE" ]] || die "missing compose file"
+[[ -f "$ENV_FILE" ]] || die "missing .env"
+create_data_dirs
+[[ -w "$HOST_DATA_DIR" ]] || die "data dir is not writable: $HOST_DATA_DIR"
+compose_cmd --profile core --profile sidecar --profile monitor config >/dev/null
+verify_image_platform "$RPKI_IMAGE" "$RPKI_PLATFORM" "runtime"
+verify_image_platform "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics"
+verify_monitor_images
+if [[ "$QUICK" == "0" ]]; then
+  verify_runtime_image
+  verify_metrics_image
+fi
+log "self-check ok"

deploy/docker-installer/scripts/start.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=common.sh
+source "$SCRIPT_DIR/common.sh"
+
+WAIT_FIRST_RUN=1
+TIMEOUT_SECS=""
+
+usage() {
+  cat <<'USAGE'
+Usage: ./scripts/start.sh [--no-wait-first-run] [--timeout-secs N]
+
+Start ours RP. If no successful run exists, wait for the first snapshot to succeed
+before starting metrics, Prometheus and Grafana.
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --no-wait-first-run)
+      WAIT_FIRST_RUN=0
+      shift
+      ;;
+    --timeout-secs)
+      TIMEOUT_SECS="$2"
+      shift 2
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+load_env
+assert_arch_compatibility
+create_data_dirs
+timeout_secs="${TIMEOUT_SECS:-$FIRST_RUN_WAIT_TIMEOUT_SECS}"
+before_latest="$(latest_run_dir || true)"
+had_success=0
+if has_success_run; then
+  had_success=1
+fi
+
+log "starting core soak service"
+compose_cmd --profile core up -d ours-rp-soak
+
+if [[ "$had_success" == "0" && "$WAIT_FIRST_RUN" == "1" ]]; then
+  log "no previous successful run found; waiting for first run timeout=${timeout_secs}s"
+  wait_for_new_success_run "$before_latest" "$timeout_secs"
+fi
+
+log "starting metrics and monitor services"
+compose_cmd --profile sidecar --profile monitor up -d artifact-metrics prometheus grafana
+"$SCRIPT_DIR/status.sh" --brief || true

deploy/docker-installer/scripts/status.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=common.sh
+source "$SCRIPT_DIR/common.sh"
+
+BRIEF=0
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --brief)
+      BRIEF=1
+      shift
+      ;;
+    -h|--help)
+      echo "Usage: ./scripts/status.sh [--brief]"
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+load_env
+host_arch="$(detect_host_arch)"
+mode="$(arch_mode)"
+echo "installer_root=$INSTALLER_ROOT"
+echo "manifest_file=$MANIFEST_FILE"
+echo "package_name=${package_name:-}"
+echo "source_commit=${source_commit:-${git_commit:-}}"
+echo "source_commit_short=${source_commit_short:-}"
+echo "source_dirty=${source_dirty:-}"
+echo "build_timestamp_utc=${build_timestamp_utc:-${created_at_utc:-}}"
+echo "host_data_dir=$HOST_DATA_DIR"
+echo "host_arch=$host_arch"
+echo "package_arch=$PACKAGE_ARCH"
+echo "package_platform=$PACKAGE_PLATFORM"
+echo "allow_cross_arch=$ALLOW_CROSS_ARCH"
+echo "arch_mode=$mode"
+echo "runtime_image=$RPKI_IMAGE"
+echo "runtime_platform=$RPKI_PLATFORM"
+echo "runtime_image_revision=${runtime_image_revision:-}"
+echo "runtime_image_dirty=${runtime_image_dirty:-}"
+echo "metrics_image=$METRICS_IMAGE"
+echo "metrics_platform=$METRICS_PLATFORM"
+echo "metrics_image_revision=${metrics_image_revision:-}"
+echo "metrics_image_dirty=${metrics_image_dirty:-}"
+echo "rtr_report_dir=$RTR_REPORT_DIR"
+echo "rtr_report_container_dir=$RTR_REPORT_CONTAINER_DIR"
+echo "monitor_platform=$MONITOR_PLATFORM"
+echo "rirs=${RIRS:-}"
+echo "max_runs=${MAX_RUNS:-}"
+echo "interval_secs=${INTERVAL_SECS:-}"
+echo "periodic_snapshot_reset=${PERIODIC_SNAPSHOT_RESET:-0}"
+echo "periodic_snapshot_max_deltas=${PERIODIC_SNAPSHOT_MAX_DELTAS:-100}"
+echo
+if command -v docker >/dev/null 2>&1; then
+  docker version --format 'docker={{.Server.Version}}' 2>/dev/null || echo "docker=unavailable"
+  docker compose version 2>/dev/null || true
+  compose_cmd --profile core --profile sidecar --profile monitor ps || true
+else
+  echo "docker=missing"
+fi
+echo
+df -h "$HOST_DATA_DIR" 2>/dev/null || true
+echo
+latest="$(latest_run_dir || true)"
+if [[ -n "$latest" ]]; then
+  echo "latest_run=$latest"
+  print_run_summary "$latest" || true
+else
+  echo "latest_run=none"
+fi
+if [[ "$BRIEF" == "0" ]]; then
+  echo
+  endpoint_ok "http://127.0.0.1:${METRICS_PORT:-9556}/metrics" && echo "metrics=ok" || echo "metrics=unavailable"
+  endpoint_ok "http://127.0.0.1:${PROMETHEUS_PORT:-9090}/-/ready" && echo "prometheus=ok" || echo "prometheus=unavailable"
+  endpoint_ok "http://127.0.0.1:${GRAFANA_PORT:-3000}/api/health" && echo "grafana=ok" || echo "grafana=unavailable"
+  echo
+  if [[ "$RTR_REPORT_DIR" == "$HOST_DATA_DIR/empty-rtr-report" ]]; then
+    echo "rtr_registration=fallback-empty-directory"
+  else
+    echo "rtr_registration=registered"
+  fi
+  echo "rtr_report_files=$(rtr_report_file_count "$RTR_REPORT_DIR")"
+  echo "rtr_latest_report=$(latest_rtr_report_file "$RTR_REPORT_DIR" || true)"
+fi

deploy/docker-installer/scripts/stop.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=common.sh
+source "$SCRIPT_DIR/common.sh"
+load_env
+compose_cmd --profile core --profile sidecar --profile monitor stop "$@"

deploy/docker-installer/scripts/uninstall.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=common.sh
+source "$SCRIPT_DIR/common.sh"
+
+PURGE_DATA=0
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --purge-data)
+      PURGE_DATA=1
+      shift
+      ;;
+    -h|--help)
+      echo "Usage: ./scripts/uninstall.sh [--purge-data]"
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+load_env
+compose_cmd --profile core --profile sidecar --profile monitor down --remove-orphans || true
+if [[ "$PURGE_DATA" == "1" ]]; then
+  [[ "$HOST_DATA_DIR" == "/" || -z "$HOST_DATA_DIR" ]] && die "refuse to purge unsafe HOST_DATA_DIR=$HOST_DATA_DIR"
+  rm -rf "$HOST_DATA_DIR"
+  log "purged data dir $HOST_DATA_DIR"
+else
+  log "containers removed; data kept at $HOST_DATA_DIR"
+fi

deploy/docker-installer/scripts/upgrade.sh

@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=common.sh
+source "$SCRIPT_DIR/common.sh"
+
+REUSE_ENV_FROM=""
+UPDATE_PACKAGE_IMAGE=1
+
+usage() {
+  cat <<'USAGE'
+Usage: ./scripts/upgrade.sh [--reuse-env-from /path/to/.env] [--keep-reused-image]
+
+By default, --reuse-env-from creates the new .env from this package's
+.env.example first, then overlays existing user settings from the old .env.
+For declared configuration keys, explicit invocation variables have highest
+priority, for example: INTERVAL_SECS=300 ./scripts/upgrade.sh ...
+Image tags are intentionally kept from the new package so the upgraded service
+actually runs the new packaged runtime and monitor images. Use
+--keep-reused-image only when you intentionally want to keep previous image tags.
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --reuse-env-from)
+      REUSE_ENV_FROM="$2"
+      shift 2
+      ;;
+    --keep-reused-image)
+      UPDATE_PACKAGE_IMAGE=0
+      shift
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+overlay_reused_env() {
+  local source_env="$1"
+  local target_env="$2"
+  local key
+  local value
+  local -a reused_keys=()
+  local -a ignored_removed_keys=()
+  local -a preserved_package_keys=()
+
+  REUSED_ENV_KEYS=()
+  parse_env_file "$source_env" || die "unable to parse reused environment: $source_env"
+  while IFS= read -r key; do
+    if ! template_has_env_key "$key"; then
+      ignored_removed_keys+=("$key")
+      continue
+    fi
+    case "$key" in
+      PACKAGE_ARCH|PACKAGE_PLATFORM|RPKI_PLATFORM|METRICS_PLATFORM|MONITOR_PLATFORM)
+        preserved_package_keys+=("$key")
+        continue
+        ;;
+      RPKI_IMAGE|METRICS_IMAGE|PROMETHEUS_IMAGE|GRAFANA_IMAGE)
+        if [[ "$UPDATE_PACKAGE_IMAGE" != "0" ]]; then
+          preserved_package_keys+=("$key")
+          continue
+        fi
+        ;;
+    esac
+    [[ -n "${ENV_PARSED_VALUES[$key]+x}" ]] || die "unable to read $key from reused environment: $source_env"
+    value="${ENV_PARSED_VALUES[$key]}"
+    env_file_set_value "$target_env" "$key" "$value"
+    REUSED_ENV_KEYS["$key"]=1
+    reused_keys+=("$key")
+  done < <(env_file_assignment_keys "$source_env")
+
+  log_env_key_group "reused_keys" "${reused_keys[@]}"
+  log_env_key_group "ignored_removed_old_keys" "${ignored_removed_keys[@]}"
+  log_env_key_group "preserved_package_keys" "${preserved_package_keys[@]}"
+}
+
+apply_invocation_env_overrides() {
+  local target_env="$1"
+  local key
+  local -a overridden_keys=()
+
+  for key in "${ENV_TEMPLATE_KEYS[@]}"; do
+    [[ -n "${INVOCATION_ENV_OVERRIDES[$key]+x}" ]] || continue
+    env_file_set_value "$target_env" "$key" "${INVOCATION_ENV_OVERRIDES[$key]}"
+    overridden_keys+=("$key")
+  done
+
+  log_env_key_group "command_override_keys" "${overridden_keys[@]}"
+}
+
+log_template_only_keys() {
+  local key
+  local -a template_only_keys=()
+
+  for key in "${ENV_TEMPLATE_KEYS[@]}"; do
+    [[ -n "${REUSED_ENV_KEYS[$key]+x}" ]] && continue
+    [[ -n "${INVOCATION_ENV_OVERRIDES[$key]+x}" ]] && continue
+    template_only_keys+=("$key")
+  done
+  log_env_key_group "template_only_keys" "${template_only_keys[@]}"
+}
+
+load_env_template_keys "$ENV_EXAMPLE"
+validate_env_file_contract "$ENV_EXAMPLE"
+capture_invocation_env_overrides
+REUSED_ENV_KEYS=()
+
+if [[ -n "$REUSE_ENV_FROM" ]]; then
+  [[ -f "$REUSE_ENV_FROM" ]] || die "missing reuse env file: $REUSE_ENV_FROM"
+  if [[ ! -f "$ENV_FILE" ]]; then
+    cp "$ENV_EXAMPLE" "$ENV_FILE"
+    overlay_reused_env "$REUSE_ENV_FROM" "$ENV_FILE"
+    log "created new package env from .env.example and overlaid user settings from $REUSE_ENV_FROM"
+  else
+    log "keeping existing env at $ENV_FILE; reuse source ignored: $REUSE_ENV_FROM"
+  fi
+fi
+
+if [[ ! -f "$ENV_FILE" ]]; then
+  cp "$ENV_EXAMPLE" "$ENV_FILE"
+  log "created new package env from .env.example"
+fi
+apply_invocation_env_overrides "$ENV_FILE"
+log_template_only_keys
+validate_env_file_contract "$ENV_FILE"
+load_env
+assert_arch_compatibility
+create_data_dirs
+install_docker_if_missing
+load_installer_images
+ensure_binfmt_if_needed
+verify_runtime_image
+verify_metrics_image
+verify_monitor_images
+compose_cmd --profile core --profile sidecar --profile monitor up -d --force-recreate
+"$SCRIPT_DIR/status.sh" --brief || true

deploy/docker-installer/tests/upgrade_env_contract_test.sh

@@ -0,0 +1,206 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+INSTALLER_SOURCE="$(cd "$SCRIPT_DIR/.." && pwd)"
+TEST_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/ours-rp-installer-env-contract.XXXXXX")"
+
+cleanup() {
+  rm -rf "$TEST_ROOT"
+}
+trap cleanup EXIT
+
+fail() {
+  printf 'FAIL: %s\n' "$*" >&2
+  exit 1
+}
+
+assert_equals() {
+  local expected="$1"
+  local actual="$2"
+  local message="$3"
+  [[ "$actual" == "$expected" ]] || fail "$message (expected=$expected actual=$actual)"
+}
+
+assert_not_contains() {
+  local needle="$1"
+  local path="$2"
+  ! grep -Fq "$needle" "$path" || fail "unexpected value in $path: $needle"
+}
+
+expect_exit() {
+  local expected_status="$1"
+  local output_path="$2"
+  shift 2
+  local actual_status
+
+  set +e
+  "$@" >"$output_path" 2>&1
+  actual_status=$?
+  set -e
+  [[ "$actual_status" == "$expected_status" ]] || {
+    cat "$output_path" >&2 || true
+    fail "unexpected exit status (expected=$expected_status actual=$actual_status)"
+  }
+}
+
+host_arch() {
+  case "$(uname -m)" in
+    x86_64) printf 'amd64\n' ;;
+    aarch64) printf 'arm64\n' ;;
+    *) fail "unsupported test host architecture: $(uname -m)" ;;
+  esac
+}
+
+env_value() {
+  local env_path="$1"
+  local key="$2"
+  (
+    # shellcheck disable=SC1090
+    source "$INSTALLER_SOURCE/scripts/common.sh"
+    env_file_value "$env_path" "$key"
+  )
+}
+
+create_fixture() {
+  local name="$1"
+  local fixture="$TEST_ROOT/$name"
+  local arch
+  arch="$(host_arch)"
+
+  mkdir -p "$fixture"
+  cp -a "$INSTALLER_SOURCE"/. "$fixture"/
+  rm -rf "$fixture/tests"
+  mkdir -p "$fixture/images" "$fixture/fake-bin"
+  sed -i \
+    -e "s|__PACKAGE_ARCH__|$arch|g" \
+    -e "s|__PACKAGE_PLATFORM__|linux/$arch|g" \
+    -e "s|__RUNTIME_IMAGE__|ours-rp-runtime-$arch:new|g" \
+    -e "s|__METRICS_IMAGE__|ours-rp-metrics-$arch:new|g" \
+    -e "s|__HOST_DATA_DIR__|$fixture/data|g" \
+    "$fixture/.env.example"
+  cat > "$fixture/PACKAGE-MANIFEST.env" <<EOF
+PACKAGE_ARCH=$arch
+PACKAGE_PLATFORM=linux/$arch
+EOF
+  cat > "$fixture/fake-bin/docker" <<'EOF'
+#!/usr/bin/env bash
+printf '%s\n' "$*" >> "$FAKE_DOCKER_LOG"
+if [[ "${1:-}" == "compose" && "${2:-}" == "version" ]]; then
+  exit 0
+fi
+exit 99
+EOF
+  chmod +x "$fixture/fake-bin/docker"
+  for command in jq rsync curl gzip tar; do
+    cat > "$fixture/fake-bin/$command" <<'EOF'
+#!/usr/bin/env bash
+exit 0
+EOF
+    chmod +x "$fixture/fake-bin/$command"
+  done
+  printf '%s\n' "$fixture"
+}
+
+run_upgrade() {
+  local fixture="$1"
+  local old_env="$2"
+  shift 2
+  (
+    cd "$fixture"
+    export PATH="$fixture/fake-bin:$PATH"
+    export FAKE_DOCKER_LOG="$fixture/fake-docker.log"
+    "$@" ./scripts/upgrade.sh --reuse-env-from "$old_env"
+  )
+}
+
+run_upgrade_without_reuse() {
+  local fixture="$1"
+  (
+    cd "$fixture"
+    export PATH="$fixture/fake-bin:$PATH"
+    export FAKE_DOCKER_LOG="$fixture/fake-docker.log"
+    ./scripts/upgrade.sh
+  )
+}
+
+new_fixture_with_old_env() {
+  local name="$1"
+  local fixture
+  fixture="$(create_fixture "$name")"
+  cat > "$fixture/old.env" <<'EOF'
+INTERVAL_SECS=120
+RPKI_IMAGE=ours-rp-runtime-old:old
+REMOVED_LEGACY_KEY=legacy-value
+EOF
+  printf '%s\n' "$fixture"
+}
+
+fixture="$(create_fixture template-defaults)"
+expect_exit 99 "$fixture/template.log" run_upgrade_without_reuse "$fixture"
+assert_equals "600" "$(env_value "$fixture/.env" INTERVAL_SECS)" "template default must be retained without reuse"
+
+fixture="$(new_fixture_with_old_env default-old)"
+expect_exit 99 "$fixture/default.log" run_upgrade "$fixture" "$fixture/old.env"
+assert_equals "120" "$(env_value "$fixture/.env" INTERVAL_SECS)" "old value must override template default"
+assert_equals "ours-rp-runtime-$(host_arch):new" "$(env_value "$fixture/.env" RPKI_IMAGE)" "standard upgrade must retain package image"
+assert_equals "10" "$(env_value "$fixture/.env" METRICS_POLL_SECS)" "template default must fill a key absent from old environment"
+! grep -q '^REMOVED_LEGACY_KEY=' "$fixture/.env" || fail "removed old key was copied into target environment"
+grep -q 'reused_keys=INTERVAL_SECS' "$fixture/default.log" || fail "reused key summary missing"
+grep -q 'ignored_removed_old_keys=REMOVED_LEGACY_KEY' "$fixture/default.log" || fail "removed key summary missing"
+grep -q 'template_only_keys=' "$fixture/default.log" || fail "template key summary missing"
+
+fixture="$(new_fixture_with_old_env invocation-overrides)"
+expect_exit 99 "$fixture/override.log" run_upgrade "$fixture" "$fixture/old.env" env INTERVAL_SECS=300 GRAFANA_ADMIN_PASSWORD='secret with space'
+assert_equals "300" "$(env_value "$fixture/.env" INTERVAL_SECS)" "invocation value must override old value"
+assert_equals "secret with space" "$(env_value "$fixture/.env" GRAFANA_ADMIN_PASSWORD)" "invocation value must preserve shell-sensitive content"
+grep -q 'command_override_keys=INTERVAL_SECS,GRAFANA_ADMIN_PASSWORD' "$fixture/override.log" || fail "override key summary missing"
+assert_not_contains 'secret with space' "$fixture/override.log"
+
+fixture="$(new_fixture_with_old_env invocation-special-characters)"
+special_password="O'Reilly"
+special_password+=' $literal `tick` slash\'
+expect_exit 99 "$fixture/special.log" run_upgrade "$fixture" "$fixture/old.env" env GRAFANA_ADMIN_PASSWORD="$special_password"
+assert_equals "$special_password" "$(env_value "$fixture/.env" GRAFANA_ADMIN_PASSWORD)" "invocation value with dotenv-sensitive characters must round-trip"
+assert_not_contains "$special_password" "$fixture/special.log"
+
+fixture="$(create_fixture existing-target-env)"
+cp "$fixture/.env.example" "$fixture/.env"
+sed -i 's/^INTERVAL_SECS=.*/INTERVAL_SECS=180/' "$fixture/.env"
+cat > "$fixture/old.env" <<'EOF'
+INTERVAL_SECS=120
+EOF
+expect_exit 99 "$fixture/existing.log" run_upgrade "$fixture" "$fixture/old.env" env INTERVAL_SECS=300
+assert_equals "300" "$(env_value "$fixture/.env" INTERVAL_SECS)" "invocation value must override an existing target environment"
+
+fixture="$(new_fixture_with_old_env explicit-empty)"
+expect_exit 1 "$fixture/empty.log" run_upgrade "$fixture" "$fixture/old.env" env INTERVAL_SECS=
+[[ ! -s "$fixture/fake-docker.log" ]] || fail "explicit empty value reached Docker"
+grep -q 'INTERVAL_SECS' "$fixture/empty.log" || fail "empty-value error did not identify key"
+
+fixture="$(create_fixture empty-old-value)"
+cat > "$fixture/old.env" <<'EOF'
+INTERVAL_SECS=
+EOF
+expect_exit 1 "$fixture/old-empty.log" run_upgrade "$fixture" "$fixture/old.env"
+[[ ! -s "$fixture/fake-docker.log" ]] || fail "empty reused value reached Docker"
+grep -q 'INTERVAL_SECS' "$fixture/old-empty.log" || fail "empty reused value error did not identify key"
+
+fixture="$(create_fixture empty-template)"
+sed -i 's/^INTERVAL_SECS=.*/INTERVAL_SECS=/' "$fixture/.env.example"
+cat > "$fixture/old.env" <<'EOF'
+INTERVAL_SECS=120
+EOF
+expect_exit 1 "$fixture/template-empty.log" run_upgrade "$fixture" "$fixture/old.env"
+[[ ! -s "$fixture/fake-docker.log" ]] || fail "empty template default reached Docker"
+grep -q 'INTERVAL_SECS' "$fixture/template-empty.log" || fail "empty template error did not identify key"
+
+fixture="$(new_fixture_with_old_env package-guard)"
+arch="$(host_arch)"
+other_arch="amd64"
+[[ "$arch" == "amd64" ]] && other_arch="arm64"
+expect_exit 1 "$fixture/arch.log" run_upgrade "$fixture" "$fixture/old.env" env PACKAGE_ARCH="$other_arch"
+[[ ! -s "$fixture/fake-docker.log" ]] || fail "package architecture mismatch reached Docker"
+grep -q 'mismatches manifest PACKAGE_ARCH' "$fixture/arch.log" || fail "package architecture guard did not reject override"
+
+printf 'PASS: Docker installer upgrade environment contract\n'

deploy/rpki-explorer/.env.example

@@ -0,0 +1,22 @@
+# rpki-explorer standalone stack configuration.
+# Copy to .env and adjust paths for the target host.
+
+# Soak outputs on the host (mounted read-only).
+SOAK_RUNS_DIR=/root/rpki096_x86_rtr_installer/data/runs
+SOAK_DB_DIR=/root/rpki096_x86_rtr_installer/data/state/db
+
+# Host directory where the query-db index is persisted (reused across restarts).
+QUERY_DB_DIR=/root/rpki128_explorer/query-db
+
+# Host loopback port the explorer is published on (SSH port-forward access).
+LISTEN_PORT=9517
+
+# Keep only the latest N indexed runs; first boot backfills the latest N runs.
+RETAIN_RUNS=10
+
+# Watch poll interval for newly completed runs.
+WATCH_INTERVAL_SECS=10
+
+# Image coordinates (built locally, transferred via docker save/load).
+IMAGE_NAME=rpki-explorer
+IMAGE_TAG=local

deploy/rpki-explorer/Dockerfile

@@ -0,0 +1,68 @@
+# rpki-explorer all-in-one image:
+#   - rpki_query_service (watch mode) + rpki_query_indexer
+#   - nginx serving the explorer SPA and proxying /api/v1/ to the query service
+#
+# Build context must be the repository root (rpki_2/rpki):
+#   docker build -f deploy/rpki-explorer/Dockerfile -t rpki-explorer:<tag> .
+#
+# Note: no `# syntax=` directive on purpose — the default images below are all
+# expected to be present locally, and pulling the dockerfile frontend from
+# docker.io may not be possible on offline build hosts.
+
+ARG RUST_BUILDER_IMAGE=rust:1-bookworm
+ARG NODE_BUILDER_IMAGE=node:slim
+ARG RUNTIME_IMAGE=nginx:1.27-bookworm
+
+FROM ${RUST_BUILDER_IMAGE} AS rust-builder
+
+WORKDIR /src
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+      build-essential \
+      ca-certificates \
+      clang \
+      cmake \
+      git \
+      libclang-dev \
+      make \
+      perl \
+      pkg-config \
+      python3 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY . .
+
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    --mount=type=cache,target=/src/target \
+    cargo build --release --bin rpki_query_service --bin rpki_query_indexer \
+    && cp target/release/rpki_query_service target/release/rpki_query_indexer /usr/local/bin/
+
+FROM ${NODE_BUILDER_IMAGE} AS ui-builder
+
+WORKDIR /ui
+
+COPY ui/rpki-explorer/package.json ui/rpki-explorer/package-lock.json ./
+RUN npm ci --no-audit --no-fund
+
+COPY ui/rpki-explorer/ ./
+RUN npm run build
+
+FROM ${RUNTIME_IMAGE}
+
+ARG GIT_REV=unknown
+LABEL org.opencontainers.image.title="rpki-explorer" \
+      org.opencontainers.image.revision="${GIT_REV}"
+
+COPY --from=rust-builder /usr/local/bin/rpki_query_service /usr/local/bin/rpki_query_service
+COPY --from=rust-builder /usr/local/bin/rpki_query_indexer /usr/local/bin/rpki_query_indexer
+COPY --from=ui-builder /ui/dist /usr/share/nginx/html
+COPY deploy/rpki-explorer/nginx.conf /etc/nginx/conf.d/default.conf
+COPY deploy/rpki-explorer/entrypoint.sh /entrypoint.sh
+
+RUN chmod +x /entrypoint.sh && mkdir -p /data
+
+EXPOSE 8080
+
+ENTRYPOINT ["/entrypoint.sh"]

deploy/rpki-explorer/README.md

@@ -0,0 +1,76 @@
+# rpki-explorer standalone stack
+
+One container running the RPKI Explorer web UI plus the query backend
+(`rpki_query_service` in watch mode, spawning `rpki_query_indexer`), deployed
+next to a running ours-RP soak. It mounts the soak outputs **read-only** and
+keeps only the latest `RETAIN_RUNS` indexed runs.
+
+```text
+browser ──ssh -L──► 127.0.0.1:9517 ──► container
+                                        ├─ nginx :8080  (dist + /api/v1/ proxy)
+                                        └─ rpki_query_service :9557
+                                             ├─ /data/query-db        (host bind mount, rw, reused on restart)
+                                             ├─ /soak/db/repo-bytes.db (ro)
+                                             └─ /soak/runs             (ro, watched)
+```
+
+## Prerequisites
+
+- Docker with the compose plugin (`docker compose version`) on the target host.
+- A running soak whose `runs/` directories contain `report.json` (and
+  `run-summary.json` once finished) — only completed runs get indexed.
+
+## Build (local machine, same arch as the target)
+
+```bash
+deploy/rpki-explorer/build_image.sh
+# tarball + provenance land in target/rpki-explorer-image/
+```
+
+The build runs entirely inside Docker (rust release binaries + vite build),
+so no local toolchain version requirements apply.
+
+## Transfer and deploy (remote host)
+
+```bash
+cat target/rpki-explorer-image/rpki-explorer-<tag>.tar.gz | ssh root@<host> 'gunzip | docker load'
+
+ssh root@<host>
+mkdir -p /root/rpki128_explorer
+cd /root/rpki128_explorer
+# place docker-compose.yml and .env here (see .env.example)
+docker compose up -d
+docker logs -f rpki-explorer
+```
+
+On first boot the entrypoint computes
+`--watch-min-run-seq = latest_completed - RETAIN_RUNS + 1` so exactly the
+latest `RETAIN_RUNS` completed runs are backfilled. On later boots the
+existing query-db drives continuation — the index built earlier is reused, no
+re-backfill happens.
+
+## Access
+
+```bash
+ssh -L 9517:127.0.0.1:9517 root@<host>
+# open http://127.0.0.1:9517
+```
+
+The stack binds loopback only; there is no authentication or TLS.
+
+## Upgrade
+
+Rebuild the image with a new tag, transfer and `docker load`, update
+`IMAGE_TAG` in `.env`, then `docker compose up -d`. The query-db on the host
+is preserved.
+
+## Rollback / removal
+
+```bash
+docker compose down
+# keep /root/rpki128_explorer/query-db to reuse the index later, or delete it
+# to force a fresh backfill of the latest RETAIN_RUNS runs.
+```
+
+The soak stack is never touched; removing the explorer stack has no effect on
+it.

deploy/rpki-explorer/build_image.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Build the rpki-explorer all-in-one image locally and export it as a tarball
+# ready to transfer to a remote host (docker load on the other side).
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+OUT_DIR="${OUT_DIR:-$REPO_ROOT/target/rpki-explorer-image}"
+IMAGE_NAME="${IMAGE_NAME:-rpki-explorer}"
+GIT_REV="$(git -C "$REPO_ROOT" rev-parse --short HEAD)"
+IMAGE_TAG="${IMAGE_TAG:-$GIT_REV}"
+
+dirty_count="$(git -C "$REPO_ROOT" status --porcelain | wc -l)"
+
+echo "[build] repo=$REPO_ROOT rev=$GIT_REV dirty_files=$dirty_count tag=$IMAGE_TAG"
+
+docker build \
+  --file "$REPO_ROOT/deploy/rpki-explorer/Dockerfile" \
+  --build-arg "GIT_REV=$GIT_REV" \
+  --tag "$IMAGE_NAME:$IMAGE_TAG" \
+  "$REPO_ROOT"
+
+mkdir -p "$OUT_DIR"
+TARBALL="$OUT_DIR/rpki-explorer-$IMAGE_TAG.tar.gz"
+docker save "$IMAGE_NAME:$IMAGE_TAG" | gzip > "$TARBALL"
+
+IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$IMAGE_NAME:$IMAGE_TAG")"
+cat > "$OUT_DIR/rpki-explorer-$IMAGE_TAG.provenance.txt" <<EOF
+git_rev=$GIT_REV
+dirty_files=$dirty_count
+build_time_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+image=$IMAGE_NAME:$IMAGE_TAG
+image_id=$IMAGE_ID
+EOF
+
+echo "[build] image tarball: $TARBALL"
+echo "[build] provenance:    $OUT_DIR/rpki-explorer-$IMAGE_TAG.provenance.txt"
+echo "[build] transfer:      cat '$TARBALL' | ssh root@<host> 'gunzip | docker load'"

deploy/rpki-explorer/docker-compose.yml

@@ -0,0 +1,28 @@
+name: rpki-explorer
+
+# Standalone explorer stack: one container with nginx + rpki_query_service
+# (+ rpki_query_indexer spawned by the service watcher). It mounts the soak
+# run root and state db read-only and persists its query-db on the host so
+# container/host restarts reuse the already-built index.
+
+services:
+  rpki-explorer:
+    image: ${IMAGE_NAME:-rpki-explorer}:${IMAGE_TAG:-local}
+    build:
+      context: ../..
+      dockerfile: deploy/rpki-explorer/Dockerfile
+      args:
+        GIT_REV: ${GIT_REV:-unknown}
+    container_name: rpki-explorer
+    restart: unless-stopped
+    ports:
+      # Loopback only: access via SSH port forwarding, e.g.
+      #   ssh -L 9517:127.0.0.1:9517 root@<host>
+      - "127.0.0.1:${LISTEN_PORT:-9517}:8080"
+    environment:
+      RETAIN_RUNS: "${RETAIN_RUNS:-10}"
+      WATCH_INTERVAL_SECS: "${WATCH_INTERVAL_SECS:-10}"
+    volumes:
+      - "${SOAK_RUNS_DIR:?set SOAK_RUNS_DIR in .env}:/soak/runs:ro"
+      - "${SOAK_DB_DIR:?set SOAK_DB_DIR in .env}:/soak/db:ro"
+      - "${QUERY_DB_DIR:-./query-db}:/data"

deploy/rpki-explorer/entrypoint.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+# Container entrypoint: run rpki_query_service (watch mode) and nginx in one
+# container. nginx serves the explorer SPA and proxies /api/v1/ to the query
+# service. If either process exits, the container exits and the compose
+# restart policy brings it back.
+set -euo pipefail
+
+RUNS_DIR="${RUNS_DIR:-/soak/runs}"
+REPO_BYTES_DB="${REPO_BYTES_DB:-/soak/db/repo-bytes.db}"
+QUERY_DB="${QUERY_DB:-/data/query-db}"
+RETAIN_RUNS="${RETAIN_RUNS:-10}"
+WATCH_INTERVAL_SECS="${WATCH_INTERVAL_SECS:-10}"
+QUERY_LISTEN="${QUERY_LISTEN:-127.0.0.1:9557}"
+
+log() { echo "[entrypoint] $*"; }
+
+latest_completed_seq() {
+  local latest=0 dir base n
+  shopt -s nullglob
+  for dir in "$RUNS_DIR"/run_*/; do
+    [ -f "${dir}report.json" ] || continue
+    base="${dir%/}"
+    n="${base##*/run_}"
+    case "$n" in
+      *[!0-9]* | "") continue ;;
+    esac
+    if [ "$n" -gt "$latest" ]; then
+      latest="$n"
+    fi
+  done
+  echo "$latest"
+}
+
+mkdir -p "$(dirname "$QUERY_DB")"
+
+ARGS=(
+  --query-db "$QUERY_DB"
+  --listen "$QUERY_LISTEN"
+  --watch-run-root "$RUNS_DIR"
+  --watch-interval-secs "$WATCH_INTERVAL_SECS"
+  --retain-indexed-runs "$RETAIN_RUNS"
+)
+
+if [ -e "$REPO_BYTES_DB" ]; then
+  ARGS+=(--repo-bytes-db "$REPO_BYTES_DB")
+else
+  log "WARNING: repo-bytes db not found at $REPO_BYTES_DB; parsed/raw/export features will be unavailable"
+fi
+
+# Decide the watcher start point:
+# - explicit WATCH_MIN_RUN_SEQ always wins;
+# - on first boot (query-db does not exist yet) backfill the latest
+#   RETAIN_RUNS completed runs;
+# - on later boots the existing query-db drives continuation
+#   (max(min_run_seq, latest_ready+1)), so no flag is needed.
+if [ -n "${WATCH_MIN_RUN_SEQ:-}" ]; then
+  ARGS+=(--watch-min-run-seq "$WATCH_MIN_RUN_SEQ")
+  log "using explicit WATCH_MIN_RUN_SEQ=$WATCH_MIN_RUN_SEQ"
+elif [ ! -e "$QUERY_DB" ]; then
+  latest="$(latest_completed_seq)"
+  if [ "$latest" -gt "$RETAIN_RUNS" ]; then
+    min_seq=$((latest - RETAIN_RUNS + 1))
+  else
+    min_seq=1
+  fi
+  log "first boot: latest completed run seq=$latest -> --watch-min-run-seq $min_seq (backfill $RETAIN_RUNS runs)"
+  ARGS+=(--watch-min-run-seq "$min_seq")
+else
+  log "existing query-db at $QUERY_DB; resuming from indexed state"
+fi
+
+log "starting rpki_query_service ${ARGS[*]}"
+rpki_query_service "${ARGS[@]}" &
+QS_PID=$!
+
+log "starting nginx"
+nginx -g 'daemon off;' &
+NGINX_PID=$!
+
+shutdown() {
+  log "shutting down (qs=$QS_PID nginx=$NGINX_PID)"
+  kill -TERM "$QS_PID" 2>/dev/null || true
+  kill -TERM "$NGINX_PID" 2>/dev/null || true
+}
+trap shutdown TERM INT
+
+wait -n "$QS_PID" "$NGINX_PID"
+EXIT_CODE=$?
+log "a process exited (code=$EXIT_CODE); stopping the remaining one"
+shutdown
+wait "$QS_PID" "$NGINX_PID" 2>/dev/null || true
+exit "$EXIT_CODE"

deploy/rpki-explorer/nginx.conf

@@ -0,0 +1,27 @@
+server {
+    listen 8080;
+    server_name _;
+
+    access_log /dev/stdout;
+    error_log /dev/stderr;
+
+    root /usr/share/nginx/html;
+    index index.html;
+
+    # query service has no CORS headers and no authentication: the API must be
+    # served same-origin with the UI.
+    location /api/v1/ {
+        proxy_pass http://127.0.0.1:9557;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_read_timeout 300s;
+        proxy_send_timeout 300s;
+    }
+
+    # SPA fallback: routing is client-side.
+    location / {
+        try_files $uri $uri/ /index.html;
+    }
+}

docker/ours-rp-metrics.Dockerfile

@@ -0,0 +1,156 @@
+ARG BUILDER_IMAGE=rust:1-bookworm
+ARG RUNTIME_IMAGE=debian:bookworm-slim
+
+FROM --platform=$BUILDPLATFORM ${BUILDER_IMAGE} AS builder
+
+ARG BUILDARCH
+ARG TARGETARCH
+ARG TARGETPLATFORM
+
+WORKDIR /src
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+      build-essential \
+      ca-certificates \
+      clang \
+      cmake \
+      git \
+      libclang-dev \
+      make \
+      perl \
+      pkg-config \
+      python3 \
+    && if [ "$TARGETARCH" != "$BUILDARCH" ]; then \
+         case "$TARGETARCH" in \
+           arm64) \
+             apt-get install -y --no-install-recommends \
+               g++-aarch64-linux-gnu \
+               gcc-aarch64-linux-gnu \
+               libc6-dev-arm64-cross \
+             ;; \
+           amd64) \
+             apt-get install -y --no-install-recommends \
+               g++-x86-64-linux-gnu \
+               gcc-x86-64-linux-gnu \
+               libc6-dev-amd64-cross \
+             ;; \
+           *) \
+             echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 \
+             ;; \
+         esac; \
+       fi \
+    && rm -rf /var/lib/apt/lists/* \
+    && case "$TARGETARCH" in \
+         amd64|arm64) ;; \
+         *) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 ;; \
+       esac
+
+COPY . .
+
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    --mount=type=cache,target=/src/target \
+    set -eux; \
+    case "$TARGETARCH" in \
+      amd64) target_triple=x86_64-unknown-linux-gnu ;; \
+      arm64) target_triple=aarch64-unknown-linux-gnu ;; \
+      *) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 ;; \
+    esac; \
+    rustup target add "$target_triple"; \
+    if [ "$TARGETARCH" != "$BUILDARCH" ]; then \
+      case "$TARGETARCH" in \
+        arm64) \
+          export \
+            CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \
+            CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc \
+            CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ \
+            AR_aarch64_unknown_linux_gnu=aarch64-linux-gnu-ar \
+            PKG_CONFIG_ALLOW_CROSS=1 \
+          ;; \
+        amd64) \
+          export \
+            CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc \
+            CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc \
+            CXX_x86_64_unknown_linux_gnu=x86_64-linux-gnu-g++ \
+            AR_x86_64_unknown_linux_gnu=x86_64-linux-gnu-ar \
+            PKG_CONFIG_ALLOW_CROSS=1 \
+          ;; \
+        *) \
+          echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 \
+          ;; \
+      esac; \
+    fi; \
+    cargo build --release --target "$target_triple" --bin rpki_artifact_metrics; \
+    mkdir -p /build-out/bin; \
+    cp "target/$target_triple/release/rpki_artifact_metrics" /build-out/bin/
+
+FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS rpki-client-builder
+
+ARG RPKI_CLIENT_VERSION=9.8
+ARG RPKI_CLIENT_SHA256=42920aac5afd0996173fb9f7848691a2c49dd234e4f10478808c1aa475620861
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+      build-essential \
+      ca-certificates \
+      curl \
+      libexpat1-dev \
+      libssl-dev \
+      libtls-dev \
+      pkg-config \
+      rsync \
+      zlib1g-dev \
+    && rm -rf /var/lib/apt/lists/* \
+    && useradd --system --no-create-home --shell /usr/sbin/nologin _rpki-client
+
+RUN set -eux; \
+    curl -fsSL "https://ftp.openbsd.org/pub/OpenBSD/rpki-client/rpki-client-${RPKI_CLIENT_VERSION}.tar.gz" -o /tmp/rpki-client.tar.gz; \
+    echo "${RPKI_CLIENT_SHA256}  /tmp/rpki-client.tar.gz" | sha256sum -c -; \
+    tar -xzf /tmp/rpki-client.tar.gz -C /tmp; \
+    cd "/tmp/rpki-client-${RPKI_CLIENT_VERSION}"; \
+    ./configure --prefix=/opt/rpki-client; \
+    make -j"$(nproc)"; \
+    make install; \
+    /opt/rpki-client/sbin/rpki-client -V
+
+FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS runtime
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+      ca-certificates \
+      libexpat1 \
+      libssl3 \
+      libtls26 \
+      tzdata \
+      zlib1g \
+    && rm -rf /var/lib/apt/lists/* \
+    && useradd --system --no-create-home --shell /usr/sbin/nologin _rpki-client \
+    && mkdir -p /opt/rpki-client/var/cache/rpki-client
+
+WORKDIR /opt/ours-rp
+
+COPY --from=builder /build-out/bin/ /opt/ours-rp/bin/
+COPY --from=rpki-client-builder /opt/rpki-client/ /opt/rpki-client/
+COPY --from=rpki-client-builder /opt/rpki-client/sbin/rpki-client /opt/ours-rp/bin/rpki-client
+
+ARG SOURCE_COMMIT=unknown
+ARG SOURCE_DIRTY=unknown
+ARG BUILD_TIMESTAMP_UTC=unknown
+ARG RPKI_CLIENT_VERSION=9.8
+
+LABEL org.opencontainers.image.title="ours-rp-metrics" \
+      org.opencontainers.image.description="Ours RP artifact metrics image for multi-arch Docker Compose deployment" \
+      org.opencontainers.image.revision="${SOURCE_COMMIT}" \
+      org.opencontainers.image.created="${BUILD_TIMESTAMP_UTC}" \
+      org.opencontainers.image.source-dirty="${SOURCE_DIRTY}" \
+      org.opencontainers.image.rpki-client.version="${RPKI_CLIENT_VERSION}"
+
+RUN chmod +x /opt/ours-rp/bin/* \
+    && mkdir -p /var/lib/ours-rp/state /var/lib/ours-rp/runs /var/lib/ours-rp/logs
+
+ENV RUN_ROOT=/var/lib/ours-rp \
+    BIN_DIR=/opt/ours-rp/bin \
+    RUST_BACKTRACE=1
+
+VOLUME ["/var/lib/ours-rp"]

docker/ours-rp-runtime.Dockerfile

@@ -0,0 +1,145 @@
+ARG BUILDER_IMAGE=rust:1-bookworm
+ARG RUNTIME_IMAGE=debian:bookworm-slim
+
+FROM --platform=$BUILDPLATFORM ${BUILDER_IMAGE} AS builder
+
+ARG BUILDARCH
+ARG TARGETARCH
+ARG TARGETPLATFORM
+
+WORKDIR /src
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+      build-essential \
+      ca-certificates \
+      clang \
+      cmake \
+      git \
+      libclang-dev \
+      make \
+      perl \
+      pkg-config \
+      python3 \
+    && if [ "$TARGETARCH" != "$BUILDARCH" ]; then \
+         case "$TARGETARCH" in \
+           arm64) \
+             apt-get install -y --no-install-recommends \
+               g++-aarch64-linux-gnu \
+               gcc-aarch64-linux-gnu \
+               libc6-dev-arm64-cross \
+             ;; \
+           amd64) \
+             apt-get install -y --no-install-recommends \
+               g++-x86-64-linux-gnu \
+               gcc-x86-64-linux-gnu \
+               libc6-dev-amd64-cross \
+             ;; \
+           *) \
+             echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 \
+             ;; \
+         esac; \
+       fi \
+    && rm -rf /var/lib/apt/lists/* \
+    && case "$TARGETARCH" in \
+         amd64|arm64) ;; \
+         *) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 ;; \
+       esac
+
+COPY . .
+
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    --mount=type=cache,target=/src/target \
+    set -eux; \
+    case "$TARGETARCH" in \
+      amd64) target_triple=x86_64-unknown-linux-gnu ;; \
+      arm64) target_triple=aarch64-unknown-linux-gnu ;; \
+      *) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 ;; \
+    esac; \
+    rustup target add "$target_triple"; \
+    if [ "$TARGETARCH" != "$BUILDARCH" ]; then \
+      case "$TARGETARCH" in \
+        arm64) \
+          export \
+            CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \
+            CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc \
+            CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ \
+            AR_aarch64_unknown_linux_gnu=aarch64-linux-gnu-ar \
+            PKG_CONFIG_ALLOW_CROSS=1 \
+          ;; \
+        amd64) \
+          export \
+            CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc \
+            CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc \
+            CXX_x86_64_unknown_linux_gnu=x86_64-linux-gnu-g++ \
+            AR_x86_64_unknown_linux_gnu=x86_64-linux-gnu-ar \
+            PKG_CONFIG_ALLOW_CROSS=1 \
+          ;; \
+        *) \
+          echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 \
+          ;; \
+      esac; \
+    fi; \
+    cargo build --release --target "$target_triple" \
+      --bin rpki \
+      --bin rpki_daemon \
+      --bin db_stats; \
+    mkdir -p /build-out/bin; \
+    cp \
+      "target/$target_triple/release/rpki" \
+      "target/$target_triple/release/rpki_daemon" \
+      "target/$target_triple/release/db_stats" \
+      /build-out/bin/
+
+FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS runtime
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+      bash \
+      ca-certificates \
+      coreutils \
+      curl \
+      findutils \
+      iputils-ping \
+      jq \
+      procps \
+      python3 \
+      rsync \
+      time \
+      tzdata \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt/ours-rp
+
+COPY --from=builder /build-out/bin/ /opt/ours-rp/bin/
+COPY scripts/soak/run_soak.sh /opt/ours-rp/run_soak.sh
+COPY scripts/soak/portable-soak.env.example /opt/ours-rp/portable-soak.env.example
+COPY tests/fixtures/tal/ /opt/ours-rp/fixtures/tal/
+COPY tests/fixtures/ta/ /opt/ours-rp/fixtures/ta/
+COPY fixtures/live_20260619/tal/ /opt/ours-rp/fixtures/live_20260619/tal/
+COPY fixtures/live_20260619/ta/ /opt/ours-rp/fixtures/live_20260619/ta/
+
+ARG SOURCE_COMMIT=unknown
+ARG SOURCE_DIRTY=unknown
+ARG BUILD_TIMESTAMP_UTC=unknown
+
+LABEL org.opencontainers.image.title="ours-rp-runtime" \
+      org.opencontainers.image.description="Ours RP runtime image for multi-arch Docker Compose deployment" \
+      org.opencontainers.image.revision="${SOURCE_COMMIT}" \
+      org.opencontainers.image.created="${BUILD_TIMESTAMP_UTC}" \
+      org.opencontainers.image.source-dirty="${SOURCE_DIRTY}"
+
+RUN chmod +x /opt/ours-rp/run_soak.sh /opt/ours-rp/bin/* \
+    && mkdir -p /var/lib/ours-rp/state /var/lib/ours-rp/runs /var/lib/ours-rp/logs /var/lib/ours-rp/tmp
+
+ENV PACKAGE_ROOT=/opt/ours-rp \
+    ENV_FILE=/opt/ours-rp/.env \
+    RUN_ROOT=/var/lib/ours-rp \
+    BIN_DIR=/opt/ours-rp/bin \
+    FIXTURE_DIR=/opt/ours-rp/fixtures \
+    RUST_BACKTRACE=1
+
+VOLUME ["/var/lib/ours-rp"]
+
+CMD ["/opt/ours-rp/run_soak.sh"]

fixtures/live_20260619/manifest.json

@@ -0,0 +1,80 @@
+{
+  "created_at_utc": "2026-06-19T08:08:03Z",
+  "items": [
+    {
+      "rir": "afrinic",
+      "ta_bytes": 1216,
+      "ta_download": "200 1216 1.351147",
+      "ta_elapsed_s": 1.227,
+      "ta_path": "rpki_2/rpki/fixtures/live_20260619/ta/afrinic-ta.cer",
+      "ta_sha256": "43a26fd28bafb9398e5b2ab19e036b450bd04f4973a7f5ad151cebdee0edac36",
+      "ta_uri": "https://rpki.afrinic.net/repository/AfriNIC.cer",
+      "tal_bytes": 496,
+      "tal_download": "200 496 1.361326",
+      "tal_elapsed_s": 1.238,
+      "tal_path": "rpki_2/rpki/fixtures/live_20260619/tal/afrinic.tal",
+      "tal_sha256": "2838ef30ea27ce5705abf5f5adb131d8c35b1f50858338a2f3c84bb207c2fa35",
+      "tal_url": "https://rpki.afrinic.net/tal/afrinic.tal"
+    },
+    {
+      "rir": "apnic",
+      "ta_bytes": 1222,
+      "ta_download": "200 1222 1.012321",
+      "ta_elapsed_s": 0.921,
+      "ta_path": "rpki_2/rpki/fixtures/live_20260619/ta/apnic-ta.cer",
+      "ta_sha256": "2014230ad49b2777ac2bde0948ddfa4b8f207114c549e26d755de88c3593e3af",
+      "ta_uri": "https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer",
+      "tal_bytes": 532,
+      "tal_download": "200 466 1.007155",
+      "tal_elapsed_s": 0.917,
+      "tal_path": "rpki_2/rpki/fixtures/live_20260619/tal/apnic.tal",
+      "tal_sha256": "472e551f7c551c2e999e582b7c9437d3bee4900fe53afff62aeb28d4940ade94",
+      "tal_url": "https://tal.apnic.net/apnic.tal"
+    },
+    {
+      "rir": "arin",
+      "ta_bytes": 1143,
+      "ta_download": "200 1143 0.816714",
+      "ta_elapsed_s": 0.743,
+      "ta_path": "rpki_2/rpki/fixtures/live_20260619/ta/arin-ta.cer",
+      "ta_sha256": "5b3c2f6f04abd19261084487a43c156f778b0b8926d63801d48c7a93ed349492",
+      "ta_uri": "https://rrdp.arin.net/arin-rpki-ta.cer",
+      "tal_bytes": 1258,
+      "tal_download": "200 1258 0.848581",
+      "tal_elapsed_s": 0.774,
+      "tal_path": "rpki_2/rpki/fixtures/live_20260619/tal/arin.tal",
+      "tal_sha256": "1f8bdb03bcc30a3b8e11fd9a87102fba250c22137a3c8baa9c81b139cb412639",
+      "tal_url": "https://www.arin.net/resources/manage/rpki/arin.tal"
+    },
+    {
+      "rir": "lacnic",
+      "ta_bytes": 1166,
+      "ta_download": "200 1166 1.025273",
+      "ta_elapsed_s": 0.932,
+      "ta_path": "rpki_2/rpki/fixtures/live_20260619/ta/lacnic-ta.cer",
+      "ta_sha256": "f44bc51008fd6998de7597b72a79b07bf3ebcb0f14daa7c7c022c0e9d66e0ad0",
+      "ta_uri": "https://rrdp.lacnic.net/ta/rta-lacnic-rpki.cer",
+      "tal_bytes": 502,
+      "tal_download": "200 502 1.729122",
+      "tal_elapsed_s": 1.565,
+      "tal_path": "rpki_2/rpki/fixtures/live_20260619/tal/lacnic.tal",
+      "tal_sha256": "d44bb9394ab009c8b53e5efebf2a1c9450bab61a27efe00de5a3e4587a3a2f6a",
+      "tal_url": "https://www.lacnic.net/innovaportal/file/4983/1/lacnic.tal"
+    },
+    {
+      "rir": "ripe",
+      "ta_bytes": 1036,
+      "ta_download": "200 1036 1.060992",
+      "ta_elapsed_s": 4.536,
+      "ta_path": "rpki_2/rpki/fixtures/live_20260619/ta/ripe-ncc-ta.cer",
+      "ta_sha256": "3e3f7e4efc8d0cea03d9cc1fde6e168b45c26d7b3272e14abd8da4871886e539",
+      "ta_uri": "https://rpki.ripe.net/ta/ripe-ncc-ta.cer",
+      "tal_bytes": 482,
+      "tal_download": "200 482 1.092082",
+      "tal_elapsed_s": 0.992,
+      "tal_path": "rpki_2/rpki/fixtures/live_20260619/tal/ripe-ncc.tal",
+      "tal_sha256": "59ca27ef93f23682749fcefe7c6d70fbc723343549ff9e4d3996acaff79817fb",
+      "tal_url": "https://tal.rpki.ripe.net/ripe-ncc.tal"
+    }
+  ]
+}

fixtures/live_20260619/tal/afrinic.tal

@@ -0,0 +1,10 @@
+rsync://rpki.afrinic.net/repository/AfriNIC.cer
+https://rpki.afrinic.net/repository/AfriNIC.cer
+
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxsAqAhWIO+ON2Ef9oRDM
+pKxv+AfmSLIdLWJtjrvUyDxJPBjgR+kVrOHUeTaujygFUp49tuN5H2C1rUuQavTH
+vve6xNF5fU3OkTcqEzMOZy+ctkbde2SRMVdvbO22+TH9gNhKDc9l7Vu01qU4LeJH
+k3X0f5uu5346YrGAOSv6AaYBXVgXxa0s9ZvgqFpim50pReQe/WI3QwFKNgpPzfQL
+6Y7fDPYdYaVOXPXSKtx7P4s4KLA/ZWmRL/bobw/i2fFviAGhDrjqqqum+/9w1hEl
+L/vqihVnV18saKTnLvkItA/Bf5i11Yhw2K7qv573YWxyuqCknO/iYLTR1DToBZcZ
+UQIDAQAB

fixtures/live_20260619/tal/apnic-rfc7730-https.tal

@@ -0,0 +1,10 @@
+https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer
+rsync://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer
+
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx9RWSL61YAAYumEiU8z8
+qH2ETVIL01ilxZlzIL9JYSORMN5Cmtf8V2JblIealSqgOTGjvSjEsiV73s67zYQI
+7C/iSOb96uf3/s86NqbxDiFQGN8qG7RNcdgVuUlAidl8WxvLNI8VhqbAB5uSg/Mr
+LeSOvXRja041VptAxIhcGzDMvlAJRwkrYK/Mo8P4E2rSQgwqCgae0ebY1CsJ3Cjf
+i67C1nw7oXqJJovvXJ4apGmEv8az23OLC6Ki54Ul/E6xk227BFttqFV3YMtKx42H
+cCcDVZZy01n7JjzvO8ccaXmHIgR7utnqhBRNNq5Xc5ZhbkrUsNtiJmrZzVlgU6Ou
+0wIDAQAB

fixtures/live_20260619/tal/apnic.tal

@@ -0,0 +1,10 @@
+https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer
+rsync://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer
+
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx9RWSL61YAAYumEiU8z8
+qH2ETVIL01ilxZlzIL9JYSORMN5Cmtf8V2JblIealSqgOTGjvSjEsiV73s67zYQI
+7C/iSOb96uf3/s86NqbxDiFQGN8qG7RNcdgVuUlAidl8WxvLNI8VhqbAB5uSg/Mr
+LeSOvXRja041VptAxIhcGzDMvlAJRwkrYK/Mo8P4E2rSQgwqCgae0ebY1CsJ3Cjf
+i67C1nw7oXqJJovvXJ4apGmEv8az23OLC6Ki54Ul/E6xk227BFttqFV3YMtKx42H
+cCcDVZZy01n7JjzvO8ccaXmHIgR7utnqhBRNNq5Xc5ZhbkrUsNtiJmrZzVlgU6Ou
+0wIDAQAB

fixtures/live_20260619/tal/arin.tal

@@ -0,0 +1,19 @@
+# THIS TRUST ANCHOR LOCATOR IS PROVIDED BY THE AMERICAN REGISTRY FOR
+# INTERNET NUMBERS (ARIN) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL ARIN BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS PUBLIC KEY, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+rsync://rpki.arin.net/repository/arin-rpki-ta.cer
+https://rrdp.arin.net/arin-rpki-ta.cer
+
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3lZPjbHvMRV5sDDqfLc/685th5FnreHMJjg8
+pEZUbG8Y8TQxSBsDebbsDpl3Ov3Cj1WtdrJ3CIfQODCPrrJdOBSrMATeUbPC+JlNf2SRP3UB+VJFgtTj
+0RN8cEYIuhBW5t6AxQbHhdNQH+A1F/OJdw0q9da2U29Lx85nfFxvnC1EpK9CbLJS4m37+RlpNbT1cba+
+b+loXpx0Qcb1C4UpJCGDy7uNf5w6/+l7RpATAHqqsX4qCtwwDYlbHzp2xk9owF3mkCxzl0HwncO+sEHH
+eaL3OjtwdIGrRGeHi2Mpt+mvWHhtQqVG+51MHTyg+nIjWFKKGx1Q9+KDx4wJStwveQIDAQAB

fixtures/live_20260619/tal/lacnic.tal

@@ -0,0 +1,4 @@
+https://rrdp.lacnic.net/ta/rta-lacnic-rpki.cer
+rsync://repository.lacnic.net/rpki/lacnic/rta-lacnic-rpki.cer
+
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqZEzhYK0+PtDOPfub/KRc3MeWx3neXx4/wbnJWGbNAtbYqXg3uU5J4HFzPgk/VIppgSKAhlO0H60DRP48by9gr5/yDHu2KXhOmnMg46sYsUIpfgtBS9+VtrqWziJfb+pkGtuOWeTnj6zBmBNZKK+5AlMCW1WPhrylIcB+XSZx8tk9GS/3SMQ+YfMVwwAyYjsex14Uzto4GjONALE5oh1M3+glRQduD6vzSwOD+WahMbc9vCOTED+2McLHRKgNaQf0YJ9a1jG9oJIvDkKXEqdfqDRktwyoD74cV57bW3tBAexB7GglITbInyQAsmdngtfg2LUMrcROHHP86QPZINjDQIDAQAB

fixtures/live_20260619/tal/ripe-ncc.tal

@@ -0,0 +1,10 @@
+https://rpki.ripe.net/ta/ripe-ncc-ta.cer
+rsync://rpki.ripe.net/ta/ripe-ncc-ta.cer
+
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0URYSGqUz2myBsOzeW1j
+Q6NsxNvlLMyhWknvnl8NiBCs/T/S2XuNKQNZ+wBZxIgPPV2pFBFeQAvoH/WK83Hw
+A26V2siwm/MY2nKZ+Olw+wlpzlZ1p3Ipj2eNcKrmit8BwBC8xImzuCGaV0jkRB0G
+Z0hoH6Ml03umLprRsn6v0xOP0+l6Qc1ZHMFVFb385IQ7FQQTcVIxrdeMsoyJq9eM
+kE6DoclHhF/NlSllXubASQ9KUWqJ0+Ot3QCXr4LXECMfkpkVR2TZT+v5v658bHVs
+6ZxRD1b6Uk1uQKAyHUbn/tXvP8lrjAibGzVsXDT2L0x4Edx+QdixPgOji3gBMyL2
+VwIDAQAB

monitor/README.md

@@ -0,0 +1,220 @@
+# Ours RP Prometheus / Grafana Monitor
+
+本目录提供本地开发监控栈，用于采集 `rpki_artifact_metrics` 暴露的 ours RP soak 指标。
+
+## 前置条件
+
+1. Docker + Docker Compose v2；
+2. 宿主机已启动 `rpki_artifact_metrics`，并监听 Docker 网桥可访问的地址，例如 `0.0.0.0:9556`；
+3. Prometheus 容器通过 `host.docker.internal:9556` 访问宿主 sidecar。
+
+Linux Docker 下 compose 已配置：
+
+```yaml
+extra_hosts:
+  - host.docker.internal:host-gateway
+```
+
+## 启动
+
+```bash
+cd rpki_2/rpki/monitor
+docker compose up -d
+```
+
+默认镜像使用官方 Docker Hub 镜像：
+
+```text
+prom/prometheus:v2.55.1
+grafana/grafana:11.3.1
+```
+
+如需切到其它镜像源：
+
+```bash
+PROMETHEUS_IMAGE=<mirror>/prom/prometheus:v2.55.1 \
+GRAFANA_IMAGE=<mirror>/grafana/grafana:11.3.1 \
+docker compose up -d
+```
+
+默认端口：
+
+- Prometheus: <http://localhost:9090>
+- Grafana: <http://localhost:3000>
+- Grafana 默认账号密码：`admin` / `admin`
+
+如端口冲突：
+
+```bash
+PROMETHEUS_PORT=19090 GRAFANA_PORT=13000 docker compose up -d
+```
+
+Prometheus 默认保留 7 天数据；可通过 `PROMETHEUS_RETENTION` 覆盖：
+
+```bash
+PROMETHEUS_RETENTION=7d docker compose up -d
+```
+
+## 长期稳定性测试
+
+portable soak package 内置 `run_24h_soak_with_metrics.sh`，用于连续运行 ours RP、启动 metrics sidecar、启动本监控栈，并每小时生成报告：
+
+```bash
+cd /path/to/portable-soak
+SOAK_DURATION_SECS=0 \
+HOURLY_REPORT_INTERVAL_SECS=3600 \
+SOAK_RETAIN_RUNS=100 \
+CLEAN_TMP_AFTER_RUN=1 \
+PROMETHEUS_RETENTION=7d \
+STOP_MONITOR_STACK_ON_EXIT=0 \
+FEISHU_WEBHOOK_SCRIPT=/home/yuyr/.codex/skills/user/feishu-webhook/scripts/send_feishu_text.py \
+./run_24h_soak_with_metrics.sh
+```
+
+`SOAK_DURATION_SECS=0` 表示持续运行不自动停止；如需 24 小时自然停止，可设置为 `86400`，脚本会等当前 run 完成后退出，不会直接 kill 半轮验证。
+
+关键产物：
+
+- `runs/run_xxxx/`：最近 100 个 run 原始产物；
+- `hourly_reports/hour_*.md`：小时级报告；
+- `hourly_reports/hourly_summary.jsonl`：小时级结构化汇总；
+- `incident_runs/run_xxxx/`：异常 run 固化副本；
+- `logs/metrics.*`、`logs/24h-soak.*`、`logs/hourly-reporter.*`：运行日志。
+
+短周期联调可把 `SOAK_DURATION_SECS` 和 `HOURLY_REPORT_INTERVAL_SECS` 调小，并设置 `FEISHU_DRY_RUN=1` 避免真实飞书通知。
+
+## 停止
+
+```bash
+cd rpki_2/rpki/monitor
+docker compose down
+```
+
+保留数据 volume。若要清理数据：
+
+```bash
+docker compose down -v
+```
+
+## 典型本地联调命令
+
+先启动 APNIC soak 和 metrics sidecar，例如：
+
+```bash
+# soak .env 关键配置
+MAX_RUNS=-1
+RIRS=apnic
+RETAIN_RUNS=5
+INTERVAL_SECS=0
+
+# metrics sidecar
+rpki_artifact_metrics \
+  --run-root /path/to/portable-soak \
+  --listen 0.0.0.0:9556 \
+  --poll-secs 5 \
+  --instance local-apnic-continuous
+```
+
+再启动监控栈：
+
+```bash
+cd rpki_2/rpki/monitor
+docker compose up -d
+```
+
+## 验证
+
+Prometheus target：
+
+```bash
+curl -s 'http://localhost:9090/api/v1/targets' | python3 -m json.tool
+```
+
+Prometheus query：
+
+```bash
+curl -G 'http://localhost:9090/api/v1/query' \
+  --data-urlencode 'query=up{job="ours-rp-artifact-metrics"}'
+
+curl -G 'http://localhost:9090/api/v1/query' \
+  --data-urlencode 'query=ours_rp_run_completed_total{status="success"}'
+```
+
+Grafana health：
+
+```bash
+curl -s http://localhost:3000/api/health | python3 -m json.tool
+```
+
+Grafana dashboard：
+
+- 打开 <http://localhost:3000/d/ours-rp-soak-overview/ours-rp-soak-overview>
+
+## 主要指标
+
+- `ours_rp_metrics_service_up`
+- `ours_rp_run_completed_total`
+- `ours_rp_run_duration_seconds`
+- `ours_rp_run_max_rss_bytes`
+- `ours_rp_vrps{kind="total|unique"}`：`total` 为去重前 VRP 条目数，`unique` 按 `(ASN, IP Prefix, Max Length)` 去重。
+- `ours_rp_vaps`
+- `ours_rp_publication_points`
+- `ours_rp_repo_sync_phase_count`
+- `ours_rp_large_publication_points{object_count_gt="10|50|100|..."}`
+- `ours_rp_cir_objects`
+- `ours_rp_ccr_state_items`
+
+## Inter-RP 持续对比监控
+
+`rpki_inter_rp_metrics` 用于汇总三方 RP 的最新产物：
+
+- ours RP：读取当前 portable soak 的 `runs/run_xxxx/run-summary.json`、`result.ccr`、CSV 产物；
+- Routinator：读取远端200同步来的 `routinator/latest/run-meta.json`、`vrps.csv`、`vaps.csv`；
+- rpki-client 9.8：读取远端200同步来的 `rpki-client/latest/run-meta.json`、`vrps.csv`、`vaps.csv`、`result.ccr`。
+
+远端231 启动 sidecar 示例：
+
+```bash
+rpki_inter_rp_metrics \
+  --ours-run-root /root/rpki_20260608_2_feature062_24h_20260608T075547Z/portable-soak \
+  --peer-root /root/inter-rp-aggregator/synced-from-200 \
+  --listen 0.0.0.0:9557 \
+  --poll-secs 30 \
+  --instance remote231-inter-rp
+```
+
+Prometheus 已新增 `ours-rp-inter-rp-metrics` scrape job，默认访问 `host.docker.internal:9557`。
+
+远端200 runner 与远端231同步脚本位于：
+
+```text
+scripts/inter_rp/run_remote200_rp_loops.sh
+scripts/inter_rp/run_single_rp_with_rss.sh
+scripts/inter_rp/sync_remote200_to_231.sh
+scripts/inter_rp/run_inter_rp_metrics_sidecar.sh
+scripts/inter_rp/inter-rp.env.example
+```
+
+如需从本机独立开关远端200上的 Routinator 或 rpki-client，使用：
+
+```bash
+scripts/inter_rp/control_remote200_rp.sh status all
+scripts/inter_rp/control_remote200_rp.sh stop routinator
+scripts/inter_rp/control_remote200_rp.sh start routinator
+scripts/inter_rp/control_remote200_rp.sh restart rpki-client
+```
+
+默认远端为 `root@43.110.128.200`，可通过 `REMOTE_HOST=...` 覆盖；脚本只管理指定 RP 的 loop 和当前子进程，不会自动影响另一个 RP。
+
+关键指标：
+
+- `inter_rp_run_wall_seconds{rp="ours-rp|routinator|rpki-client"}`
+- `inter_rp_run_max_rss_bytes{rp="...",kind="aggregate_peak"}`
+- `inter_rp_vrps{rp="..."}`：按 `(ASN, IP Prefix, Max Length)` 去重。
+- `inter_rp_vaps{rp="..."}`：按 `(Customer ASN, Providers)` 去重，Routinator 使用 `--enable-aspa` JSON 输出转换，rpki-client 使用 `-j` JSON 输出转换。
+- `inter_rp_ccr_digest_match{left="ours-rp",right="rpki-client",state="overall|mfts|vrps|vaps|tas|rks"}`
+- `inter_rp_sync_age_seconds`
+
+Grafana dashboard：
+
+- <http://localhost:3000/d/ours-rp-inter-rp/ours-rp-inter-rp>

monitor/docker-compose.yml

@@ -0,0 +1,38 @@
+services:
+  prometheus:
+    image: ${PROMETHEUS_IMAGE:-prom/prometheus:v2.55.1}
+    container_name: ours-rp-prometheus
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-7d}
+      - --web.enable-lifecycle
+    extra_hosts:
+      - host.docker.internal:host-gateway
+    ports:
+      - "${PROMETHEUS_PORT:-9090}:9090"
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    restart: unless-stopped
+
+  grafana:
+    image: ${GRAFANA_IMAGE:-grafana/grafana:11.3.1}
+    container_name: ours-rp-grafana
+    depends_on:
+      - prometheus
+    ports:
+      - "${GRAFANA_PORT:-3000}:3000"
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+    volumes:
+      - grafana-data:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+    restart: unless-stopped
+
+volumes:
+  prometheus-data:
+  grafana-data:

monitor/grafana/dashboards/ours-rp-inter-rp.json

@@ -0,0 +1,826 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "id": 1,
+      "title": "Ours Only Repo Count",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(inter_rp_repo_sync_overlap_total{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_ours\"})",
+          "legendFormat": "only ours",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 2,
+      "title": "Routinator Only Repo Count",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(inter_rp_repo_sync_overlap_total{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_routinator\"})",
+          "legendFormat": "only routinator",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 3,
+      "title": "Ours vs Routinator VAP Diff",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(inter_rp_vaps_diff{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"})",
+          "legendFormat": "vap diff",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 4,
+      "title": "Ours vs Routinator VRP Diff",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(inter_rp_vrps_diff{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"})",
+          "legendFormat": "vrp diff",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 5,
+      "title": "Wall Time by RP",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_run_wall_seconds{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
+          "legendFormat": "{{rp}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 6,
+      "title": "Max RSS Aggregate Peak by RP",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_run_max_rss_bytes{exported_instance=~\".*inter-rp\",kind=\"aggregate_peak\",rp=~\"ours-rp|routinator\"}",
+          "legendFormat": "{{rp}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 7,
+      "title": "VRPs by RP (unique ASN/Prefix/MaxLen)",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 12
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_vrps{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
+          "legendFormat": "{{rp}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 8,
+      "title": "VAPs / ASPAs by RP (unique Customer/Providers)",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 12
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_vaps{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
+          "legendFormat": "{{rp}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 9,
+      "title": "Latest RP Runs",
+      "type": "table",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 20
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_run_seq{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{rp}} seq",
+          "refId": "A"
+        },
+        {
+          "expr": "inter_rp_run_success{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{rp}} success",
+          "refId": "B"
+        },
+        {
+          "expr": "inter_rp_run_wall_seconds{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{rp}} wall",
+          "refId": "C"
+        }
+      ]
+    },
+    {
+      "id": 10,
+      "title": "Output Count Diffs (unique)",
+      "type": "table",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 20
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_vrps_diff{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"}",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "vrps ours-rp-routinator",
+          "refId": "A"
+        },
+        {
+          "expr": "inter_rp_vaps_diff{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"}",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "vaps ours-rp-routinator",
+          "refId": "B"
+        }
+      ]
+    },
+    {
+      "id": 15,
+      "title": "VRP Diff Trend by Class",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 28
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "min": 0,
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_vrps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"total\"}",
+          "legendFormat": "total diff",
+          "refId": "A"
+        },
+        {
+          "expr": "inter_rp_vrps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_ours\"}",
+          "legendFormat": "only ours",
+          "refId": "B"
+        },
+        {
+          "expr": "inter_rp_vrps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_routinator\"}",
+          "legendFormat": "only routinator",
+          "refId": "C"
+        }
+      ]
+    },
+    {
+      "id": 16,
+      "title": "VAP Diff Trend by Class",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 28
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "min": 0,
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_vaps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"total\"}",
+          "legendFormat": "total diff",
+          "refId": "A"
+        },
+        {
+          "expr": "inter_rp_vaps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_ours\"}",
+          "legendFormat": "only ours",
+          "refId": "B"
+        },
+        {
+          "expr": "inter_rp_vaps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_routinator\"}",
+          "legendFormat": "only routinator",
+          "refId": "C"
+        }
+      ]
+    },
+    {
+      "id": 11,
+      "title": "Artifact Age by RP",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 36
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_artifact_age_seconds{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
+          "legendFormat": "{{rp}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 12,
+      "title": "Repo Sync Availability by RP",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 44
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "min": 0,
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_repo_sync_total{exported_instance=~\".*inter-rp\",state=~\"available|failed\",rp=~\"ours-rp|routinator\"}",
+          "legendFormat": "{{rp}} {{state}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 13,
+      "title": "Repo Sync Overlap Classes",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 44
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "min": 0,
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_repo_sync_overlap_total{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"}",
+          "legendFormat": "{{class}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 14,
+      "title": "Repo Sync Diff URIs",
+      "type": "table",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 24,
+        "x": 0,
+        "y": 52
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "uri"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 760
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "class"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 140
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "^(mft|crl|crt|roa|aspa)$"
+            },
+            "properties": [
+              {
+                "id": "custom.align",
+                "value": "right"
+              },
+              {
+                "id": "custom.width",
+                "value": 80
+              }
+            ]
+          }
+        ]
+      },
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_repo_sync_diff_info{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"}",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{class}} #{{rank}}",
+          "refId": "A"
+        }
+      ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__name__": true,
+              "exported_instance": true,
+              "instance": true,
+              "job": true,
+              "left": true,
+              "right": true,
+              "rank": true,
+              "routinator_duration": true
+            },
+            "indexByName": {
+              "class": 0,
+              "uri": 1,
+              "mft": 2,
+              "crl": 3,
+              "crt": 4,
+              "roa": 5,
+              "aspa": 6
+            },
+            "renameByName": {
+              "class": "class",
+              "uri": "uri",
+              "mft": "mft",
+              "crl": "crl",
+              "crt": "crt",
+              "roa": "roa",
+              "aspa": "aspa"
+            }
+          }
+        }
+      ]
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 40,
+  "tags": [
+    "rpki",
+    "inter-rp",
+    "routinator"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timezone": "browser",
+  "title": "Ours RP vs Routinator",
+  "uid": "ours-rp-inter-rp",
+  "version": 4
+}

monitor/grafana/dashboards/ours-rp-repo-sync.json

@@ -0,0 +1,761 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_publication_points",
+          "legendFormat": "publication points",
+          "refId": "A"
+        }
+      ],
+      "title": "Publication Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_ok\"}",
+          "legendFormat": "rrdp ok",
+          "refId": "A"
+        }
+      ],
+      "title": "RRDP OK Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "sum(ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}) or vector(0)",
+          "legendFormat": "fallback",
+          "refId": "A"
+        }
+      ],
+      "title": "Rsync Fallback Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache",
+          "refId": "A"
+        }
+      ],
+      "title": "Failed No Cache Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 4
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"repo_sync_total\"}",
+          "legendFormat": "repo sync total",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"rrdp_download_total\"}",
+          "legendFormat": "rrdp download",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"rsync_download_total\"}",
+          "legendFormat": "rsync download",
+          "refId": "C"
+        }
+      ],
+      "title": "Repo Sync Download Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 12,
+        "w": 12,
+        "h": 8
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count",
+          "legendFormat": "{{phase}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repo Sync Phase Counts",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 12,
+        "w": 12,
+        "h": 8
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}",
+          "legendFormat": "rrdp failed, rsync ok",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_failed\"}",
+          "legendFormat": "rrdp failed, rsync failed",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache",
+          "refId": "C"
+        },
+        {
+          "expr": "ours_rp_tree_instances{state=\"failed\"}",
+          "legendFormat": "tree failed",
+          "refId": "D"
+        }
+      ],
+      "title": "Repo Failure / Fallback Counts",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 20,
+        "w": 12,
+        "h": 8
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_ok\"}",
+          "legendFormat": "rsync fallback duration",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_failed\"}",
+          "legendFormat": "failed duration",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_repo_terminal_state_duration_seconds_total{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache duration",
+          "refId": "C"
+        }
+      ],
+      "title": "Repo Failure / Fallback Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 29,
+        "w": 12,
+        "h": 9
+      },
+      "id": 9,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_rrdp_rsync_failed_repository_duration_seconds",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "RRDP + Rsync Failed Repositories",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "terminal_state": true,
+              "rank": true,
+              "transport": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "phase": 2,
+              "uri": 3,
+              "Value": 4
+            },
+            "renameByName": {
+              "Value": "duration"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 29,
+        "w": 12,
+        "h": 9
+      },
+      "id": 11,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "topk(20, ours_rp_top_repository_sync_duration_seconds)",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Top 20 Repositories by Sync Duration",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "terminal_state": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "phase": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "rank": 2,
+              "transport": 3,
+              "uri": 4,
+              "Value": 5
+            },
+            "renameByName": {
+              "Value": "value"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 38,
+        "w": 24,
+        "h": 9
+      },
+      "id": 10,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "topk(20, ours_rp_top_publication_point_object_count)",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Top Publication Points by Objects",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "phase": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "rank": 2,
+              "terminal_state": 3,
+              "transport": 4,
+              "uri": 5,
+              "Value": 6
+            },
+            "renameByName": {}
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository sync success in the latest successful run; 1 means successful, 0 means failed or failed_no_cache.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bool"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 47
+      },
+      "id": 12,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_sync_success",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Sync Success by Repo",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository total sync duration aggregated from publication point repo_sync_duration_ms.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 55
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_sync_duration_seconds{stat=\"sum\"}",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Sync Duration by Repo",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository downloaded bytes attributed from report.json downloads events.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 63
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_download_bytes",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Download Bytes by Repo",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": [
+    "ours-rp",
+    "rpki",
+    "soak",
+    "repo-sync"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Ours RP Repo Sync",
+  "uid": "ours-rp-repo-sync",
+  "version": 3,
+  "weekStart": ""
+}

monitor/grafana/dashboards/ours-rp-rtr-overview.json

@@ -0,0 +1,666 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "id": 1,
+      "title": "RTR Metrics Enabled",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 0,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(ours_rp_rtr_metrics_enabled)",
+          "legendFormat": "enabled",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 2,
+      "title": "Refresh Success",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 5,
+        "x": 4,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(ours_rp_rtr_source_refresh_status{status=\"success\"})",
+          "legendFormat": "success",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 3,
+      "title": "Consecutive Failures",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 5,
+        "x": 9,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(ours_rp_rtr_source_refresh_consecutive_failures)",
+          "legendFormat": "failures",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 4,
+      "title": "Last Success Age",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 5,
+        "x": 14,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "decimals": 0,
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(ours_rp_rtr_source_last_success_age_seconds)",
+          "legendFormat": "age",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 5,
+      "title": "RTR RSS",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 5,
+        "x": 19,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "decimals": 0,
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(ours_rp_rtr_process_rss_bytes)",
+          "legendFormat": "rss",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 6,
+      "title": "Data Quality Totals",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_rtr_data_quality_items{stage=~\"ccr_input|before_slurm|after_slurm\",type=\"total\"}",
+          "legendFormat": "{{stage}} total",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_rtr_data_quality_items{stage=\"after_slurm\",type=\"vrp\"}",
+          "legendFormat": "after_slurm vrp",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_rtr_data_quality_items{stage=\"after_slurm\",type=\"aspa\"}",
+          "legendFormat": "after_slurm aspa",
+          "refId": "C"
+        }
+      ]
+    },
+    {
+      "id": 7,
+      "title": "SLURM Filters / Assertions",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_rtr_slurm_filters",
+          "legendFormat": "filter {{type}}",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_rtr_slurm_assertions",
+          "legendFormat": "assert {{type}}",
+          "refId": "B"
+        }
+      ]
+    },
+    {
+      "id": 8,
+      "title": "Cache Ready / Delta Window",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 12
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_rtr_cache_ready",
+          "legendFormat": "ready",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_rtr_cache_delta_window_length",
+          "legendFormat": "v{{version}} length",
+          "refId": "B"
+        }
+      ]
+    },
+    {
+      "id": 9,
+      "title": "Cache Snapshot Items",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 12
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_rtr_cache_snapshot_items",
+          "legendFormat": "v{{version}} {{type}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 10,
+      "title": "Latest Delta Items",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 12
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_rtr_cache_delta_items",
+          "legendFormat": "v{{version}} {{direction}} {{type}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 11,
+      "title": "Connections",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 20
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_rtr_active_connections",
+          "legendFormat": "active",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_rtr_connections",
+          "legendFormat": "{{transport}}",
+          "refId": "B"
+        }
+      ]
+    },
+    {
+      "id": 12,
+      "title": "Connection Utilization / Max",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 20
+      },
+      "fieldConfig": {
+        "defaults": {
+          "min": 0
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": ".*utilization.*"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "percentunit"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": ".*max.*"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "short"
+              }
+            ]
+          }
+        ]
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_rtr_connection_utilization",
+          "legendFormat": "utilization",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_rtr_max_connections",
+          "legendFormat": "max",
+          "refId": "B"
+        }
+      ]
+    },
+    {
+      "id": 13,
+      "title": "Report Age",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 20
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_rtr_source_report_age_seconds",
+          "legendFormat": "source",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_rtr_runtime_report_age_seconds",
+          "legendFormat": "runtime",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_rtr_clients_report_age_seconds",
+          "legendFormat": "clients",
+          "refId": "C"
+        }
+      ]
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 40,
+  "tags": [
+    "rpki",
+    "inter-rp",
+    "routinator"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timezone": "browser",
+  "title": "RTR Service Overview",
+  "uid": "ours-rp-rtr-overview",
+  "version": 1
+}

monitor/grafana/dashboards/ours-rp-soak-overview.json

@@ -0,0 +1,875 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_cir_trust_anchors",
+          "legendFormat": "RIRs",
+          "refId": "A"
+        }
+      ],
+      "title": "Current Run RIRs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 6,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_run_duration_seconds",
+          "legendFormat": "wall",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest Wall Time",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_run_max_rss_bytes",
+          "legendFormat": "rss",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest Max RSS",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 18,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_publication_points",
+          "legendFormat": "publication points",
+          "refId": "A"
+        }
+      ],
+      "title": "Publication Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 9,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_run_sequence",
+          "legendFormat": "seq",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest Run Sequence",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 2,
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 90
+              },
+              {
+                "color": "green",
+                "value": 98
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 6,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 10,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "100 * sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"publication_point_cache\"}) / sum by (job, instance, exported_instance) (ours_rp_publication_points)",
+          "legendFormat": "PP cache hit ratio",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest PP Cache Hit Ratio",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 11,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_vrps{kind=\"total\"}",
+          "legendFormat": "VRPs raw",
+          "refId": "A"
+        }
+      ],
+      "title": "VRPs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 18,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 12,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_vaps",
+          "legendFormat": "VAPs",
+          "refId": "A"
+        }
+      ],
+      "title": "VAPs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 8,
+        "w": 12,
+        "h": 8
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_duration_seconds",
+          "legendFormat": "wall",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_stage_duration_seconds{stage=\"validation\"}",
+          "legendFormat": "validation",
+          "refId": "B"
+        }
+      ],
+      "title": "Run / Validation Duration",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 8,
+        "w": 12,
+        "h": 8
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_vrps{kind=\"total\"}",
+          "legendFormat": "VRPs raw",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_vrps{kind=\"unique\"}",
+          "legendFormat": "VRPs unique",
+          "refId": "D"
+        },
+        {
+          "expr": "ours_rp_vaps",
+          "legendFormat": "VAPs",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_cir_objects",
+          "legendFormat": "CIR objects",
+          "refId": "C"
+        }
+      ],
+      "title": "Output and Input Sizes",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 16,
+        "w": 12,
+        "h": 8
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_large_publication_points",
+          "legendFormat": "> {{object_count_gt}} objects",
+          "refId": "A"
+        }
+      ],
+      "title": "Large Publication Points by Object Count",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 16,
+        "w": 12,
+        "h": 8
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"validation\"}",
+          "legendFormat": "validation",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"report_write\"}",
+          "legendFormat": "report write",
+          "refId": "E"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"ccr_write\"}",
+          "legendFormat": "ccr write",
+          "refId": "F"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"cir_write\"}",
+          "legendFormat": "cir write",
+          "refId": "G"
+        }
+      ],
+      "title": "Output Stage Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "decimals": 2
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 24,
+        "w": 12,
+        "h": 8
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_max_rss_bytes",
+          "legendFormat": "Max RSS",
+          "refId": "A"
+        }
+      ],
+      "title": "Max RSS Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "decimals": 2,
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 90
+              },
+              {
+                "color": "green",
+                "value": 98
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 24,
+        "w": 12,
+        "h": 8
+      },
+      "id": 17,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "min",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "100 * sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"publication_point_cache\"}) / sum by (job, instance, exported_instance) (ours_rp_publication_points)",
+          "legendFormat": "PP cache hit ratio",
+          "refId": "A"
+        }
+      ],
+      "title": "PP Cache Hit Ratio",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "decimals": 2
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 32,
+        "w": 24,
+        "h": 8
+      },
+      "id": 15,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_state_db_size_bytes",
+          "legendFormat": "{{db}}",
+          "refId": "A"
+        }
+      ],
+      "title": "State DB Size Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 40,
+        "w": 24,
+        "h": 8
+      },
+      "id": 16,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_state_db_files",
+          "legendFormat": "{{db}}",
+          "refId": "A"
+        }
+      ],
+      "title": "State DB File Count Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0,
+          "min": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 48,
+        "w": 24,
+        "h": 8
+      },
+      "id": 18,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"fresh\"})",
+          "legendFormat": "fresh pp",
+          "refId": "A"
+        },
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"roa\"})",
+          "legendFormat": "fresh roa",
+          "refId": "B"
+        },
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"manifest\"})",
+          "legendFormat": "fresh mft",
+          "refId": "C"
+        },
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"certificate\"})",
+          "legendFormat": "fresh crt",
+          "refId": "D"
+        },
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"crl\"})",
+          "legendFormat": "fresh crl",
+          "refId": "E"
+        }
+      ],
+      "title": "Fresh PP / Object Counts by Run",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": [
+    "ours-rp",
+    "rpki",
+    "soak"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Ours RP Soak Overview",
+  "uid": "ours-rp-soak-overview",
+  "version": 4,
+  "weekStart": ""
+}

monitor/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: ours-rp
+    orgId: 1
+    folder: Ours RP
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards

monitor/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    uid: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: true

monitor/prometheus/prometheus.yml

@@ -0,0 +1,20 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+
+scrape_configs:
+  - job_name: ours-rp-artifact-metrics
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - host.docker.internal:9556
+        labels:
+          rp: ours-rp
+          source: artifact-sidecar
+  - job_name: ours-rp-inter-rp-metrics
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - host.docker.internal:9557
+        labels:
+          source: inter-rp-sidecar

scripts/benchmark/README.md

@@ -0,0 +1,70 @@
+# RPKI Benchmarks (Stage2, selected_der_v2)
+
+This directory contains a reproducible, one-click benchmark to measure **decode + profile validate**
+performance for all supported object types and compare **OURS** against the **Routinator baseline**
+(`rpki` crate `=0.19.1` with `repository` feature).
+
+## What it measures
+
+Dataset:
+
+- Fixtures: `rpki/tests/benchmark/selected_der_v2/`
+- Objects: `cer`, `crl`, `manifest` (`.mft`), `roa`, `aspa` (`.asa`)
+- Samples: 10 quantiles per type (`min/p01/p10/p25/p50/p75/p90/p95/p99/max`) → 50 files total
+
+Metrics:
+
+- **decode+validate**: `decode_der` (parse + profile validate) for each object file
+- **landing** (OURS only): `PackFile::from_bytes_compute_sha256` + CBOR encode + `RocksDB put_raw`
+- **compare**: ratio `ours_ns/op ÷ rout_ns/op` for decode+validate
+
+## Default benchmark settings
+
+Both OURS and Routinator baseline use the same run settings:
+
+- warmup: `10` iterations
+- rounds: `3`
+- adaptive loop target: `min_round_ms=200` (with an internal max of `1_000_000` iters)
+- strict DER: `true` (baseline)
+- cert inspect: `false` (baseline)
+
+You can override the settings via environment variables in the runner script:
+
+- `BENCH_WARMUP_ITERS` (default `10`)
+- `BENCH_ROUNDS` (default `3`)
+- `BENCH_MIN_ROUND_MS` (default `200`)
+
+## One-click run (OURS + Routinator compare)
+
+From the `rpki/` crate directory:
+
+```bash
+./scripts/benchmark/run_stage2_selected_der_v2_release.sh
+```
+
+Outputs are written under:
+
+- `rpki/target/bench/`
+  - OURS decode+validate: `stage2_selected_der_v2_decode_release_<TS>.{md,csv}`
+  - OURS landing: `stage2_selected_der_v2_landing_release_<TS>.{md,csv}`
+  - Routinator: `stage2_selected_der_v2_routinator_decode_release_<TS>.{md,csv}`
+  - Compare: `stage2_selected_der_v2_compare_ours_vs_routinator_decode_release_<TS>.{md,csv}`
+  - Summary: `stage2_selected_der_v2_compare_summary_<TS>.md`
+
+### Why decode and landing are separated
+
+The underlying benchmark can run in `BENCH_MODE=both`, but the **landing** part writes to RocksDB
+and may trigger background work (e.g., compactions) that can **skew subsequent decode timings**.
+For a fair OURS-vs-Routinator comparison, the runner script:
+
+- runs `BENCH_MODE=decode_validate` for comparison, and
+- runs `BENCH_MODE=landing` separately for landing-only numbers.
+
+## Notes
+
+- The Routinator baseline benchmark is implemented in-repo under:
+  - `rpki/benchmark/routinator_object_bench/`
+  - It pins `rpki = "=0.19.1"` in its `Cargo.toml`.
+- This benchmark is implemented as an `#[ignore]` integration test:
+  - `rpki/tests/bench_stage2_decode_profile_selected_der_v2.rs`
+  - The runner script invokes it with `cargo test --release ... -- --ignored --nocapture`.

scripts/benchmark/run_stage2_selected_der_v2_release.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Stage2 (selected_der_v2) decode+profile validate benchmark.
+# Runs:
+#  1) OURS decode+validate benchmark and writes MD/CSV.
+#  2) OURS landing benchmark and writes MD/CSV.
+#  3) Routinator baseline decode benchmark (rpki crate =0.19.1).
+#  4) Produces a joined compare CSV/MD and a short geomean summary.
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+OUT_DIR="$ROOT_DIR/target/bench"
+mkdir -p "$OUT_DIR"
+
+TS="$(date -u +%Y%m%dT%H%M%SZ)"
+
+WARMUP_ITERS="${BENCH_WARMUP_ITERS:-10}"
+ROUNDS="${BENCH_ROUNDS:-3}"
+MIN_ROUND_MS="${BENCH_MIN_ROUND_MS:-200}"
+
+OURS_MD="$OUT_DIR/stage2_selected_der_v2_decode_release_${TS}.md"
+OURS_CSV="$OUT_DIR/stage2_selected_der_v2_decode_release_${TS}.csv"
+
+OURS_LANDING_MD="$OUT_DIR/stage2_selected_der_v2_landing_release_${TS}.md"
+OURS_LANDING_CSV="$OUT_DIR/stage2_selected_der_v2_landing_release_${TS}.csv"
+
+ROUT_MD="$OUT_DIR/stage2_selected_der_v2_routinator_decode_release_${TS}.md"
+ROUT_CSV="$OUT_DIR/stage2_selected_der_v2_routinator_decode_release_${TS}.csv"
+
+COMPARE_MD="$OUT_DIR/stage2_selected_der_v2_compare_ours_vs_routinator_decode_release_${TS}.md"
+COMPARE_CSV="$OUT_DIR/stage2_selected_der_v2_compare_ours_vs_routinator_decode_release_${TS}.csv"
+
+SUMMARY_MD="$OUT_DIR/stage2_selected_der_v2_compare_summary_${TS}.md"
+
+echo "[1/4] OURS: decode+validate benchmark (release)..." >&2
+BENCH_MODE="decode_validate" \
+BENCH_WARMUP_ITERS="$WARMUP_ITERS" \
+BENCH_ROUNDS="$ROUNDS" \
+BENCH_MIN_ROUND_MS="$MIN_ROUND_MS" \
+BENCH_OUT_MD="$OURS_MD" \
+BENCH_OUT_CSV="$OURS_CSV" \
+  cargo test --release --test bench_stage2_decode_profile_selected_der_v2 -- --ignored --nocapture >/dev/null
+
+echo "[2/4] OURS: landing benchmark (release)..." >&2
+BENCH_MODE="landing" \
+BENCH_WARMUP_ITERS="$WARMUP_ITERS" \
+BENCH_ROUNDS="$ROUNDS" \
+BENCH_MIN_ROUND_MS="$MIN_ROUND_MS" \
+BENCH_OUT_MD_LANDING="$OURS_LANDING_MD" \
+BENCH_OUT_CSV_LANDING="$OURS_LANDING_CSV" \
+  cargo test --release --test bench_stage2_decode_profile_selected_der_v2 -- --ignored --nocapture >/dev/null
+
+echo "[3/4] Routinator baseline + compare join..." >&2
+OURS_CSV="$OURS_CSV" \
+ROUT_CSV="$ROUT_CSV" \
+ROUT_MD="$ROUT_MD" \
+COMPARE_CSV="$COMPARE_CSV" \
+COMPARE_MD="$COMPARE_MD" \
+WARMUP_ITERS="$WARMUP_ITERS" \
+ROUNDS="$ROUNDS" \
+MIN_ROUND_MS="$MIN_ROUND_MS" \
+  scripts/stage2_perf_compare_m4.sh >/dev/null
+
+echo "[4/4] Summary (geomean ratios)..." >&2
+python3 - "$COMPARE_CSV" "$SUMMARY_MD" <<'PY'
+import csv
+import math
+import sys
+from pathlib import Path
+from datetime import datetime, timezone
+
+in_csv = Path(sys.argv[1])
+out_md = Path(sys.argv[2])
+
+rows = list(csv.DictReader(in_csv.open(newline="")))
+ratios = {}
+for r in rows:
+    ratios.setdefault(r["type"], []).append(float(r["ratio_ours_over_rout"]))
+
+def geomean(vals):
+    return math.exp(sum(math.log(v) for v in vals) / len(vals))
+
+def p50(vals):
+    v = sorted(vals)
+    n = len(v)
+    if n % 2 == 1:
+        return v[n // 2]
+    return (v[n // 2 - 1] + v[n // 2]) / 2.0
+
+all_vals = [float(r["ratio_ours_over_rout"]) for r in rows]
+types = ["all"] + sorted(ratios.keys())
+
+now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+lines = []
+lines.append("# Stage2 selected_der_v2 compare summary (release)\n\n")
+lines.append(f"- recorded_at_utc: `{now}`\n")
+lines.append(f"- inputs_csv: `{in_csv}`\n\n")
+lines.append("| type | n | min | p50 | geomean | max | >1 count |\n")
+lines.append("|---|---:|---:|---:|---:|---:|---:|\n")
+
+for t in types:
+    vals = all_vals if t == "all" else ratios[t]
+    vals_sorted = sorted(vals)
+    lines.append(
+        f"| {t} | {len(vals_sorted)} | {vals_sorted[0]:.4f} | {p50(vals_sorted):.4f} | "
+        f"{geomean(vals_sorted):.4f} | {vals_sorted[-1]:.4f} | {sum(1 for v in vals_sorted if v>1.0)} |\n"
+    )
+
+out_md.write_text("".join(lines), encoding="utf-8")
+print(out_md)
+PY
+
+echo "Done." >&2
+echo "- OURS decode MD:   $OURS_MD" >&2
+echo "- OURS decode CSV:  $OURS_CSV" >&2
+echo "- OURS landing MD:  $OURS_LANDING_MD" >&2
+echo "- OURS landing CSV: $OURS_LANDING_CSV" >&2
+echo "- Routinator:       $ROUT_MD" >&2
+echo "- Compare MD:       $COMPARE_MD" >&2
+echo "- Compare CSV:      $COMPARE_CSV" >&2
+echo "- Summary MD:       $SUMMARY_MD" >&2

scripts/benchmark/run_sync_worker_ablation.py

@@ -0,0 +1,667 @@
+#!/usr/bin/env python3
+"""Run ours RP sync-worker-count ablation on a remote host.
+
+The experiment intentionally mirrors the Feature #100 ours-pp-object-cache
+profile and changes only --parallel-max-repo-sync-workers-global.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import html
+import json
+import os
+import re
+import shlex
+import subprocess
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+PRODUCT_ROOT = Path(__file__).resolve().parents[2]
+DEFAULT_REMOTE = "root@47.77.204.233"
+DEFAULT_WORKERS = "1,2,4,8"
+DEFAULT_INTERVAL_SECS = 0
+DEFAULT_RUNS_PER_WORKER = 5
+
+RIRS = ["afrinic", "apnic", "arin", "lacnic", "ripe"]
+TAL_URLS = {
+    "afrinic": "https://rpki.afrinic.net/tal/afrinic.tal",
+    "apnic": "https://tal.apnic.net/apnic.tal",
+    "arin": "https://www.arin.net/resources/manage/rpki/arin.tal",
+    "lacnic": "https://www.lacnic.net/innovaportal/file/4983/1/lacnic.tal",
+    "ripe": "https://tal.rpki.ripe.net/ripe-ncc.tal",
+}
+
+
+REMOTE_RUNNER = r"""#!/usr/bin/env bash
+set -euo pipefail
+mode="$1"
+state_dir="$2"
+run_dir="$3"
+shift 3
+mkdir -p "$run_dir"
+if [[ "$mode" == "snapshot" ]]; then
+  rm -rf "$state_dir"
+fi
+mkdir -p "$state_dir" "$(dirname "$run_dir")"
+date -u +"%Y-%m-%dT%H:%M:%SZ" > "$run_dir/start-time.txt"
+printf '%q ' "$@" > "$run_dir/command.txt"
+printf '\n' >> "$run_dir/command.txt"
+set +e
+/usr/bin/time -v -o "$run_dir/process-time.txt" "$@" > "$run_dir/stdout.log" 2> "$run_dir/stderr.log"
+status=$?
+set -e
+date -u +"%Y-%m-%dT%H:%M:%SZ" > "$run_dir/end-time.txt"
+printf '%s\n' "$status" > "$run_dir/exit-code.txt"
+exit "$status"
+"""
+
+
+REMOTE_METRICS = r"""import csv
+import json
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+run_dir = Path(os.environ["RUN_DIR"])
+state_dir = Path(os.environ["STATE_DIR"])
+workers = int(os.environ["WORKERS"])
+mode = os.environ["MODE"]
+run_index = int(os.environ["RUN_INDEX"])
+scheduled_epoch = float(os.environ["SCHEDULED_EPOCH"])
+started_epoch = float(os.environ["STARTED_EPOCH"])
+
+
+def read_text(path):
+    try:
+        return Path(path).read_text(errors="replace")
+    except FileNotFoundError:
+        return ""
+
+
+def read_json(path):
+    try:
+        with Path(path).open() as fh:
+            return json.load(fh)
+    except FileNotFoundError:
+        return None
+    except json.JSONDecodeError as exc:
+        return {"_json_error": str(exc)}
+
+
+def parse_time(path):
+    text = read_text(path)
+    out = {}
+    match = re.search(r"Elapsed \(wall clock\) time.*: (?:(\d+):)?(\d+):(\d+(?:\.\d+)?)", text)
+    if match:
+        hours = int(match.group(1) or 0)
+        minutes = int(match.group(2))
+        seconds = float(match.group(3))
+        out["wall_ms_timev"] = int(round(((hours * 60 + minutes) * 60 + seconds) * 1000))
+    for name, key in [
+        ("Maximum resident set size \\(kbytes\\)", "max_rss_kb"),
+        ("Percent of CPU this job got", "cpu_percent_text"),
+        ("Major \\(requiring I/O\\) page faults", "major_faults"),
+        ("Minor \\(reclaiming a frame\\) page faults", "minor_faults"),
+    ]:
+        found = re.search(name + r":\s+(.+)", text)
+        if found:
+            value = found.group(1).strip()
+            if key.endswith("_faults") or key == "max_rss_kb":
+                try:
+                    out[key] = int(value)
+                except ValueError:
+                    out[key] = value
+            else:
+                out[key] = value
+    return out
+
+
+def count_csv(path):
+    rows = 0
+    unique = set()
+    try:
+        with Path(path).open(newline="") as fh:
+            reader = csv.reader(fh)
+            header = next(reader, None)
+            for row in reader:
+                if not row:
+                    continue
+                rows += 1
+                unique.add(tuple(row))
+    except FileNotFoundError:
+        return {"rows": None, "unique_rows": None}
+    return {"rows": rows, "unique_rows": len(unique)}
+
+
+def dir_bytes(path):
+    path = Path(path)
+    if not path.exists():
+        return 0
+    total = 0
+    for item in path.rglob("*"):
+        try:
+            if item.is_file():
+                total += item.stat().st_size
+        except OSError:
+            pass
+    return total
+
+
+def file_bytes(path):
+    try:
+        return Path(path).stat().st_size
+    except FileNotFoundError:
+        return None
+
+
+stage = read_json(run_dir / "stage-timing.json") or {}
+process_time = parse_time(run_dir / "process-time.txt")
+exit_text = read_text(run_dir / "exit-code.txt").strip()
+start_text = read_text(run_dir / "start-time.txt").strip()
+end_text = read_text(run_dir / "end-time.txt").strip()
+
+df = subprocess.run(["df", "-P", "/"], text=True, capture_output=True)
+df_line = df.stdout.strip().splitlines()[-1].split() if df.returncode == 0 and len(df.stdout.strip().splitlines()) >= 2 else []
+df_root = {}
+if len(df_line) >= 5:
+    df_root = {
+        "filesystem": df_line[0],
+        "blocks_1k": int(df_line[1]),
+        "used_1k": int(df_line[2]),
+        "available_1k": int(df_line[3]),
+        "use_percent": int(df_line[4].rstrip("%")),
+    }
+
+metrics = {
+    "workers": workers,
+    "mode": mode,
+    "run_index": run_index,
+    "scheduled_epoch": scheduled_epoch,
+    "started_epoch": started_epoch,
+    "schedule_lag_ms": int(round((started_epoch - scheduled_epoch) * 1000)),
+    "start_time_utc": start_text,
+    "end_time_utc": end_text,
+    "exit_code": int(exit_text) if exit_text.isdigit() else None,
+    "run_dir": str(run_dir),
+    "state_dir": str(state_dir),
+    "stage": stage,
+    "process_time": process_time,
+    "vrps": count_csv(run_dir / "vrps.csv"),
+    "vaps": count_csv(run_dir / "vaps.csv"),
+    "artifact_bytes": {
+        "report_json": file_bytes(run_dir / "report.json"),
+        "result_ccr": file_bytes(run_dir / "result.ccr"),
+        "input_cir": file_bytes(run_dir / "input.cir"),
+        "vrps_csv": file_bytes(run_dir / "vrps.csv"),
+        "vaps_csv": file_bytes(run_dir / "vaps.csv"),
+    },
+    "state_bytes": dir_bytes(state_dir),
+    "run_bytes": dir_bytes(run_dir),
+    "df_root": df_root,
+}
+print(json.dumps(metrics, ensure_ascii=False, sort_keys=True))
+"""
+
+
+def utc_stamp() -> str:
+    return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+
+
+def run_cmd(cmd: list[str], *, cwd: Path | None = None, check: bool = True, input_text: str | None = None) -> subprocess.CompletedProcess[str]:
+    result = subprocess.run(cmd, cwd=cwd, input=input_text, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    if check and result.returncode != 0:
+        sys.stderr.write(result.stdout)
+        sys.stderr.write(result.stderr)
+        raise SystemExit(result.returncode)
+    return result
+
+
+def ssh(remote: str, command: str, *, check: bool = True, input_text: str | None = None) -> subprocess.CompletedProcess[str]:
+    return run_cmd(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", remote, command], check=check, input_text=input_text)
+
+
+def scp_to(remote: str, local: Path, remote_path: str) -> None:
+    ssh(remote, "mkdir -p " + shlex.quote(str(Path(remote_path).parent)))
+    run_cmd(["scp", "-q", str(local), f"{remote}:{remote_path}"])
+
+
+def parse_workers(value: str) -> list[int]:
+    workers = []
+    for part in value.split(","):
+        part = part.strip()
+        if not part:
+            continue
+        worker = int(part)
+        if worker <= 0:
+            raise argparse.ArgumentTypeError("worker count must be positive")
+        workers.append(worker)
+    if len(set(workers)) != len(workers):
+        raise argparse.ArgumentTypeError("worker counts must be unique")
+    return workers
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Run ours RP sync worker ablation.")
+    parser.add_argument("--remote", default=DEFAULT_REMOTE)
+    parser.add_argument("--workers", type=parse_workers, default=parse_workers(DEFAULT_WORKERS))
+    parser.add_argument("--interval-secs", type=int, default=DEFAULT_INTERVAL_SECS)
+    parser.add_argument("--runs-per-worker", type=int, default=DEFAULT_RUNS_PER_WORKER)
+    parser.add_argument("--remote-root")
+    parser.add_argument("--out-root", type=Path, default=Path("specs/develop/20260714/feature115_sync_worker_ablation_runs"))
+    parser.add_argument("--binary", type=Path)
+    parser.add_argument("--skip-build", action="store_true")
+    parser.add_argument("--dry-run", action="store_true")
+    return parser.parse_args()
+
+
+def ensure_binary(args: argparse.Namespace) -> Path:
+    if args.binary:
+        binary = args.binary.resolve()
+        if not binary.exists():
+            raise SystemExit(f"binary does not exist: {binary}")
+        return binary
+    binary = PRODUCT_ROOT / "target" / "release" / "rpki"
+    if not args.skip_build:
+        print("building release rpki binary", flush=True)
+        run_cmd(["cargo", "build", "--release", "--bin", "rpki"], cwd=PRODUCT_ROOT)
+    if not binary.exists():
+        raise SystemExit(f"release binary not found: {binary}")
+    return binary
+
+
+def fixed_args(remote_bin: str, state_dir: str, run_dir: str, workers: int) -> list[str]:
+    args = [
+        remote_bin,
+        "--db", f"{state_dir}/work-db",
+        "--repo-bytes-db", f"{state_dir}/repo-bytes.db",
+        "--rsync-mirror-root", f"{state_dir}/rsync-mirror",
+        "--rsync-scope", "module-root",
+        "--report-json", f"{run_dir}/report.json",
+        "--report-json-compact",
+        "--ccr-out", f"{run_dir}/result.ccr",
+        "--cir-enable",
+        "--cir-out", f"{run_dir}/input.cir",
+        "--vrps-csv-out", f"{run_dir}/vrps.csv",
+        "--vaps-csv-out", f"{run_dir}/vaps.csv",
+        "--compare-view-trust-anchor", "all5",
+        "--parallel-phase2-ready-batch-size", "256",
+        "--parallel-phase2-ready-batch-wall-time-budget-ms", "100",
+        "--parallel-phase2-result-drain-batch-size", "2048",
+        "--parallel-phase2-finalize-batch-size", "256",
+        "--parallel-phase2-finalize-batch-wall-time-budget-ms", "100",
+        "--parallel-max-repo-sync-workers-global", str(workers),
+        "--enable-publication-point-validation-cache",
+        "--enable-roa-validation-cache",
+        "--enable-child-certificate-validation-cache",
+    ]
+    for rir in RIRS:
+        url = TAL_URLS[rir]
+        args.extend(["--tal-url", url, "--cir-tal-uri", url])
+    return args
+
+
+def plan_runs(workers: list[int], runs_per_worker: int, interval_secs: int, remote_root: str, remote_bin: str) -> list[dict[str, Any]]:
+    if runs_per_worker < 2:
+        raise SystemExit("--runs-per-worker must be at least 2")
+    planned = []
+    ordinal = 0
+    for worker in workers:
+        case_root = f"{remote_root}/cases/workers-{worker:02d}"
+        state_dir = f"{case_root}/state"
+        for run_index in range(1, runs_per_worker + 1):
+            mode = "snapshot" if run_index == 1 else "delta"
+            run_dir = f"{case_root}/runs/run_{run_index:04d}_{mode}"
+            planned.append({
+                "ordinal": ordinal,
+                "workers": worker,
+                "mode": mode,
+                "run_index": run_index,
+                "state_dir": state_dir,
+                "run_dir": run_dir,
+                "interval_secs": interval_secs,
+                "command": fixed_args(remote_bin, state_dir, run_dir, worker),
+            })
+            ordinal += 1
+    return planned
+
+
+def write_json(path: Path, data: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, indent=2, ensure_ascii=False, sort_keys=True) + "\n")
+
+
+def write_text(path: Path, text: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(text)
+
+
+def prepare_remote(remote: str, remote_root: str, binary: Path) -> str:
+    remote_bin = f"{remote_root}/bin/rpki"
+    preflight = f"""
+set -euo pipefail
+mkdir -p {shlex.quote(remote_root)}/bin {shlex.quote(remote_root)}/env
+systemctl disable --now rpki-client.timer >/dev/null 2>&1 || true
+systemctl stop rpki-client.service >/dev/null 2>&1 || true
+pkill -x rpki-client >/dev/null 2>&1 || true
+pkill -x routinator >/dev/null 2>&1 || true
+pkill -x fort >/dev/null 2>&1 || true
+pkill -x rpki >/dev/null 2>&1 || true
+{{
+  date -u +"%Y-%m-%dT%H:%M:%SZ"
+  hostname
+  uname -a
+  nproc
+  df -h /
+  free -h || true
+  ps -eo pid,ppid,comm,%cpu,%mem,args --sort=-%cpu | head -30
+  systemctl is-active rpki-client.timer 2>/dev/null || true
+}} > {shlex.quote(remote_root)}/env/preflight.txt
+"""
+    ssh(remote, "bash -s", input_text=preflight)
+    scp_to(remote, binary, remote_bin)
+    ssh(remote, "chmod +x " + shlex.quote(remote_bin))
+    return remote_bin
+
+
+def run_remote_plan(remote: str, plan: dict[str, Any], scheduled_epoch: float) -> dict[str, Any]:
+    now = time.time()
+    if now < scheduled_epoch:
+        wait_secs = int(round(scheduled_epoch - now))
+        print(f"waiting {wait_secs}s before workers={plan['workers']} {plan['mode']}", flush=True)
+        time.sleep(scheduled_epoch - now)
+    started_epoch = time.time()
+    command = shlex.join(["bash", "-s", "--", plan["mode"], plan["state_dir"], plan["run_dir"], *plan["command"]])
+    print(f"start workers={plan['workers']} {plan['mode']} run={plan['run_index']} at {datetime.now(timezone.utc).isoformat()}", flush=True)
+    result = ssh(remote, command, input_text=REMOTE_RUNNER, check=False)
+    if result.returncode != 0:
+        sys.stderr.write(result.stdout)
+        sys.stderr.write(result.stderr)
+    metrics_script = "\n".join([
+        f"export RUN_DIR={shlex.quote(plan['run_dir'])}",
+        f"export STATE_DIR={shlex.quote(plan['state_dir'])}",
+        f"export WORKERS={plan['workers']}",
+        f"export MODE={shlex.quote(plan['mode'])}",
+        f"export RUN_INDEX={plan['run_index']}",
+        f"export SCHEDULED_EPOCH={scheduled_epoch}",
+        f"export STARTED_EPOCH={started_epoch}",
+        "python3 - <<'PY'",
+        REMOTE_METRICS,
+        "PY",
+    ])
+    metrics_result = ssh(remote, "bash -s", input_text=metrics_script, check=True)
+    metrics = json.loads(metrics_result.stdout)
+    metrics["ssh_return_code"] = result.returncode
+    metrics["command_digest"] = hashlib.sha256("\0".join(plan["command"]).encode()).hexdigest()
+    ssh(
+        remote,
+        "cat > " + shlex.quote(plan["run_dir"] + "/metrics.json"),
+        input_text=json.dumps(metrics, ensure_ascii=False, sort_keys=True) + "\n",
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"remote run failed: workers={plan['workers']} mode={plan['mode']} rc={result.returncode}")
+    validate_metrics(metrics)
+    print(f"done workers={plan['workers']} {plan['mode']} wall={metrics.get('wall_ms')}ms vrps={metrics['vrps'].get('rows')} vaps={metrics['vaps'].get('rows')}", flush=True)
+    return metrics
+
+
+def validate_metrics(metrics: dict[str, Any]) -> None:
+    stage = metrics.get("stage") or {}
+    expected = {
+        "enable_transport_request_prefetch": False,
+        "enable_publication_point_validation_cache": True,
+        "enable_roa_validation_cache": True,
+        "enable_child_certificate_validation_cache": True,
+    }
+    for key, value in expected.items():
+        if stage.get(key) is not value:
+            raise RuntimeError(f"unexpected {key}: {stage.get(key)!r}, expected {value!r}")
+    if metrics["vrps"].get("rows") in (None, 0):
+        raise RuntimeError("VRP CSV is missing or empty")
+    if metrics["vaps"].get("rows") is None:
+        raise RuntimeError("VAP CSV is missing")
+    df_root = metrics.get("df_root") or {}
+    if df_root.get("use_percent", 0) >= 90:
+        raise RuntimeError(f"remote disk is above threshold: {df_root}")
+    stage_total = stage.get("total_ms")
+    timev_total = metrics.get("process_time", {}).get("wall_ms_timev")
+    metrics["wall_ms"] = stage_total if isinstance(stage_total, int) else timev_total
+
+
+def append_metric(out_root: Path, metrics: dict[str, Any]) -> None:
+    path = out_root / "per_run_metrics.jsonl"
+    with path.open("a") as fh:
+        fh.write(json.dumps(metrics, ensure_ascii=False, sort_keys=True) + "\n")
+
+
+def load_metrics(out_root: Path) -> list[dict[str, Any]]:
+    path = out_root / "per_run_metrics.jsonl"
+    if not path.exists():
+        return []
+    return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
+
+
+def metric_row(metric: dict[str, Any]) -> dict[str, Any]:
+    stage = metric.get("stage") or {}
+    roa_cache = stage.get("roa_validation_cache") or {}
+    return {
+        "workers": metric["workers"],
+        "mode": metric["mode"],
+        "run_index": metric["run_index"],
+        "start_time_utc": metric.get("start_time_utc"),
+        "wall_ms": metric.get("wall_ms"),
+        "timev_wall_ms": metric.get("process_time", {}).get("wall_ms_timev"),
+        "max_rss_kb": metric.get("process_time", {}).get("max_rss_kb"),
+        "cpu_percent": metric.get("process_time", {}).get("cpu_percent_text"),
+        "validation_ms": stage.get("validation_ms"),
+        "repo_sync_ms_total": stage.get("repo_sync_ms_total"),
+        "rrdp_download_ms_total": stage.get("rrdp_download_ms_total"),
+        "rsync_download_ms_total": stage.get("rsync_download_ms_total"),
+        "download_bytes_total": stage.get("download_bytes_total"),
+        "publication_points": stage.get("publication_points"),
+        "pp_cache_hit_ratio": stage.get("publication_point_cache_hit_ratio"),
+        "roa_hit_roas": roa_cache.get("hit_roas"),
+        "roa_miss_roas": roa_cache.get("miss_roas"),
+        "vrps_rows": metric.get("vrps", {}).get("rows"),
+        "vrps_unique_rows": metric.get("vrps", {}).get("unique_rows"),
+        "vaps_rows": metric.get("vaps", {}).get("rows"),
+        "vaps_unique_rows": metric.get("vaps", {}).get("unique_rows"),
+        "state_bytes": metric.get("state_bytes"),
+        "run_bytes": metric.get("run_bytes"),
+        "remote_run_dir": metric.get("run_dir"),
+    }
+
+
+def write_csv_report(out_root: Path, metrics: list[dict[str, Any]]) -> None:
+    rows = [metric_row(item) for item in metrics]
+    if not rows:
+        return
+    with (out_root / "per_run_metrics.csv").open("w", newline="") as fh:
+        writer = csv.DictWriter(fh, fieldnames=list(rows[0]))
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def fmt_ms(value: Any) -> str:
+    if value is None:
+        return "-"
+    try:
+        return f"{float(value) / 1000:.2f}s"
+    except (TypeError, ValueError):
+        return str(value)
+
+
+def fmt_bytes(value: Any) -> str:
+    if value is None:
+        return "-"
+    try:
+        value = float(value)
+    except (TypeError, ValueError):
+        return str(value)
+    for unit in ["B", "KiB", "MiB", "GiB"]:
+        if value < 1024 or unit == "GiB":
+            return f"{value:.1f} {unit}"
+        value /= 1024
+    return str(value)
+
+
+def write_markdown_report(out_root: Path, metrics: list[dict[str, Any]], provenance: dict[str, Any]) -> None:
+    rows = [metric_row(item) for item in metrics]
+    lines = [
+        "# Feature #115 Sync Worker Ablation Summary",
+        "",
+        "## Experiment",
+        "",
+        f"- Remote: {provenance['remote']}",
+        f"- Remote root: {provenance['remote_root']}",
+        f"- Local commit: {provenance['git_commit']}",
+        f"- Worker counts: {', '.join(str(w) for w in provenance['workers'])}",
+        f"- Runs per worker: {provenance['runs_per_worker']}",
+        f"- Interval seconds: {provenance['interval_secs']}",
+        "- Fixed profile: all5 live TAL URL, module-root rsync scope, PP cache enabled, ROA cache enabled, child-certificate cache enabled, transport prefetch disabled.",
+        "",
+        "## Per Run",
+        "",
+        "| workers | mode | wall | rss | validation | repo sync | RRDP | rsync | download | PP | VRPs | VAPs |",
+        "|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|",
+    ]
+    for row in rows:
+        lines.append(
+            f"| {row['workers']} | {row['mode']} | {fmt_ms(row['wall_ms'])} | "
+            f"{fmt_bytes((row['max_rss_kb'] or 0) * 1024 if row['max_rss_kb'] else None)} | "
+            f"{fmt_ms(row['validation_ms'])} | {fmt_ms(row['repo_sync_ms_total'])} | "
+            f"{fmt_ms(row['rrdp_download_ms_total'])} | {fmt_ms(row['rsync_download_ms_total'])} | "
+            f"{fmt_bytes(row['download_bytes_total'])} | {row['publication_points']} | "
+            f"{row['vrps_rows']} | {row['vaps_rows']} |"
+        )
+    lines.extend(["", "## Artifacts", "", f"- Per-run metrics JSONL: {out_root / 'per_run_metrics.jsonl'}", f"- Per-run metrics CSV: {out_root / 'per_run_metrics.csv'}"])
+    write_text(out_root / "summary.md", "\n".join(lines) + "\n")
+
+
+def write_html_report(out_root: Path, metrics: list[dict[str, Any]], provenance: dict[str, Any]) -> None:
+    rows = [metric_row(item) for item in metrics]
+    if not rows:
+        return
+    max_wall = max(float(row["wall_ms"] or 0) for row in rows) or 1
+    bars = []
+    for row in rows:
+        width = max(1, int((float(row["wall_ms"] or 0) / max_wall) * 100))
+        bars.append(
+            f"<tr><td>{row['workers']}</td><td>{html.escape(str(row['mode']))}</td>"
+            f"<td>{fmt_ms(row['wall_ms'])}</td><td><div class='bar'><span style='width:{width}%'></span></div></td>"
+            f"<td>{fmt_bytes((row['max_rss_kb'] or 0) * 1024 if row['max_rss_kb'] else None)}</td>"
+            f"<td>{fmt_ms(row['validation_ms'])}</td><td>{fmt_ms(row['repo_sync_ms_total'])}</td>"
+            f"<td>{row['publication_points']}</td><td>{row['vrps_rows']}</td><td>{row['vaps_rows']}</td></tr>"
+        )
+    html_doc = f"""<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>Feature #115 Sync Worker Ablation</title>
+<style>
+body {{ font-family: system-ui, -apple-system, Segoe UI, sans-serif; margin: 32px; color: #1f2933; }}
+h1, h2 {{ margin-bottom: 0.35rem; }}
+.meta {{ display: grid; grid-template-columns: repeat(4, minmax(160px, 1fr)); gap: 12px; margin: 20px 0; }}
+.card {{ border: 1px solid #d8dee9; border-radius: 8px; padding: 14px; background: #f8fafc; }}
+.card b {{ display: block; font-size: 12px; color: #52606d; text-transform: uppercase; letter-spacing: .04em; }}
+.card span {{ font-size: 18px; }}
+table {{ border-collapse: collapse; width: 100%; margin-top: 12px; font-size: 13px; }}
+th, td {{ border: 1px solid #d8dee9; padding: 8px; text-align: left; vertical-align: middle; }}
+th {{ background: #eef2f7; }}
+.bar {{ width: 180px; height: 12px; background: #edf2f7; border-radius: 999px; overflow: hidden; }}
+.bar span {{ display: block; height: 100%; background: #2563eb; }}
+code {{ background: #edf2f7; padding: 2px 5px; border-radius: 4px; }}
+</style>
+</head>
+<body>
+<h1>Feature #115 Sync Worker Ablation</h1>
+<p>This report compares only the global repo-sync worker count while holding the Feature #100 ours-cache profile fixed.</p>
+<section class="meta">
+<div class="card"><b>Remote</b><span>{html.escape(provenance['remote'])}</span></div>
+<div class="card"><b>Remote Root</b><span>{html.escape(provenance['remote_root'])}</span></div>
+<div class="card"><b>Workers</b><span>{html.escape(', '.join(str(w) for w in provenance['workers']))}</span></div>
+<div class="card"><b>Runs</b><span>{provenance['runs_per_worker']} per worker, {provenance['interval_secs']}s interval</span></div>
+</section>
+<h2>Per Run Metrics</h2>
+<table>
+<thead><tr><th>Workers</th><th>Mode</th><th>Wall</th><th>Wall Bar</th><th>Max RSS</th><th>Validation</th><th>Repo Sync</th><th>PP</th><th>VRPs</th><th>VAPs</th></tr></thead>
+<tbody>
+{''.join(bars)}
+</tbody>
+</table>
+<h2>Configuration</h2>
+<p>all5 live TAL URL, module-root rsync scope, PP cache enabled, ROA cache enabled, child-certificate cache enabled, transport prefetch disabled.</p>
+<p>Local commit: <code>{html.escape(provenance['git_commit'])}</code>. Full JSONL and CSV evidence are stored beside this report.</p>
+</body>
+</html>
+"""
+    write_text(out_root / "summary.html", html_doc)
+
+
+def write_reports(out_root: Path, provenance: dict[str, Any]) -> None:
+    metrics = load_metrics(out_root)
+    write_csv_report(out_root, metrics)
+    write_markdown_report(out_root, metrics, provenance)
+    write_html_report(out_root, metrics, provenance)
+
+
+def git_metadata() -> dict[str, str]:
+    commit = run_cmd(["git", "rev-parse", "--short", "HEAD"], cwd=PRODUCT_ROOT).stdout.strip()
+    status = run_cmd(["git", "status", "--short"], cwd=PRODUCT_ROOT).stdout
+    return {"git_commit": commit, "git_status_short": status}
+
+
+def main() -> int:
+    args = parse_args()
+    stamp = utc_stamp()
+    out_root = (args.out_root / stamp).resolve()
+    remote_root = args.remote_root or f"/root/rpki_feature115_sync_worker_ablation_{stamp}"
+    remote_bin = f"{remote_root}/bin/rpki"
+    plans = plan_runs(args.workers, args.runs_per_worker, args.interval_secs, remote_root, remote_bin)
+
+    provenance = {
+        "created_at_utc": stamp,
+        "remote": args.remote,
+        "remote_root": remote_root,
+        "workers": args.workers,
+        "runs_per_worker": args.runs_per_worker,
+        "interval_secs": args.interval_secs,
+        "fixed_profile": "feature100 ours-pp-object-cache",
+        **git_metadata(),
+    }
+    write_json(out_root / "provenance.json", provenance)
+    write_text(out_root / "remote_root.txt", remote_root + "\n")
+    write_json(out_root / "execution_plan.json", plans)
+
+    if args.dry_run:
+        print(f"dry-run plan written to {out_root}")
+        return 0
+
+    binary = ensure_binary(args)
+    remote_bin = prepare_remote(args.remote, remote_root, binary)
+    plans = plan_runs(args.workers, args.runs_per_worker, args.interval_secs, remote_root, remote_bin)
+    write_json(out_root / "execution_plan.json", plans)
+    start_epoch = time.time()
+    for plan in plans:
+        write_json(
+            out_root / "run_configs" / f"workers-{plan['workers']:02d}-run-{plan['run_index']:04d}-{plan['mode']}.json",
+            plan,
+        )
+        scheduled_epoch = start_epoch + plan["ordinal"] * args.interval_secs
+        metrics = run_remote_plan(args.remote, plan, scheduled_epoch)
+        append_metric(out_root, metrics)
+        write_reports(out_root, provenance)
+    print(f"reports written under {out_root}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/ccr/check_rpki_client_ccr.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  check_rpki_client_ccr.sh \
+    --rpki-client <path> \
+    --ccr-root <directory-or-ccr-file> \
+    --out <result.jsonl> \
+    [--limit <count>]
+
+Runs the supplied rpki-client importer against CCR files. A non-zero result
+means that rpki-client reported a CCR-file parsing error or could not start.
+Per-file command output is retained next to the JSONL result file.
+EOF
+}
+
+RPKI_CLIENT=""
+CCR_ROOT=""
+OUT=""
+LIMIT=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rpki-client) RPKI_CLIENT=${2:?missing value for --rpki-client}; shift 2 ;;
+    --ccr-root) CCR_ROOT=${2:?missing value for --ccr-root}; shift 2 ;;
+    --out) OUT=${2:?missing value for --out}; shift 2 ;;
+    --limit) LIMIT=${2:?missing value for --limit}; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "error: unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+[[ -n "$RPKI_CLIENT" && -n "$CCR_ROOT" && -n "$OUT" ]] || {
+  echo "error: --rpki-client, --ccr-root, and --out are required" >&2
+  usage >&2
+  exit 2
+}
+[[ -x "$RPKI_CLIENT" ]] || { echo "error: rpki-client is not executable: $RPKI_CLIENT" >&2; exit 2; }
+[[ "$LIMIT" =~ ^[0-9]+$ ]] || { echo "error: --limit must be a non-negative integer" >&2; exit 2; }
+
+mkdir -p "$(dirname "$OUT")"
+LOG_DIR="${OUT%.jsonl}.logs"
+mkdir -p "$LOG_DIR"
+: > "$OUT"
+
+declare -a CCR_FILES=()
+if [[ -f "$CCR_ROOT" ]]; then
+  CCR_FILES=("$(readlink -f "$CCR_ROOT")")
+elif [[ -d "$CCR_ROOT" ]]; then
+  while IFS= read -r -d '' file; do
+    CCR_FILES+=("$file")
+  done < <(find "$CCR_ROOT" -type f -name '*.ccr' -print0 | sort -z)
+else
+  echo "error: CCR root does not exist: $CCR_ROOT" >&2
+  exit 2
+fi
+
+if (( LIMIT > 0 && ${#CCR_FILES[@]} > LIMIT )); then
+  CCR_FILES=("${CCR_FILES[@]:0:LIMIT}")
+fi
+if (( ${#CCR_FILES[@]} == 0 )); then
+  echo "error: no .ccr files found under: $CCR_ROOT" >&2
+  exit 2
+fi
+
+passed=0
+failed=0
+for ccr in "${CCR_FILES[@]}"; do
+  digest=$(sha256sum "$ccr" | awk '{print $1}')
+  log="$LOG_DIR/${digest}.log"
+  set +e
+  "$RPKI_CLIENT" -f "$ccr" >"$log" 2>&1
+  exit_code=$?
+  set -e
+
+  classification=pass
+  if (( exit_code != 0 )); then
+    classification=tool_error
+  elif grep -Fq "rpki-client: ${ccr}:" "$log"; then
+    classification=ccr_parse_error
+  fi
+  if [[ "$classification" == pass ]]; then
+    ((passed += 1))
+  else
+    ((failed += 1))
+  fi
+
+  python3 - "$OUT" "$ccr" "$digest" "$exit_code" "$classification" "$log" <<'PY'
+import json
+import sys
+
+out, ccr, sha256, exit_code, classification, log = sys.argv[1:]
+with open(out, "a", encoding="utf-8") as stream:
+    stream.write(json.dumps({
+        "ccr": ccr,
+        "sha256": sha256,
+        "rpkiClientExitCode": int(exit_code),
+        "classification": classification,
+        "log": log,
+    }, ensure_ascii=False, sort_keys=True) + "\n")
+PY
+done
+
+echo "checked=${#CCR_FILES[@]} passed=${passed} failed=${failed} jsonl=${OUT} logs=${LOG_DIR}"
+(( failed == 0 ))

scripts/cir/README.md

@@ -0,0 +1,56 @@
+# CIR Scripts
+
+## `cir-rsync-wrapper`
+
+一个用于 CIR 黑盒 replay 的 rsync wrapper。
+
+### 环境变量
+
+- `REAL_RSYNC_BIN`
+  - 真实 rsync 二进制路径
+  - 默认优先 `/usr/bin/rsync`
+- `CIR_MIRROR_ROOT`
+  - 本地镜像树根目录
+  - 当命令行中出现 `rsync://...` source 时必需
+
+### 语义
+
+- 仅改写 `rsync://host/path` 类型参数
+- 其它参数原样透传给真实 rsync
+- 改写目标：
+  - `rsync://example.net/repo/a.roa`
+  - →
+  - `<CIR_MIRROR_ROOT>/example.net/repo/a.roa`
+
+### 兼容目标
+
+- Routinator `--rsync-command`
+- `rpki-client -e rsync_prog`
+
+## 其它脚本
+
+- `run_cir_replay_ours.sh`
+- `run_cir_replay_routinator.sh`
+- `run_cir_replay_rpki_client.sh`
+- `run_cir_replay_matrix.sh`
+
+## `cir-local-link-sync.py`
+
+当 `CIR_LOCAL_LINK_MODE=1` 且 wrapper 检测到 source 已经被改写为本地 mirror 路径时，
+wrapper 不再调用真实 `rsync`，而是调用这个 helper 完成：
+
+- `hardlink` 优先的本地树同步
+- 失败时回退到 copy
+- 支持 `--delete`
+
+`run_cir_replay_matrix.sh` 会顺序执行：
+
+- `ours`
+- Routinator
+- `rpki-client`
+
+并汇总生成：
+
+- `summary.json`
+- `summary.md`
+- `detail.md`

scripts/cir/cir-local-link-sync.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+import argparse
+import errno
+import os
+import shutil
+from pathlib import Path
+
+
+def _same_inode(src: Path, dst: Path) -> bool:
+    try:
+        src_stat = src.stat()
+        dst_stat = dst.stat()
+    except FileNotFoundError:
+        return False
+    return (src_stat.st_dev, src_stat.st_ino) == (dst_stat.st_dev, dst_stat.st_ino)
+
+
+def _remove_path(path: Path) -> None:
+    if not path.exists() and not path.is_symlink():
+        return
+    if path.is_dir() and not path.is_symlink():
+        shutil.rmtree(path)
+    else:
+        path.unlink()
+
+
+def _prune_empty_dirs(root: Path) -> None:
+    if not root.exists():
+        return
+    for path in sorted((p for p in root.rglob("*") if p.is_dir()), key=lambda p: len(p.parts), reverse=True):
+        try:
+            path.rmdir()
+        except OSError:
+            pass
+
+
+def _link_or_copy(src: Path, dst: Path) -> str:
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    if dst.exists() or dst.is_symlink():
+        if _same_inode(src, dst):
+            return "reused"
+        _remove_path(dst)
+    try:
+        os.link(src, dst)
+        return "linked"
+    except OSError as err:
+        if err.errno not in (errno.EXDEV, errno.EPERM, errno.EMLINK, errno.ENOTSUP, errno.EACCES):
+            raise
+    shutil.copy2(src, dst)
+    return "copied"
+
+
+def _file_map(src_arg: str, dest_arg: str) -> tuple[Path, dict[str, Path]]:
+    src = Path(src_arg.rstrip(os.sep))
+    if not src.exists():
+        raise FileNotFoundError(src)
+    mapping: dict[str, Path] = {}
+    if src.is_dir():
+        copy_contents = src_arg.endswith(os.sep)
+        if copy_contents:
+            root = src
+            for path in root.rglob("*"):
+                if path.is_file():
+                    mapping[path.relative_to(root).as_posix()] = path
+        else:
+            root = src
+            base = src.name
+            for path in root.rglob("*"):
+                if path.is_file():
+                    rel = Path(base) / path.relative_to(root)
+                    mapping[rel.as_posix()] = path
+    else:
+        dest_path = Path(dest_arg)
+        if dest_arg.endswith(os.sep) or dest_path.is_dir():
+            mapping[src.name] = src
+        else:
+            mapping[dest_path.name] = src
+    return Path(dest_arg), mapping
+
+
+def sync_local_tree(src_arg: str, dst_arg: str, delete: bool) -> dict[str, int]:
+    dst_root, mapping = _file_map(src_arg, dst_arg)
+    dst_root.mkdir(parents=True, exist_ok=True)
+
+    expected = {dst_root / rel for rel in mapping.keys()}
+
+    deleted = 0
+    if delete and dst_root.exists():
+        for path in sorted(dst_root.rglob("*"), key=lambda p: len(p.parts), reverse=True):
+            if path.is_dir():
+                continue
+            if path not in expected:
+                _remove_path(path)
+                deleted += 1
+        _prune_empty_dirs(dst_root)
+
+    linked = 0
+    copied = 0
+    reused = 0
+    for rel, src in mapping.items():
+        dst = dst_root / rel
+        result = _link_or_copy(src, dst)
+        if result == "linked":
+            linked += 1
+        elif result == "copied":
+            copied += 1
+        else:
+            reused += 1
+
+    return {
+        "files": len(mapping),
+        "linked": linked,
+        "copied": copied,
+        "reused": reused,
+        "deleted": deleted,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Sync a local CIR mirror tree using hardlinks when possible.")
+    parser.add_argument("--delete", action="store_true", help="Delete target files not present in source")
+    parser.add_argument("source")
+    parser.add_argument("dest")
+    args = parser.parse_args()
+
+    summary = sync_local_tree(args.source, args.dest, args.delete)
+    print(
+        "local-link-sync files={files} linked={linked} copied={copied} reused={reused} deleted={deleted}".format(
+            **summary
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/cir/cir-rsync-wrapper

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+import os
+import shutil
+import sys
+from pathlib import Path
+from urllib.parse import urlparse
+
+
+def real_rsync_bin() -> str:
+    env = os.environ.get("REAL_RSYNC_BIN")
+    if env:
+        return env
+    default = "/usr/bin/rsync"
+    if Path(default).exists():
+        return default
+    found = shutil.which("rsync")
+    if found:
+        return found
+    raise SystemExit("cir-rsync-wrapper: REAL_RSYNC_BIN is not set and rsync was not found")
+
+
+def rewrite_arg(arg: str, mirror_root: str | None) -> str:
+    if not arg.startswith("rsync://"):
+        return arg
+    if not mirror_root:
+        raise SystemExit(
+            "cir-rsync-wrapper: CIR_MIRROR_ROOT is required when an rsync:// source is present"
+        )
+    parsed = urlparse(arg)
+    if parsed.scheme != "rsync" or not parsed.hostname:
+        raise SystemExit(f"cir-rsync-wrapper: invalid rsync URI: {arg}")
+    path = parsed.path.lstrip("/")
+    local = Path(mirror_root).resolve() / parsed.hostname
+    if path:
+        local = local / path
+    local_str = str(local)
+    if local.exists() and local.is_dir() and not local_str.endswith("/"):
+        local_str += "/"
+    elif arg.endswith("/") and not local_str.endswith("/"):
+        local_str += "/"
+    return local_str
+
+
+def filter_args(args: list[str]) -> list[str]:
+    mirror_root = os.environ.get("CIR_MIRROR_ROOT")
+    rewritten_any = any(arg.startswith("rsync://") for arg in args)
+    out: list[str] = []
+    i = 0
+    while i < len(args):
+        arg = args[i]
+        if rewritten_any:
+            if arg == "--address":
+                i += 2
+                continue
+            if arg.startswith("--address="):
+                i += 1
+                continue
+            if arg == "--contimeout":
+                i += 2
+                continue
+            if arg.startswith("--contimeout="):
+                i += 1
+                continue
+        out.append(rewrite_arg(arg, mirror_root))
+        i += 1
+    return out
+
+
+def local_link_mode_enabled() -> bool:
+    value = os.environ.get("CIR_LOCAL_LINK_MODE", "")
+    return value.lower() in {"1", "true", "yes", "on"}
+
+
+def extract_source_and_dest(args: list[str]) -> tuple[str, str]:
+    expects_value = {
+        "--timeout",
+        "--min-size",
+        "--max-size",
+        "--include",
+        "--exclude",
+        "--compare-dest",
+    }
+    positionals: list[str] = []
+    i = 0
+    while i < len(args):
+        arg = args[i]
+        if arg in expects_value:
+            i += 2
+            continue
+        if any(arg.startswith(prefix + "=") for prefix in expects_value):
+            i += 1
+            continue
+        if arg.startswith("-"):
+            i += 1
+            continue
+        positionals.append(arg)
+        i += 1
+    if len(positionals) < 2:
+        raise SystemExit("cir-rsync-wrapper: expected source and destination arguments")
+    return positionals[-2], positionals[-1]
+
+
+def maybe_exec_local_link_sync(args: list[str], rewritten_any: bool) -> None:
+    if not rewritten_any or not local_link_mode_enabled():
+        return
+    source, dest = extract_source_and_dest(args)
+    if source.startswith("rsync://"):
+        raise SystemExit("cir-rsync-wrapper: expected rewritten local source for CIR_LOCAL_LINK_MODE")
+    helper = Path(__file__).with_name("cir-local-link-sync.py")
+    cmd = [sys.executable, str(helper)]
+    if "--delete" in args:
+        cmd.append("--delete")
+    cmd.extend([source, dest])
+    os.execv(sys.executable, cmd)
+
+
+def main() -> int:
+    args = sys.argv[1:]
+    rewritten_any = any(arg.startswith("rsync://") for arg in args)
+    rewritten = filter_args(args)
+    maybe_exec_local_link_sync(rewritten, rewritten_any)
+    os.execv(real_rsync_bin(), [real_rsync_bin(), *rewritten])
+    return 127
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/cir/fetch_cir_sequence_from_remote.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/fetch_cir_sequence_from_remote.sh \
+    --ssh-target <user@host> \
+    --remote-path <path> \
+    --local-path <path>
+EOF
+}
+
+SSH_TARGET=""
+REMOTE_PATH=""
+LOCAL_PATH=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --remote-path) REMOTE_PATH="$2"; shift 2 ;;
+    --local-path) LOCAL_PATH="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$SSH_TARGET" && -n "$REMOTE_PATH" && -n "$LOCAL_PATH" ]] || { usage >&2; exit 2; }
+
+mkdir -p "$(dirname "$LOCAL_PATH")"
+rsync -a "$SSH_TARGET:$REMOTE_PATH/" "$LOCAL_PATH/"
+echo "done: $LOCAL_PATH"

scripts/cir/json_to_vaps_csv.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+from pathlib import Path
+
+
+def normalize_asn(value: str | int) -> str:
+    text = str(value).strip().upper()
+    if text.startswith("AS"):
+        text = text[2:]
+    return f"AS{int(text)}"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input", required=True, type=Path)
+    parser.add_argument("--csv-out", required=True, type=Path)
+    args = parser.parse_args()
+
+    obj = json.loads(args.input.read_text(encoding="utf-8"))
+    rows: list[tuple[str, str, str]] = []
+    for aspa in obj.get("aspas", []):
+        providers = sorted(
+            {normalize_asn(item) for item in aspa.get("providers", [])},
+            key=lambda s: int(s[2:]),
+        )
+        rows.append(
+            (
+                normalize_asn(aspa["customer"]),
+                ";".join(providers),
+                str(aspa.get("ta", "")).strip().lower(),
+            )
+        )
+    rows.sort(key=lambda row: (int(row[0][2:]), row[1], row[2]))
+
+    args.csv_out.parent.mkdir(parents=True, exist_ok=True)
+    with args.csv_out.open("w", encoding="utf-8", newline="") as fh:
+        writer = csv.writer(fh)
+        writer.writerow(["Customer ASN", "Providers", "Trust Anchor"])
+        writer.writerows(rows)
+    print(args.csv_out)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/cir/run_cir_drop_sequence.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_drop_sequence.sh \
+    --sequence-root <path> \
+    [--drop-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SEQUENCE_ROOT=""
+DROP_BIN="${DROP_BIN:-$ROOT_DIR/target/release/cir_drop_report}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --sequence-root) SEQUENCE_ROOT="$2"; shift 2 ;;
+    --drop-bin) DROP_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$SEQUENCE_ROOT" ]] || { usage >&2; exit 2; }
+
+python3 - <<'PY' "$SEQUENCE_ROOT" "$DROP_BIN"
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+sequence_root = Path(sys.argv[1]).resolve()
+drop_bin = sys.argv[2]
+sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
+repo_bytes_db = sequence_root / sequence["repoBytesDbPath"]
+
+summaries = []
+for step in sequence["steps"]:
+    step_id = step["stepId"]
+    out_dir = sequence_root / "drop" / step_id
+    out_dir.mkdir(parents=True, exist_ok=True)
+    cmd = [
+        drop_bin,
+        "--cir",
+        str(sequence_root / step["cirPath"]),
+        "--ccr",
+        str(sequence_root / step["ccrPath"]),
+        "--report-json",
+        str(sequence_root / step["reportPath"]),
+        "--json-out",
+        str(out_dir / "drop.json"),
+        "--md-out",
+        str(out_dir / "drop.md"),
+    ]
+    cmd.extend(["--repo-bytes-db", str(repo_bytes_db)])
+    proc = subprocess.run(cmd, capture_output=True, text=True)
+    if proc.returncode != 0:
+        raise SystemExit(
+            f"drop report failed for {step_id}: stdout={proc.stdout} stderr={proc.stderr}"
+        )
+    result = json.loads((out_dir / "drop.json").read_text(encoding="utf-8"))
+    summaries.append(
+        {
+            "stepId": step_id,
+            "droppedVrpCount": result["summary"]["droppedVrpCount"],
+            "droppedObjectCount": result["summary"]["droppedObjectCount"],
+            "reportPath": str(out_dir / "drop.json"),
+        }
+    )
+
+summary = {"version": 1, "steps": summaries}
+(sequence_root / "drop-summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
+PY
+
+echo "done: $SEQUENCE_ROOT"

scripts/cir/run_cir_record_full_delta.sh

@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_record_full_delta.sh \
+    --out-dir <path> \
+    --tal-path <path> \
+    --ta-path <path> \
+    --cir-tal-uri <url> \
+    --payload-replay-archive <path> \
+    --payload-replay-locks <path> \
+    --payload-base-archive <path> \
+    --payload-base-locks <path> \
+    --payload-delta-archive <path> \
+    --payload-delta-locks <path> \
+    [--base-validation-time <rfc3339>] \
+    [--delta-validation-time <rfc3339>] \
+    [--max-depth <n>] \
+    [--max-instances <n>] \
+    [--rpki-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+OUT_DIR=""
+TAL_PATH=""
+TA_PATH=""
+CIR_TAL_URI=""
+PAYLOAD_REPLAY_ARCHIVE=""
+PAYLOAD_REPLAY_LOCKS=""
+PAYLOAD_BASE_ARCHIVE=""
+PAYLOAD_BASE_LOCKS=""
+PAYLOAD_DELTA_ARCHIVE=""
+PAYLOAD_DELTA_LOCKS=""
+BASE_VALIDATION_TIME=""
+DELTA_VALIDATION_TIME=""
+MAX_DEPTH=0
+MAX_INSTANCES=1
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --tal-path) TAL_PATH="$2"; shift 2 ;;
+    --ta-path) TA_PATH="$2"; shift 2 ;;
+    --cir-tal-uri) CIR_TAL_URI="$2"; shift 2 ;;
+    --payload-replay-archive) PAYLOAD_REPLAY_ARCHIVE="$2"; shift 2 ;;
+    --payload-replay-locks) PAYLOAD_REPLAY_LOCKS="$2"; shift 2 ;;
+    --payload-base-archive) PAYLOAD_BASE_ARCHIVE="$2"; shift 2 ;;
+    --payload-base-locks) PAYLOAD_BASE_LOCKS="$2"; shift 2 ;;
+    --payload-delta-archive) PAYLOAD_DELTA_ARCHIVE="$2"; shift 2 ;;
+    --payload-delta-locks) PAYLOAD_DELTA_LOCKS="$2"; shift 2 ;;
+    --base-validation-time) BASE_VALIDATION_TIME="$2"; shift 2 ;;
+    --delta-validation-time) DELTA_VALIDATION_TIME="$2"; shift 2 ;;
+    --max-depth) MAX_DEPTH="$2"; shift 2 ;;
+    --max-instances) MAX_INSTANCES="$2"; shift 2 ;;
+    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$OUT_DIR" && -n "$TAL_PATH" && -n "$TA_PATH" && -n "$CIR_TAL_URI" && -n "$PAYLOAD_REPLAY_ARCHIVE" && -n "$PAYLOAD_REPLAY_LOCKS" && -n "$PAYLOAD_BASE_ARCHIVE" && -n "$PAYLOAD_BASE_LOCKS" && -n "$PAYLOAD_DELTA_ARCHIVE" && -n "$PAYLOAD_DELTA_LOCKS" ]] || {
+  usage >&2
+  exit 2
+}
+
+if [[ ! -x "$RPKI_BIN" ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin rpki
+  )
+fi
+
+resolve_validation_time() {
+  local path="$1"
+  python3 - <<'PY' "$path"
+import json, sys
+print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['validationTime'])
+PY
+}
+
+if [[ -z "$BASE_VALIDATION_TIME" ]]; then
+  BASE_VALIDATION_TIME="$(resolve_validation_time "$PAYLOAD_REPLAY_LOCKS")"
+fi
+
+if [[ -z "$DELTA_VALIDATION_TIME" ]]; then
+  DELTA_VALIDATION_TIME="$(resolve_validation_time "$PAYLOAD_DELTA_LOCKS")"
+fi
+
+rm -rf "$OUT_DIR"
+mkdir -p "$OUT_DIR/full" "$OUT_DIR/delta-001"
+REPO_BYTES_DB="$OUT_DIR/repo-bytes.db"
+
+FULL_DB="$OUT_DIR/full/db"
+DELTA_DB="$OUT_DIR/delta-001/db"
+
+"$RPKI_BIN" \
+  --db "$FULL_DB" \
+  --tal-path "$TAL_PATH" \
+  --ta-path "$TA_PATH" \
+  --payload-replay-archive "$PAYLOAD_REPLAY_ARCHIVE" \
+  --payload-replay-locks "$PAYLOAD_REPLAY_LOCKS" \
+  --validation-time "$BASE_VALIDATION_TIME" \
+  --max-depth "$MAX_DEPTH" \
+  --max-instances "$MAX_INSTANCES" \
+  --ccr-out "$OUT_DIR/full/result.ccr" \
+  --report-json "$OUT_DIR/full/report.json" \
+  --cir-enable \
+  --cir-out "$OUT_DIR/full/input.cir" \
+  --repo-bytes-db "$REPO_BYTES_DB" \
+  --cir-tal-uri "$CIR_TAL_URI" \
+  >"$OUT_DIR/full/run.stdout.log" 2>"$OUT_DIR/full/run.stderr.log"
+
+"$RPKI_BIN" \
+  --db "$DELTA_DB" \
+  --tal-path "$TAL_PATH" \
+  --ta-path "$TA_PATH" \
+  --payload-base-archive "$PAYLOAD_BASE_ARCHIVE" \
+  --payload-base-locks "$PAYLOAD_BASE_LOCKS" \
+  --payload-delta-archive "$PAYLOAD_DELTA_ARCHIVE" \
+  --payload-delta-locks "$PAYLOAD_DELTA_LOCKS" \
+  --payload-base-validation-time "$BASE_VALIDATION_TIME" \
+  --validation-time "$DELTA_VALIDATION_TIME" \
+  --max-depth "$MAX_DEPTH" \
+  --max-instances "$MAX_INSTANCES" \
+  --ccr-out "$OUT_DIR/delta-001/result.ccr" \
+  --report-json "$OUT_DIR/delta-001/report.json" \
+  --cir-enable \
+  --cir-out "$OUT_DIR/delta-001/input.cir" \
+  --repo-bytes-db "$REPO_BYTES_DB" \
+  --cir-tal-uri "$CIR_TAL_URI" \
+  >"$OUT_DIR/delta-001/run.stdout.log" 2>"$OUT_DIR/delta-001/run.stderr.log"
+
+python3 - <<'PY' "$OUT_DIR" "$BASE_VALIDATION_TIME" "$DELTA_VALIDATION_TIME"
+import json
+import os
+import sys
+from pathlib import Path
+
+out = Path(sys.argv[1])
+base_validation_time = sys.argv[2]
+delta_validation_time = sys.argv[3]
+summary = {
+    "version": 1,
+    "kind": "cir_pair",
+    "baseValidationTime": base_validation_time,
+    "deltaValidationTime": delta_validation_time,
+    "repoBytesDbPath": "repo-bytes.db",
+    "steps": [
+        {
+            "kind": "full",
+            "cirPath": "full/input.cir",
+            "ccrPath": "full/result.ccr",
+            "reportPath": "full/report.json",
+        },
+        {
+            "kind": "delta",
+            "cirPath": "delta-001/input.cir",
+            "ccrPath": "delta-001/result.ccr",
+            "reportPath": "delta-001/report.json",
+            "previous": "full",
+        },
+    ],
+    "repoBytesDbExists": (out / "repo-bytes.db").exists(),
+}
+(out / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
+PY
+
+echo "done: $OUT_DIR"

scripts/cir/run_cir_record_sequence_multi_rir_offline.sh

@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_record_sequence_multi_rir_offline.sh \
+    [--bundle-root <path>] \
+    [--rir <afrinic,apnic,arin,lacnic,ripe>] \
+    [--delta-count <n>] \
+    [--full-repo] \
+    [--out-root <path>] \
+    [--rpki-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+CASE_INFO="$ROOT_DIR/scripts/payload_replay/multi_rir_case_info.py"
+SINGLE_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_record_sequence_offline.sh"
+
+BUNDLE_ROOT="/home/yuyr/dev/rust_playground/routinator/bench/multi_rir_demo/runs/20260316-112341-multi-final3"
+RIRS="afrinic,apnic,arin,lacnic,ripe"
+DELTA_COUNT=2
+FULL_REPO=0
+OUT_ROOT="$ROOT_DIR/target/replay/cir_sequence_multi_rir_offline_$(date -u +%Y%m%dT%H%M%SZ)"
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --bundle-root) BUNDLE_ROOT="$2"; shift 2 ;;
+    --rir) RIRS="$2"; shift 2 ;;
+    --delta-count) DELTA_COUNT="$2"; shift 2 ;;
+    --full-repo) FULL_REPO=1; shift 1 ;;
+    --out-root) OUT_ROOT="$2"; shift 2 ;;
+    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+mkdir -p "$OUT_ROOT"
+SUMMARY_JSON="$OUT_ROOT/summary.json"
+SUMMARY_MD="$OUT_ROOT/summary.md"
+
+IFS=',' read -r -a RIR_ITEMS <<< "$RIRS"
+
+for rir in "${RIR_ITEMS[@]}"; do
+  CASE_JSON="$(python3 "$CASE_INFO" --bundle-root "$BUNDLE_ROOT" --repo-root "$ROOT_DIR" --rir "$rir")"
+  TAL_PATH="$(python3 - <<'PY' "$CASE_JSON"
+import json,sys
+print(json.loads(sys.argv[1])['tal_path'])
+PY
+)"
+  TA_PATH="$(python3 - <<'PY' "$CASE_JSON"
+import json,sys
+print(json.loads(sys.argv[1])['ta_path'])
+PY
+)"
+  BASE_ARCHIVE="$(python3 - <<'PY' "$CASE_JSON"
+import json,sys
+print(json.loads(sys.argv[1])['base_archive'])
+PY
+)"
+  BASE_LOCKS="$(python3 - <<'PY' "$CASE_JSON"
+import json,sys
+print(json.loads(sys.argv[1])['base_locks'])
+PY
+)"
+  DELTA_ARCHIVE="$(python3 - <<'PY' "$CASE_JSON"
+import json,sys
+print(json.loads(sys.argv[1])['delta_archive'])
+PY
+)"
+  DELTA_LOCKS="$(python3 - <<'PY' "$CASE_JSON"
+import json,sys
+print(json.loads(sys.argv[1])['delta_locks'])
+PY
+)"
+  OUT_DIR="$OUT_ROOT/$rir"
+  args=(
+    "$SINGLE_SCRIPT"
+    --out-dir "$OUT_DIR" \
+    --tal-path "$TAL_PATH" \
+    --ta-path "$TA_PATH" \
+    --cir-tal-uri "https://example.test/$rir.tal" \
+    --payload-replay-archive "$BASE_ARCHIVE" \
+    --payload-replay-locks "$BASE_LOCKS" \
+    --payload-base-archive "$BASE_ARCHIVE" \
+    --payload-base-locks "$BASE_LOCKS" \
+    --payload-delta-archive "$DELTA_ARCHIVE" \
+    --payload-delta-locks "$DELTA_LOCKS" \
+    --delta-count "$DELTA_COUNT" \
+    --rpki-bin "$RPKI_BIN"
+  )
+  if [[ "$FULL_REPO" -ne 1 ]]; then
+    args+=(--max-depth 0 --max-instances 1)
+  else
+    args+=(--full-repo)
+  fi
+  "${args[@]}"
+done
+
+python3 - <<'PY' "$OUT_ROOT" "$RIRS" "$SUMMARY_JSON" "$SUMMARY_MD"
+import json, sys
+from pathlib import Path
+out_root = Path(sys.argv[1])
+rirs = [item for item in sys.argv[2].split(',') if item]
+summary_json = Path(sys.argv[3])
+summary_md = Path(sys.argv[4])
+items = []
+for rir in rirs:
+    root = out_root / rir
+    seq = json.loads((root / "sequence.json").read_text(encoding="utf-8"))
+    summ = json.loads((root / "summary.json").read_text(encoding="utf-8"))
+    items.append({
+        "rir": rir,
+        "root": str(root),
+        "stepCount": len(seq["steps"]),
+        "repoBytesDbExists": summ.get("repoBytesDbExists", False),
+    })
+summary = {"version": 1, "rirs": items}
+summary_json.write_text(json.dumps(summary, indent=2), encoding="utf-8")
+lines = ["# Multi-RIR Offline CIR Sequence Summary", ""]
+for item in items:
+    lines.append(f"- `{item['rir']}`: `stepCount={item['stepCount']}` `repoBytesDbExists={item['repoBytesDbExists']}` `root={item['root']}`")
+summary_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
+PY
+
+echo "done: $OUT_ROOT"

scripts/cir/run_cir_record_sequence_offline.sh

@@ -0,0 +1,208 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_record_sequence_offline.sh \
+    --out-dir <path> \
+    --tal-path <path> \
+    --ta-path <path> \
+    --cir-tal-uri <url> \
+    --payload-replay-archive <path> \
+    --payload-replay-locks <path> \
+    --payload-base-archive <path> \
+    --payload-base-locks <path> \
+    --payload-delta-archive <path> \
+    --payload-delta-locks <path> \
+    [--delta-count <n>] \
+    [--base-validation-time <rfc3339>] \
+    [--delta-validation-time <rfc3339>] \
+    [--full-repo] \
+    [--max-depth <n>] \
+    [--max-instances <n>] \
+    [--rpki-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+OUT_DIR=""
+TAL_PATH=""
+TA_PATH=""
+CIR_TAL_URI=""
+PAYLOAD_REPLAY_ARCHIVE=""
+PAYLOAD_REPLAY_LOCKS=""
+PAYLOAD_BASE_ARCHIVE=""
+PAYLOAD_BASE_LOCKS=""
+PAYLOAD_DELTA_ARCHIVE=""
+PAYLOAD_DELTA_LOCKS=""
+BASE_VALIDATION_TIME=""
+DELTA_VALIDATION_TIME=""
+DELTA_COUNT=2
+FULL_REPO=0
+MAX_DEPTH=0
+MAX_INSTANCES=1
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --tal-path) TAL_PATH="$2"; shift 2 ;;
+    --ta-path) TA_PATH="$2"; shift 2 ;;
+    --cir-tal-uri) CIR_TAL_URI="$2"; shift 2 ;;
+    --payload-replay-archive) PAYLOAD_REPLAY_ARCHIVE="$2"; shift 2 ;;
+    --payload-replay-locks) PAYLOAD_REPLAY_LOCKS="$2"; shift 2 ;;
+    --payload-base-archive) PAYLOAD_BASE_ARCHIVE="$2"; shift 2 ;;
+    --payload-base-locks) PAYLOAD_BASE_LOCKS="$2"; shift 2 ;;
+    --payload-delta-archive) PAYLOAD_DELTA_ARCHIVE="$2"; shift 2 ;;
+    --payload-delta-locks) PAYLOAD_DELTA_LOCKS="$2"; shift 2 ;;
+    --base-validation-time) BASE_VALIDATION_TIME="$2"; shift 2 ;;
+    --delta-validation-time) DELTA_VALIDATION_TIME="$2"; shift 2 ;;
+    --delta-count) DELTA_COUNT="$2"; shift 2 ;;
+    --full-repo) FULL_REPO=1; shift 1 ;;
+    --max-depth) MAX_DEPTH="$2"; shift 2 ;;
+    --max-instances) MAX_INSTANCES="$2"; shift 2 ;;
+    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$OUT_DIR" && -n "$TAL_PATH" && -n "$TA_PATH" && -n "$CIR_TAL_URI" && -n "$PAYLOAD_REPLAY_ARCHIVE" && -n "$PAYLOAD_REPLAY_LOCKS" && -n "$PAYLOAD_BASE_ARCHIVE" && -n "$PAYLOAD_BASE_LOCKS" && -n "$PAYLOAD_DELTA_ARCHIVE" && -n "$PAYLOAD_DELTA_LOCKS" ]] || {
+  usage >&2
+  exit 2
+}
+
+if [[ ! -x "$RPKI_BIN" ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin rpki
+  )
+fi
+
+resolve_validation_time() {
+  local path="$1"
+  python3 - <<'PY' "$path"
+import json, sys
+print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['validationTime'])
+PY
+}
+
+if [[ -z "$BASE_VALIDATION_TIME" ]]; then
+  BASE_VALIDATION_TIME="$(resolve_validation_time "$PAYLOAD_REPLAY_LOCKS")"
+fi
+if [[ -z "$DELTA_VALIDATION_TIME" ]]; then
+  DELTA_VALIDATION_TIME="$(resolve_validation_time "$PAYLOAD_DELTA_LOCKS")"
+fi
+
+rm -rf "$OUT_DIR"
+mkdir -p "$OUT_DIR/full"
+REPO_BYTES_DB="$OUT_DIR/repo-bytes.db"
+
+run_step() {
+  local kind="$1"
+  local step_dir="$2"
+  local db_dir="$3"
+  shift 3
+  mkdir -p "$step_dir"
+  local -a cmd=(
+    "$RPKI_BIN"
+    --db "$db_dir" \
+    --tal-path "$TAL_PATH" \
+    --ta-path "$TA_PATH" \
+    --ccr-out "$step_dir/result.ccr" \
+    --report-json "$step_dir/report.json" \
+    --cir-enable \
+    --cir-out "$step_dir/input.cir" \
+    --repo-bytes-db "$REPO_BYTES_DB" \
+    --cir-tal-uri "$CIR_TAL_URI"
+  )
+  if [[ "$FULL_REPO" -ne 1 ]]; then
+    cmd+=(--max-depth "$MAX_DEPTH" --max-instances "$MAX_INSTANCES")
+  fi
+  cmd+=("$@")
+  "${cmd[@]}" >"$step_dir/run.stdout.log" 2>"$step_dir/run.stderr.log"
+}
+
+run_step \
+  full \
+  "$OUT_DIR/full" \
+  "$OUT_DIR/full/db" \
+  --payload-replay-archive "$PAYLOAD_REPLAY_ARCHIVE" \
+  --payload-replay-locks "$PAYLOAD_REPLAY_LOCKS" \
+  --validation-time "$BASE_VALIDATION_TIME"
+
+for idx in $(seq 1 "$DELTA_COUNT"); do
+  step_id="$(printf 'delta-%03d' "$idx")"
+  run_step \
+    delta \
+    "$OUT_DIR/$step_id" \
+    "$OUT_DIR/$step_id/db" \
+    --payload-base-archive "$PAYLOAD_BASE_ARCHIVE" \
+    --payload-base-locks "$PAYLOAD_BASE_LOCKS" \
+    --payload-delta-archive "$PAYLOAD_DELTA_ARCHIVE" \
+    --payload-delta-locks "$PAYLOAD_DELTA_LOCKS" \
+    --payload-base-validation-time "$BASE_VALIDATION_TIME" \
+    --validation-time "$DELTA_VALIDATION_TIME"
+done
+
+python3 - <<'PY' "$OUT_DIR" "$BASE_VALIDATION_TIME" "$DELTA_VALIDATION_TIME" "$DELTA_COUNT"
+import json
+import sys
+from pathlib import Path
+
+out = Path(sys.argv[1])
+base_validation_time = sys.argv[2]
+delta_validation_time = sys.argv[3]
+delta_count = int(sys.argv[4])
+
+steps = [
+    {
+        "stepId": "full",
+        "kind": "full",
+        "validationTime": base_validation_time,
+        "cirPath": "full/input.cir",
+        "ccrPath": "full/result.ccr",
+        "reportPath": "full/report.json",
+        "previousStepId": None,
+    }
+]
+previous = "full"
+for idx in range(1, delta_count + 1):
+    step_id = f"delta-{idx:03d}"
+    steps.append(
+        {
+            "stepId": step_id,
+            "kind": "delta",
+            "validationTime": delta_validation_time,
+            "cirPath": f"{step_id}/input.cir",
+            "ccrPath": f"{step_id}/result.ccr",
+            "reportPath": f"{step_id}/report.json",
+            "previousStepId": previous,
+        }
+    )
+    previous = step_id
+
+summary = {
+    "version": 1,
+    "kind": "cir_sequence_offline",
+    "repoBytesDbPath": "repo-bytes.db",
+    "steps": steps,
+}
+(out / "sequence.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
+(out / "summary.json").write_text(
+    json.dumps(
+        {
+            "version": 1,
+            "stepCount": len(steps),
+            "repoBytesDbPath": "repo-bytes.db",
+            "repoBytesDbExists": (out / "repo-bytes.db").exists(),
+        },
+        indent=2,
+    ),
+    encoding="utf-8",
+)
+PY
+
+echo "done: $OUT_DIR"

scripts/cir/run_cir_record_sequence_remote.sh

@@ -0,0 +1,246 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_record_sequence_remote.sh \
+    --rir <name> \
+    --remote-root <path> \
+    [--ssh-target <user@host>] \
+    [--out-subdir <path>] \
+    [--delta-count <n>] \
+    [--sleep-secs <n>] \
+    [--full-repo] \
+    [--max-depth <n>] \
+    [--max-instances <n>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
+RIR=""
+REMOTE_ROOT=""
+OUT_SUBDIR=""
+DELTA_COUNT=2
+SLEEP_SECS=30
+FULL_REPO=0
+MAX_DEPTH=0
+MAX_INSTANCES=1
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rir) RIR="$2"; shift 2 ;;
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --out-subdir) OUT_SUBDIR="$2"; shift 2 ;;
+    --delta-count) DELTA_COUNT="$2"; shift 2 ;;
+    --sleep-secs) SLEEP_SECS="$2"; shift 2 ;;
+    --full-repo) FULL_REPO=1; shift 1 ;;
+    --max-depth) MAX_DEPTH="$2"; shift 2 ;;
+    --max-instances) MAX_INSTANCES="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$RIR" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
+
+case "$RIR" in
+  afrinic) TAL_REL="tests/fixtures/tal/afrinic.tal"; TA_REL="tests/fixtures/ta/afrinic-ta.cer" ;;
+  apnic) TAL_REL="tests/fixtures/tal/apnic-rfc7730-https.tal"; TA_REL="tests/fixtures/ta/apnic-ta.cer" ;;
+  arin) TAL_REL="tests/fixtures/tal/arin.tal"; TA_REL="tests/fixtures/ta/arin-ta.cer" ;;
+  lacnic) TAL_REL="tests/fixtures/tal/lacnic.tal"; TA_REL="tests/fixtures/ta/lacnic-ta.cer" ;;
+  ripe) TAL_REL="tests/fixtures/tal/ripe-ncc.tal"; TA_REL="tests/fixtures/ta/ripe-ncc-ta.cer" ;;
+  *) echo "unsupported rir: $RIR" >&2; exit 2 ;;
+esac
+
+rsync -a --delete \
+  --exclude target \
+  --exclude .git \
+  "$ROOT_DIR/" "$SSH_TARGET:$REMOTE_ROOT/"
+ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/target/release'"
+rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_ROOT/target/release/"
+
+ssh "$SSH_TARGET" \
+  RIR="$RIR" \
+  REMOTE_ROOT="$REMOTE_ROOT" \
+  OUT_SUBDIR="$OUT_SUBDIR" \
+  DELTA_COUNT="$DELTA_COUNT" \
+  SLEEP_SECS="$SLEEP_SECS" \
+  FULL_REPO="$FULL_REPO" \
+  MAX_DEPTH="$MAX_DEPTH" \
+  MAX_INSTANCES="$MAX_INSTANCES" \
+  TAL_REL="$TAL_REL" \
+  TA_REL="$TA_REL" \
+  'bash -s' <<'EOS'
+set -euo pipefail
+
+cd "$REMOTE_ROOT"
+
+if [[ -n "${OUT_SUBDIR}" ]]; then
+  OUT="${OUT_SUBDIR}"
+else
+  OUT="target/replay/cir_sequence_remote_${RIR}_$(date -u +%Y%m%dT%H%M%SZ)"
+fi
+
+mkdir -p "$OUT"
+DB="$OUT/work-db"
+RAW_STORE_DB="$OUT/raw-store.db"
+REPO_BYTES_DB="$OUT/repo-bytes.db"
+ROWS="$OUT/.sequence_rows.tsv"
+: > "$ROWS"
+
+write_step_timing() {
+  local path="$1"
+  local start_ms="$2"
+  local end_ms="$3"
+  local started_at="$4"
+  local finished_at="$5"
+  python3 - <<'PY' "$path" "$start_ms" "$end_ms" "$started_at" "$finished_at"
+import json, sys
+path, start_ms, end_ms, started_at, finished_at = sys.argv[1:]
+start_ms = int(start_ms)
+end_ms = int(end_ms)
+with open(path, "w", encoding="utf-8") as fh:
+    json.dump(
+        {
+            "durationMs": end_ms - start_ms,
+            "startedAt": started_at,
+            "finishedAt": finished_at,
+        },
+        fh,
+        indent=2,
+    )
+PY
+}
+
+run_step() {
+  local step_id="$1"
+  local kind="$2"
+  local previous_step_id="$3"
+  shift 3
+
+  local started_at_iso started_at_ms finished_at_iso finished_at_ms prefix
+  started_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+  started_at_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  prefix="${started_at_iso}-test"
+
+  local cir_out="$OUT/${prefix}.cir"
+  local ccr_out="$OUT/${prefix}.ccr"
+  local report_out="$OUT/${prefix}.report.json"
+  local timing_out="$OUT/${prefix}.timing.json"
+  local stdout_out="$OUT/${prefix}.stdout.log"
+  local stderr_out="$OUT/${prefix}.stderr.log"
+
+  local -a cmd=(
+    target/release/rpki
+    --db "$DB"
+    --raw-store-db "$RAW_STORE_DB"
+    --repo-bytes-db "$REPO_BYTES_DB"
+    --tal-path "$TAL_REL"
+    --ta-path "$TA_REL"
+    --ccr-out "$ccr_out"
+    --report-json "$report_out"
+    --cir-enable
+    --cir-out "$cir_out"
+    --cir-tal-uri "https://example.test/${RIR}.tal"
+  )
+  if [[ "$FULL_REPO" -ne 1 ]]; then
+    cmd+=(--max-depth "$MAX_DEPTH" --max-instances "$MAX_INSTANCES")
+  fi
+  cmd+=("$@")
+  env RPKI_PROGRESS_LOG=1 "${cmd[@]}" >"$stdout_out" 2>"$stderr_out"
+
+  finished_at_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  finished_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+  write_step_timing "$timing_out" "$started_at_ms" "$finished_at_ms" "$started_at_iso" "$finished_at_iso"
+
+  local validation_time
+  validation_time="$(python3 - <<'PY' "$report_out"
+import json, sys
+print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['meta']['validation_time_rfc3339_utc'])
+PY
+)"
+
+  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
+    "$step_id" \
+    "$kind" \
+    "$validation_time" \
+    "$(basename "$cir_out")" \
+    "$(basename "$ccr_out")" \
+    "$(basename "$report_out")" \
+    "$(basename "$timing_out")" \
+    "$(basename "$stdout_out")" \
+    "$(basename "$stderr_out")" >> "$ROWS"
+}
+
+run_step "full" "full" "" 
+
+prev="full"
+for idx in $(seq 1 "$DELTA_COUNT"); do
+  sleep "$SLEEP_SECS"
+  step="$(printf 'delta-%03d' "$idx")"
+  run_step "$step" "delta" "$prev"
+  prev="$step"
+done
+
+python3 - <<'PY' "$OUT" "$ROWS" "$RIR"
+import json, sys
+from pathlib import Path
+
+out = Path(sys.argv[1])
+rows = Path(sys.argv[2]).read_text(encoding='utf-8').splitlines()
+rir = sys.argv[3]
+steps = []
+for idx, row in enumerate(rows):
+    step_id, kind, validation_time, cir_name, ccr_name, report_name, timing_name, stdout_name, stderr_name = row.split('\t')
+    steps.append({
+        "stepId": step_id,
+        "kind": kind,
+        "validationTime": validation_time,
+        "cirPath": cir_name,
+        "ccrPath": ccr_name,
+        "reportPath": report_name,
+        "timingPath": timing_name,
+        "stdoutLogPath": stdout_name,
+        "stderrLogPath": stderr_name,
+        "artifactPrefix": cir_name[:-4],  # strip .cir
+        "previousStepId": None if idx == 0 else steps[idx - 1]["stepId"],
+    })
+
+(out / "sequence.json").write_text(
+    json.dumps({"version": 1, "repoBytesDbPath": "repo-bytes.db", "steps": steps}, indent=2),
+    encoding="utf-8",
+)
+
+summary = {
+    "version": 1,
+    "rir": rir,
+    "stepCount": len(steps),
+    "steps": [],
+}
+for step in steps:
+    timing = json.loads((out / step["timingPath"]).read_text(encoding="utf-8"))
+    summary["steps"].append({
+        "stepId": step["stepId"],
+        "kind": step["kind"],
+        "validationTime": step["validationTime"],
+        "artifactPrefix": step["artifactPrefix"],
+        **timing,
+    })
+
+(out / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
+PY
+
+rm -f "$ROWS"
+echo "$OUT"
+EOS

scripts/cir/run_cir_record_sequence_remote_multi_rir.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_record_sequence_remote_multi_rir.sh \
+    --remote-root <path> \
+    [--rir <afrinic,apnic,arin,lacnic,ripe>] \
+    [--ssh-target <user@host>] \
+    [--out-subdir-root <path>] \
+    [--delta-count <n>] \
+    [--sleep-secs <n>] \
+    [--full-repo] \
+    [--max-depth <n>] \
+    [--max-instances <n>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
+REMOTE_ROOT=""
+RIRS="afrinic,apnic,arin,lacnic,ripe"
+OUT_SUBDIR_ROOT=""
+DELTA_COUNT=2
+SLEEP_SECS=30
+FULL_REPO=0
+MAX_DEPTH=0
+MAX_INSTANCES=1
+SINGLE="$ROOT_DIR/scripts/cir/run_cir_record_sequence_remote.sh"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --rir) RIRS="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --out-subdir-root) OUT_SUBDIR_ROOT="$2"; shift 2 ;;
+    --delta-count) DELTA_COUNT="$2"; shift 2 ;;
+    --sleep-secs) SLEEP_SECS="$2"; shift 2 ;;
+    --full-repo) FULL_REPO=1; shift 1 ;;
+    --max-depth) MAX_DEPTH="$2"; shift 2 ;;
+    --max-instances) MAX_INSTANCES="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
+if [[ -z "$OUT_SUBDIR_ROOT" ]]; then
+  OUT_SUBDIR_ROOT="target/replay/cir_sequence_remote_multi_rir_$(date -u +%Y%m%dT%H%M%SZ)"
+fi
+
+IFS=',' read -r -a ITEMS <<< "$RIRS"
+for rir in "${ITEMS[@]}"; do
+  args=(
+    "$SINGLE"
+    --rir "$rir" \
+    --remote-root "$REMOTE_ROOT" \
+    --ssh-target "$SSH_TARGET" \
+    --out-subdir "$OUT_SUBDIR_ROOT/$rir" \
+    --delta-count "$DELTA_COUNT" \
+    --sleep-secs "$SLEEP_SECS" \
+  )
+  if [[ "$FULL_REPO" -eq 1 ]]; then
+    args+=(--full-repo)
+  else
+    args+=(--max-depth "$MAX_DEPTH" --max-instances "$MAX_INSTANCES")
+  fi
+  "${args[@]}"
+done
+
+echo "$OUT_SUBDIR_ROOT"

scripts/cir/run_cir_record_sequence_ta_only_multi_rir.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_record_sequence_ta_only_multi_rir.sh \
+    [--rir <afrinic,apnic,arin,lacnic,ripe>] \
+    [--delta-count <n>] \
+    [--out-root <path>] \
+    [--rpki-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+HELPER_BIN="${HELPER_BIN:-$ROOT_DIR/target/release/cir_ta_only_fixture}"
+MATERIALIZE_BIN="${MATERIALIZE_BIN:-$ROOT_DIR/target/release/cir_materialize}"
+EXTRACT_BIN="${EXTRACT_BIN:-$ROOT_DIR/target/release/cir_extract_inputs}"
+WRAPPER="$ROOT_DIR/scripts/cir/cir-rsync-wrapper"
+RIRS="afrinic,apnic,arin,lacnic,ripe"
+DELTA_COUNT=2
+OUT_ROOT="$ROOT_DIR/target/replay/cir_sequence_multi_rir_ta_only_$(date -u +%Y%m%dT%H%M%SZ)"
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rir) RIRS="$2"; shift 2 ;;
+    --delta-count) DELTA_COUNT="$2"; shift 2 ;;
+    --out-root) OUT_ROOT="$2"; shift 2 ;;
+    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+if [[ ! -x "$HELPER_BIN" ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin cir_ta_only_fixture --bin rpki --bin cir_materialize --bin cir_extract_inputs
+  )
+fi
+
+case_paths() {
+  case "$1" in
+    afrinic) echo "tests/fixtures/tal/afrinic.tal tests/fixtures/ta/afrinic-ta.cer" ;;
+    apnic) echo "tests/fixtures/tal/apnic-rfc7730-https.tal tests/fixtures/ta/apnic-ta.cer" ;;
+    arin) echo "tests/fixtures/tal/arin.tal tests/fixtures/ta/arin-ta.cer" ;;
+    lacnic) echo "tests/fixtures/tal/lacnic.tal tests/fixtures/ta/lacnic-ta.cer" ;;
+    ripe) echo "tests/fixtures/tal/ripe-ncc.tal tests/fixtures/ta/ripe-ncc-ta.cer" ;;
+    *) return 1 ;;
+  esac
+}
+
+mkdir -p "$OUT_ROOT"
+IFS=',' read -r -a ITEMS <<< "$RIRS"
+for rir in "${ITEMS[@]}"; do
+  read -r tal_rel ta_rel < <(case_paths "$rir")
+  rir_root="$OUT_ROOT/$rir"
+  mkdir -p "$rir_root/full"
+  repo_bytes_db="$rir_root/repo-bytes.db"
+  "$HELPER_BIN" \
+    --tal-path "$ROOT_DIR/$tal_rel" \
+    --ta-path "$ROOT_DIR/$ta_rel" \
+    --tal-uri "https://example.test/$rir.tal" \
+    --validation-time "2026-04-09T00:00:00Z" \
+    --cir-out "$rir_root/full/input.cir" \
+    --repo-bytes-db "$repo_bytes_db"
+  "$EXTRACT_BIN" --cir "$rir_root/full/input.cir" --tals-dir "$rir_root/.tmp/tals" --meta-json "$rir_root/.tmp/meta.json"
+  "$MATERIALIZE_BIN" --cir "$rir_root/full/input.cir" --repo-bytes-db "$repo_bytes_db" --mirror-root "$rir_root/.tmp/mirror"
+  FIRST_TAL="$(python3 - <<'PY' "$rir_root/.tmp/meta.json"
+import json,sys
+print(json.load(open(sys.argv[1]))["talFiles"][0]["path"])
+PY
+)"
+  export CIR_MIRROR_ROOT="$rir_root/.tmp/mirror"
+  export REAL_RSYNC_BIN=/usr/bin/rsync
+  export CIR_LOCAL_LINK_MODE=1
+  "$RPKI_BIN" \
+    --db "$rir_root/full/db" \
+    --tal-path "$FIRST_TAL" \
+    --disable-rrdp \
+    --rsync-command "$WRAPPER" \
+    --validation-time "2026-04-09T00:00:00Z" \
+    --ccr-out "$rir_root/full/result.ccr" \
+    --report-json "$rir_root/full/report.json" >/dev/null 2>&1
+  for idx in $(seq 1 "$DELTA_COUNT"); do
+    step="$(printf 'delta-%03d' "$idx")"
+    mkdir -p "$rir_root/$step"
+    cp "$rir_root/full/input.cir" "$rir_root/$step/input.cir"
+    cp "$rir_root/full/result.ccr" "$rir_root/$step/result.ccr"
+    cp "$rir_root/full/report.json" "$rir_root/$step/report.json"
+  done
+  python3 - <<'PY' "$rir_root" "$DELTA_COUNT"
+import json, sys
+from pathlib import Path
+root = Path(sys.argv[1]); delta_count = int(sys.argv[2])
+steps = [{"stepId":"full","kind":"full","validationTime":"2026-04-09T00:00:00Z","cirPath":"full/input.cir","ccrPath":"full/result.ccr","reportPath":"full/report.json","previousStepId":None}]
+prev = "full"
+for i in range(1, delta_count + 1):
+    step = f"delta-{i:03d}"
+    steps.append({"stepId":step,"kind":"delta","validationTime":"2026-04-09T00:00:00Z","cirPath":f"{step}/input.cir","ccrPath":f"{step}/result.ccr","reportPath":f"{step}/report.json","previousStepId":prev})
+    prev = step
+(root/"sequence.json").write_text(json.dumps({"version":1,"repoBytesDbPath":"repo-bytes.db","steps":steps}, indent=2), encoding="utf-8")
+(root/"summary.json").write_text(json.dumps({"version":1,"stepCount":len(steps)}, indent=2), encoding="utf-8")
+PY
+done
+
+python3 - <<'PY' "$OUT_ROOT" "$RIRS"
+import json, sys
+from pathlib import Path
+root = Path(sys.argv[1]); rirs = [x for x in sys.argv[2].split(',') if x]
+items=[]
+for rir in rirs:
+    seq=json.loads((root/rir/'sequence.json').read_text())
+    items.append({"rir":rir,"stepCount":len(seq['steps'])})
+(root/'summary.json').write_text(json.dumps({"version":1,"rirs":items}, indent=2), encoding='utf-8')
+PY
+
+echo "done: $OUT_ROOT"

scripts/cir/run_cir_record_sequence_ta_only_remote_multi_rir.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_record_sequence_ta_only_remote_multi_rir.sh \
+    --remote-root <path> \
+    [--ssh-target <user@host>] \
+    [--rir <afrinic,apnic,arin,lacnic,ripe>] \
+    [--delta-count <n>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
+REMOTE_ROOT=""
+RIRS="afrinic,apnic,arin,lacnic,ripe"
+DELTA_COUNT=2
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --rir) RIRS="$2"; shift 2 ;;
+    --delta-count) DELTA_COUNT="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
+
+rsync -a --delete \
+  --exclude target \
+  --exclude .git \
+  "$ROOT_DIR/" "$SSH_TARGET:$REMOTE_ROOT/"
+ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/target/release'"
+for bin in rpki cir_ta_only_fixture cir_materialize cir_extract_inputs; do
+  rsync -a "$ROOT_DIR/target/release/$bin" "$SSH_TARGET:$REMOTE_ROOT/target/release/"
+done
+
+ssh "$SSH_TARGET" "bash -lc '
+set -euo pipefail
+cd $REMOTE_ROOT
+OUT=target/replay/cir_sequence_remote_ta_only_\$(date -u +%Y%m%dT%H%M%SZ)
+./scripts/cir/run_cir_record_sequence_ta_only_multi_rir.sh --rir $RIRS --delta-count $DELTA_COUNT --out-root \"\$OUT\"
+echo \"\$OUT\"
+'"

scripts/cir/run_cir_replay_matrix.sh

@@ -0,0 +1,293 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_replay_matrix.sh \
+    --cir <path> \
+    --repo-bytes-db <path> \
+    --out-dir <path> \
+    --reference-ccr <path> \
+    --rpki-client-build-dir <path> \
+    [--keep-db] \
+    [--rpki-bin <path>] \
+    [--routinator-root <path>] \
+    [--routinator-bin <path>] \
+    [--real-rsync-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+CIR=""
+REPO_BYTES_DB=""
+OUT_DIR=""
+REFERENCE_CCR=""
+RPKI_CLIENT_BUILD_DIR=""
+KEEP_DB=0
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+ROUTINATOR_ROOT="${ROUTINATOR_ROOT:-/home/yuyr/dev/rust_playground/routinator}"
+ROUTINATOR_BIN="${ROUTINATOR_BIN:-$ROUTINATOR_ROOT/target/debug/routinator}"
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+OURS_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_ours.sh"
+ROUTINATOR_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_routinator.sh"
+RPKI_CLIENT_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_rpki_client.sh"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --cir) CIR="$2"; shift 2 ;;
+    --repo-bytes-db) REPO_BYTES_DB="$2"; shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
+    --rpki-client-build-dir) RPKI_CLIENT_BUILD_DIR="$2"; shift 2 ;;
+    --keep-db) KEEP_DB=1; shift ;;
+    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
+    --routinator-root) ROUTINATOR_ROOT="$2"; shift 2 ;;
+    --routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$CIR" && -n "$REPO_BYTES_DB" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" && -n "$RPKI_CLIENT_BUILD_DIR" ]] || {
+  usage >&2
+  exit 2
+}
+
+mkdir -p "$OUT_DIR"
+
+run_with_timing() {
+  local summary_path="$1"
+  local timing_path="$2"
+  shift 2
+  local start end status
+  start="$(python3 - <<'PY'
+import time
+print(time.perf_counter_ns())
+PY
+)"
+  if "$@"; then
+    status=0
+  else
+    status=$?
+  fi
+  end="$(python3 - <<'PY'
+import time
+print(time.perf_counter_ns())
+PY
+)"
+  python3 - <<'PY' "$summary_path" "$timing_path" "$status" "$start" "$end"
+import json, sys
+summary_path, timing_path, status, start, end = sys.argv[1:]
+duration_ms = max(0, (int(end) - int(start)) // 1_000_000)
+data = {"exitCode": int(status), "durationMs": duration_ms}
+try:
+    with open(summary_path, "r", encoding="utf-8") as f:
+        data["compare"] = json.load(f)
+except FileNotFoundError:
+    data["compare"] = None
+with open(timing_path, "w", encoding="utf-8") as f:
+    json.dump(data, f, indent=2)
+PY
+  return "$status"
+}
+
+OURS_OUT="$OUT_DIR/ours"
+ROUTINATOR_OUT="$OUT_DIR/routinator"
+RPKI_CLIENT_OUT="$OUT_DIR/rpki-client"
+mkdir -p "$OURS_OUT" "$ROUTINATOR_OUT" "$RPKI_CLIENT_OUT"
+
+ours_cmd=(
+  "$OURS_SCRIPT"
+  --cir "$CIR"
+  --repo-bytes-db "$REPO_BYTES_DB"
+  --out-dir "$OURS_OUT"
+  --reference-ccr "$REFERENCE_CCR"
+  --rpki-bin "$RPKI_BIN"
+  --real-rsync-bin "$REAL_RSYNC_BIN"
+)
+routinator_cmd=(
+  "$ROUTINATOR_SCRIPT"
+  --cir "$CIR"
+  --repo-bytes-db "$REPO_BYTES_DB"
+  --out-dir "$ROUTINATOR_OUT"
+  --reference-ccr "$REFERENCE_CCR"
+  --routinator-root "$ROUTINATOR_ROOT"
+  --routinator-bin "$ROUTINATOR_BIN"
+  --real-rsync-bin "$REAL_RSYNC_BIN"
+)
+rpki_client_cmd=(
+  "$RPKI_CLIENT_SCRIPT"
+  --cir "$CIR"
+  --repo-bytes-db "$REPO_BYTES_DB"
+  --out-dir "$RPKI_CLIENT_OUT"
+  --reference-ccr "$REFERENCE_CCR"
+  --build-dir "$RPKI_CLIENT_BUILD_DIR"
+  --real-rsync-bin "$REAL_RSYNC_BIN"
+)
+
+if [[ "$KEEP_DB" -eq 1 ]]; then
+  ours_cmd+=(--keep-db)
+  routinator_cmd+=(--keep-db)
+  rpki_client_cmd+=(--keep-db)
+fi
+
+ours_status=0
+routinator_status=0
+rpki_client_status=0
+if run_with_timing "$OURS_OUT/compare-summary.json" "$OURS_OUT/timing.json" "${ours_cmd[@]}"; then
+  :
+else
+  ours_status=$?
+fi
+if run_with_timing "$ROUTINATOR_OUT/compare-summary.json" "$ROUTINATOR_OUT/timing.json" "${routinator_cmd[@]}"; then
+  :
+else
+  routinator_status=$?
+fi
+if run_with_timing "$RPKI_CLIENT_OUT/compare-summary.json" "$RPKI_CLIENT_OUT/timing.json" "${rpki_client_cmd[@]}"; then
+  :
+else
+  rpki_client_status=$?
+fi
+
+SUMMARY_JSON="$OUT_DIR/summary.json"
+SUMMARY_MD="$OUT_DIR/summary.md"
+DETAIL_MD="$OUT_DIR/detail.md"
+
+python3 - <<'PY' \
+  "$CIR" \
+  "$REPO_BYTES_DB" \
+  "$REFERENCE_CCR" \
+  "$OURS_OUT" \
+  "$ROUTINATOR_OUT" \
+  "$RPKI_CLIENT_OUT" \
+  "$SUMMARY_JSON" \
+  "$SUMMARY_MD" \
+  "$DETAIL_MD"
+import json
+import sys
+from pathlib import Path
+
+cir_path, repo_bytes_db, reference_ccr, ours_out, routinator_out, rpki_client_out, summary_json, summary_md, detail_md = sys.argv[1:]
+
+participants = []
+all_match = True
+for name, out_dir in [
+    ("ours", ours_out),
+    ("routinator", routinator_out),
+    ("rpki-client", rpki_client_out),
+]:
+    out = Path(out_dir)
+    timing = json.loads((out / "timing.json").read_text(encoding="utf-8"))
+    compare = timing.get("compare") or {}
+    vrps = compare.get("vrps") or {}
+    vaps = compare.get("vaps") or {}
+    participant = {
+        "name": name,
+        "outDir": str(out),
+        "tmpRoot": str(out / ".tmp"),
+        "mirrorPath": str(out / ".tmp" / "mirror"),
+        "timingPath": str(out / "timing.json"),
+        "summaryPath": str(out / "compare-summary.json"),
+        "exitCode": timing["exitCode"],
+        "durationMs": timing["durationMs"],
+        "compareMode": compare.get("compareMode"),
+        "talCount": compare.get("talCount"),
+        "talPaths": compare.get("talPaths", []),
+        "vrps": vrps,
+        "vaps": vaps,
+        "match": bool(vrps.get("match")) and bool(vaps.get("match")) and timing["exitCode"] == 0,
+        "logPaths": [str(path) for path in sorted(out.glob("*.log"))],
+    }
+    participants.append(participant)
+    all_match = all_match and participant["match"]
+
+summary = {
+    "cirPath": cir_path,
+    "repoBytesDb": repo_bytes_db,
+    "referenceCcr": reference_ccr,
+    "participants": participants,
+    "allMatch": all_match,
+}
+Path(summary_json).write_text(json.dumps(summary, indent=2), encoding="utf-8")
+
+lines = [
+    "# CIR Replay Matrix Summary",
+    "",
+    f"- `cir`: `{cir_path}`",
+    f"- `repo_bytes_db`: `{repo_bytes_db}`",
+    f"- `reference_ccr`: `{reference_ccr}`",
+    f"- `all_match`: `{all_match}`",
+    "",
+    "| Participant | Exit | Duration (ms) | TALs | Compare mode | VRP actual/ref | VRP match | VAP actual/ref | VAP match | Log |",
+    "| --- | ---: | ---: | ---: | --- | --- | --- | --- | --- | --- |",
+]
+for participant in participants:
+    vrps = participant["vrps"] or {}
+    vaps = participant["vaps"] or {}
+    log_path = participant["logPaths"][0] if participant["logPaths"] else ""
+    lines.append(
+        "| {name} | {exit_code} | {duration_ms} | {tal_count} | {compare_mode} | {vrp_actual}/{vrp_ref} | {vrp_match} | {vap_actual}/{vap_ref} | {vap_match} | `{log_path}` |".format(
+            name=participant["name"],
+            exit_code=participant["exitCode"],
+            duration_ms=participant["durationMs"],
+            tal_count=participant.get("talCount") if participant.get("talCount") is not None else "-",
+            compare_mode=participant.get("compareMode") or "-",
+            vrp_actual=vrps.get("actual", "-"),
+            vrp_ref=vrps.get("reference", "-"),
+            vrp_match=vrps.get("match", False),
+            vap_actual=vaps.get("actual", "-"),
+            vap_ref=vaps.get("reference", "-"),
+            vap_match=vaps.get("match", False),
+            log_path=log_path,
+        )
+    )
+Path(summary_md).write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+detail_lines = [
+    "# CIR Replay Matrix Detail",
+    "",
+]
+for participant in participants:
+    vrps = participant["vrps"] or {}
+    vaps = participant["vaps"] or {}
+    detail_lines.extend([
+        f"## {participant['name']}",
+        f"- `exit_code`: `{participant['exitCode']}`",
+        f"- `duration_ms`: `{participant['durationMs']}`",
+        f"- `out_dir`: `{participant['outDir']}`",
+        f"- `tmp_root`: `{participant['tmpRoot']}`",
+        f"- `mirror_path`: `{participant['mirrorPath']}`",
+        f"- `summary_path`: `{participant['summaryPath']}`",
+        f"- `timing_path`: `{participant['timingPath']}`",
+        f"- `compare_mode`: `{participant.get('compareMode')}`",
+        f"- `tal_count`: `{participant.get('talCount')}`",
+        f"- `log_paths`: `{', '.join(participant['logPaths'])}`",
+        f"- `vrps`: `actual={vrps.get('actual', '-')}` `reference={vrps.get('reference', '-')}` `match={vrps.get('match', False)}`",
+        f"- `vaps`: `actual={vaps.get('actual', '-')}` `reference={vaps.get('reference', '-')}` `match={vaps.get('match', False)}`",
+        f"- `vrps.only_in_actual`: `{vrps.get('only_in_actual', [])}`",
+        f"- `vrps.only_in_reference`: `{vrps.get('only_in_reference', [])}`",
+        f"- `vaps.only_in_actual`: `{vaps.get('only_in_actual', [])}`",
+        f"- `vaps.only_in_reference`: `{vaps.get('only_in_reference', [])}`",
+        "",
+    ])
+Path(detail_md).write_text("\n".join(detail_lines), encoding="utf-8")
+PY
+
+if [[ "$ours_status" -ne 0 || "$routinator_status" -ne 0 || "$rpki_client_status" -ne 0 ]]; then
+  exit 1
+fi
+
+all_match="$(python3 - <<'PY' "$SUMMARY_JSON"
+import json,sys
+print("true" if json.load(open(sys.argv[1]))["allMatch"] else "false")
+PY
+)"
+if [[ "$all_match" != "true" ]]; then
+  exit 1
+fi
+
+echo "done: $OUT_DIR"

scripts/cir/run_cir_replay_ours.sh

@@ -0,0 +1,252 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_replay_ours.sh \
+    --cir <path> \
+    --repo-bytes-db <path> \
+    --out-dir <path> \
+    --reference-ccr <path> \
+    [--keep-db] \
+    [--write-actual-ccr] \
+    [--write-report-json] \
+    [--report-json-compact] \
+    [--phase2-object-workers <n>] \
+    [--phase2-worker-queue-capacity <n>] \
+    [--rpki-bin <path>] \
+    [--real-rsync-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+CIR=""
+REPO_BYTES_DB=""
+OUT_DIR=""
+REFERENCE_CCR=""
+KEEP_DB=0
+WRITE_ACTUAL_CCR=0
+WRITE_REPORT_JSON=0
+REPORT_JSON_COMPACT=0
+PHASE2_OBJECT_WORKERS="${CIR_REPLAY_PHASE2_OBJECT_WORKERS:-4}"
+PHASE2_WORKER_QUEUE_CAPACITY="${CIR_REPLAY_PHASE2_WORKER_QUEUE_CAPACITY:-64}"
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+CIR_MATERIALIZE_BIN="${CIR_MATERIALIZE_BIN:-$ROOT_DIR/target/release/cir_materialize}"
+CIR_EXTRACT_INPUTS_BIN="${CIR_EXTRACT_INPUTS_BIN:-$ROOT_DIR/target/release/cir_extract_inputs}"
+CCR_TO_COMPARE_VIEWS_BIN="${CCR_TO_COMPARE_VIEWS_BIN:-$ROOT_DIR/target/release/ccr_to_compare_views}"
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+WRAPPER="$ROOT_DIR/scripts/cir/cir-rsync-wrapper"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --cir) CIR="$2"; shift 2 ;;
+    --repo-bytes-db) REPO_BYTES_DB="$2"; shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
+    --keep-db) KEEP_DB=1; shift ;;
+    --write-actual-ccr) WRITE_ACTUAL_CCR=1; shift ;;
+    --write-report-json) WRITE_REPORT_JSON=1; shift ;;
+    --report-json-compact) WRITE_REPORT_JSON=1; REPORT_JSON_COMPACT=1; shift ;;
+    --phase2-object-workers) PHASE2_OBJECT_WORKERS="$2"; shift 2 ;;
+    --phase2-worker-queue-capacity) PHASE2_WORKER_QUEUE_CAPACITY="$2"; shift 2 ;;
+    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$CIR" && -n "$REPO_BYTES_DB" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
+  usage >&2
+  exit 2
+}
+
+mkdir -p "$OUT_DIR"
+
+needs_build=0
+if [[ ! -x "$RPKI_BIN" || ! -x "$CIR_MATERIALIZE_BIN" || ! -x "$CIR_EXTRACT_INPUTS_BIN" || ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
+  needs_build=1
+elif [[ "$RPKI_BIN" == "$ROOT_DIR/target/release/rpki" ]] && find "$ROOT_DIR/src" "$ROOT_DIR/Cargo.toml" -newer "$RPKI_BIN" -print -quit | grep -q .; then
+  needs_build=1
+fi
+if [[ "$needs_build" -eq 1 ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin rpki --bin cir_materialize --bin cir_extract_inputs --bin ccr_to_compare_views
+  )
+fi
+
+TMP_ROOT="$OUT_DIR/.tmp"
+TALS_DIR="$TMP_ROOT/tals"
+META_JSON="$TMP_ROOT/meta.json"
+MIRROR_ROOT="$TMP_ROOT/mirror"
+DB_DIR="$TMP_ROOT/work-db"
+REPLAY_RAW_STORE_DB="$TMP_ROOT/replay-raw-store.db"
+REPLAY_REPO_BYTES_DB="$TMP_ROOT/replay-repo-bytes.db"
+ACTUAL_CCR="$OUT_DIR/actual.ccr"
+ACTUAL_REPORT="$OUT_DIR/report.json"
+ACTUAL_VRPS="$OUT_DIR/actual-vrps.csv"
+ACTUAL_VAPS="$OUT_DIR/actual-vaps.csv"
+REF_VRPS="$OUT_DIR/reference-vrps.csv"
+REF_VAPS="$OUT_DIR/reference-vaps.csv"
+COMPARE_JSON="$OUT_DIR/compare-summary.json"
+RUN_LOG="$OUT_DIR/run.log"
+
+rm -rf "$TMP_ROOT"
+mkdir -p "$TMP_ROOT"
+
+"$CIR_EXTRACT_INPUTS_BIN" --cir "$CIR" --tals-dir "$TALS_DIR" --meta-json "$META_JSON"
+materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --repo-bytes-db "$REPO_BYTES_DB" --mirror-root "$MIRROR_ROOT")
+if [[ "$KEEP_DB" -eq 1 ]]; then
+  materialize_cmd+=(--keep-db)
+fi
+"${materialize_cmd[@]}"
+
+VALIDATION_TIME="$(python3 - <<'PY' "$META_JSON"
+import json,sys
+print(json.load(open(sys.argv[1]))["validationTime"])
+PY
+)"
+mapfile -t TAL_PATHS < <(python3 - <<'PY' "$META_JSON"
+import json, sys
+for item in json.load(open(sys.argv[1], encoding="utf-8"))["talFiles"]:
+    print(item["path"])
+PY
+)
+TAL_ARGS=()
+for tal_path in "${TAL_PATHS[@]}"; do
+  TAL_ARGS+=(--tal-path "$tal_path")
+done
+
+export CIR_MIRROR_ROOT="$(python3 - <<'PY' "$MIRROR_ROOT"
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
+export CIR_LOCAL_LINK_MODE=1
+
+REPORT_JSON_ARGS=(--skip-report-build)
+VCIR_ARGS=(--skip-vcir-persist)
+if [[ "$WRITE_REPORT_JSON" -eq 1 ]]; then
+  REPORT_JSON_ARGS=(--report-json "$ACTUAL_REPORT")
+  if [[ "$REPORT_JSON_COMPACT" -eq 1 ]]; then
+    REPORT_JSON_ARGS+=(--report-json-compact)
+  fi
+fi
+CCR_ARGS=()
+if [[ "$WRITE_ACTUAL_CCR" -eq 1 ]]; then
+  CCR_ARGS=(--ccr-out "$ACTUAL_CCR")
+fi
+
+"$RPKI_BIN" \
+  --db "$DB_DIR" \
+  --raw-store-db "$REPLAY_RAW_STORE_DB" \
+  --repo-bytes-db "$REPLAY_REPO_BYTES_DB" \
+  "${TAL_ARGS[@]}" \
+  --parallel-phase2-object-workers "$PHASE2_OBJECT_WORKERS" \
+  --parallel-phase2-worker-queue-capacity "$PHASE2_WORKER_QUEUE_CAPACITY" \
+  --disable-rrdp \
+  --rsync-command "$WRAPPER" \
+  --validation-time "$VALIDATION_TIME" \
+  "${CCR_ARGS[@]}" \
+  --vrps-csv-out "$ACTUAL_VRPS" \
+  --vaps-csv-out "$ACTUAL_VAPS" \
+  --compare-view-trust-anchor unknown \
+  "${VCIR_ARGS[@]}" \
+  "${REPORT_JSON_ARGS[@]}" \
+  >"$RUN_LOG" 2>&1
+
+sort_compare_csv() {
+  local path="$1"
+  local tmp="${path}.sorted.tmp"
+  {
+    head -n 1 "$path"
+    tail -n +2 "$path" | LC_ALL=C sort -u
+  } >"$tmp"
+  mv "$tmp" "$path"
+}
+
+sort_compare_csv "$ACTUAL_VRPS"
+sort_compare_csv "$ACTUAL_VAPS"
+"$CCR_TO_COMPARE_VIEWS_BIN" --ccr "$REFERENCE_CCR" --vrps-out "$REF_VRPS" --vaps-out "$REF_VAPS" --trust-anchor unknown
+
+python3 - <<'PY' "$ACTUAL_VRPS" "$REF_VRPS" "$ACTUAL_VAPS" "$REF_VAPS" "$COMPARE_JSON" "$META_JSON" "$WRITE_REPORT_JSON" "$WRITE_ACTUAL_CCR" "$PHASE2_OBJECT_WORKERS" "$PHASE2_WORKER_QUEUE_CAPACITY"
+import csv, json, sys
+
+def next_row(reader):
+    try:
+        return tuple(next(reader))
+    except StopIteration:
+        return None
+
+def compare_sorted_csv(actual_path, ref_path):
+    actual_count = 0
+    ref_count = 0
+    only_actual_count = 0
+    only_ref_count = 0
+    only_actual_sample = []
+    only_ref_sample = []
+    with open(actual_path, newline="") as actual_file, open(ref_path, newline="") as ref_file:
+        actual_reader = csv.reader(actual_file)
+        ref_reader = csv.reader(ref_file)
+        next(actual_reader, None)
+        next(ref_reader, None)
+        actual = next_row(actual_reader)
+        ref = next_row(ref_reader)
+        while actual is not None or ref is not None:
+            if ref is None or (actual is not None and actual < ref):
+                actual_count += 1
+                only_actual_count += 1
+                if len(only_actual_sample) < 20:
+                    only_actual_sample.append(list(actual))
+                actual = next_row(actual_reader)
+            elif actual is None or ref < actual:
+                ref_count += 1
+                only_ref_count += 1
+                if len(only_ref_sample) < 20:
+                    only_ref_sample.append(list(ref))
+                ref = next_row(ref_reader)
+            else:
+                actual_count += 1
+                ref_count += 1
+                actual = next_row(actual_reader)
+                ref = next_row(ref_reader)
+    return {
+        "actual": actual_count,
+        "reference": ref_count,
+        "only_in_actual": only_actual_sample,
+        "only_in_reference": only_ref_sample,
+        "only_in_actual_count": only_actual_count,
+        "only_in_reference_count": only_ref_count,
+        "match": only_actual_count == 0 and only_ref_count == 0,
+    }
+
+vrps = compare_sorted_csv(sys.argv[1], sys.argv[2])
+vaps = compare_sorted_csv(sys.argv[3], sys.argv[4])
+meta = json.load(open(sys.argv[6], encoding="utf-8"))
+summary = {
+  "compareMode": "trust-anchor-agnostic",
+  "talCount": len(meta["talFiles"]),
+  "talPaths": [item["path"] for item in meta["talFiles"]],
+  "actualCcrWritten": sys.argv[8] == "1",
+  "reportJsonWritten": sys.argv[7] == "1",
+  "replayParallelism": {
+    "phase2ObjectWorkers": int(sys.argv[9]),
+    "phase2WorkerQueueCapacity": int(sys.argv[10]),
+  },
+  "vrps": vrps,
+  "vaps": vaps,
+}
+with open(sys.argv[5], "w") as f:
+    json.dump(summary, f, indent=2)
+PY
+
+if [[ "$KEEP_DB" -ne 1 ]]; then
+  rm -rf "$TMP_ROOT"
+fi
+
+echo "done: $OUT_DIR"

scripts/cir/run_cir_replay_routinator.sh

@@ -0,0 +1,239 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_replay_routinator.sh \
+    --cir <path> \
+    --repo-bytes-db <path> \
+    --out-dir <path> \
+    --reference-ccr <path> \
+    [--keep-db] \
+    [--routinator-root <path>] \
+    [--routinator-bin <path>] \
+    [--real-rsync-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RPKI_DEV_ROOT="${RPKI_DEV_ROOT:-$ROOT_DIR}"
+
+CIR=""
+REPO_BYTES_DB=""
+OUT_DIR=""
+REFERENCE_CCR=""
+KEEP_DB=0
+ROUTINATOR_ROOT="${ROUTINATOR_ROOT:-/home/yuyr/dev/rust_playground/routinator}"
+ROUTINATOR_BIN="${ROUTINATOR_BIN:-$ROUTINATOR_ROOT/target/debug/routinator}"
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+CIR_MATERIALIZE_BIN="${CIR_MATERIALIZE_BIN:-$ROOT_DIR/target/release/cir_materialize}"
+CIR_EXTRACT_INPUTS_BIN="${CIR_EXTRACT_INPUTS_BIN:-$ROOT_DIR/target/release/cir_extract_inputs}"
+CCR_TO_COMPARE_VIEWS_BIN="${CCR_TO_COMPARE_VIEWS_BIN:-$ROOT_DIR/target/release/ccr_to_compare_views}"
+WRAPPER="$ROOT_DIR/scripts/cir/cir-rsync-wrapper"
+JSON_TO_VAPS="$ROOT_DIR/scripts/cir/json_to_vaps_csv.py"
+FAKETIME_LIB="${FAKETIME_LIB:-$ROOT_DIR/target/tools/faketime_pkg/extracted/libfaketime/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --cir) CIR="$2"; shift 2 ;;
+    --repo-bytes-db) REPO_BYTES_DB="$2"; shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
+    --keep-db) KEEP_DB=1; shift ;;
+    --routinator-root) ROUTINATOR_ROOT="$2"; shift 2 ;;
+    --routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$CIR" && -n "$REPO_BYTES_DB" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
+  usage >&2
+  exit 2
+}
+if [[ ! -x "$ROUTINATOR_BIN" ]]; then
+  echo "routinator binary not executable: $ROUTINATOR_BIN" >&2
+  exit 2
+fi
+
+mkdir -p "$OUT_DIR"
+if [[ ! -x "$CIR_MATERIALIZE_BIN" || ! -x "$CIR_EXTRACT_INPUTS_BIN" || ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin cir_materialize --bin cir_extract_inputs --bin ccr_to_compare_views
+  )
+fi
+
+TMP_ROOT="$OUT_DIR/.tmp"
+TALS_DIR="$TMP_ROOT/tals"
+META_JSON="$TMP_ROOT/meta.json"
+MIRROR_ROOT="$TMP_ROOT/mirror"
+WORK_REPO="$TMP_ROOT/repository"
+RUN_LOG="$OUT_DIR/routinator.log"
+ACTUAL_VRPS="$OUT_DIR/actual-vrps.csv"
+ACTUAL_VAPS_JSON="$OUT_DIR/actual-vaps.json"
+ACTUAL_VAPS="$OUT_DIR/actual-vaps.csv"
+REF_VRPS="$OUT_DIR/reference-vrps.csv"
+REF_VAPS="$OUT_DIR/reference-vaps.csv"
+SUMMARY_JSON="$OUT_DIR/compare-summary.json"
+
+rm -rf "$TMP_ROOT"
+mkdir -p "$TMP_ROOT"
+
+"$CIR_EXTRACT_INPUTS_BIN" --cir "$CIR" --tals-dir "$TALS_DIR" --meta-json "$META_JSON"
+python3 - <<'PY' "$TALS_DIR"
+from pathlib import Path
+import sys
+for tal in Path(sys.argv[1]).glob("*.tal"):
+    lines = tal.read_text(encoding="utf-8").splitlines()
+    rsync_uris = [line for line in lines if line.startswith("rsync://")]
+    base64_lines = []
+    seen_sep = False
+    for line in lines:
+        if seen_sep:
+            if line.strip():
+                base64_lines.append(line)
+        elif line.strip() == "":
+            seen_sep = True
+    tal.write_text("\n".join(rsync_uris) + "\n\n" + "\n".join(base64_lines) + "\n", encoding="utf-8")
+PY
+materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --repo-bytes-db "$REPO_BYTES_DB" --mirror-root "$MIRROR_ROOT")
+if [[ "$KEEP_DB" -eq 1 ]]; then
+  materialize_cmd+=(--keep-db)
+fi
+"${materialize_cmd[@]}"
+
+VALIDATION_TIME="$(python3 - <<'PY' "$META_JSON"
+import json,sys
+print(json.load(open(sys.argv[1]))["validationTime"])
+PY
+)"
+mapfile -t TAL_PATHS < <(python3 - <<'PY' "$META_JSON"
+import json, sys
+for item in json.load(open(sys.argv[1], encoding="utf-8"))["talFiles"]:
+    print(item["path"])
+PY
+)
+COMPARE_TRUST_ANCHOR="unknown"
+FAKE_EPOCH="$(python3 - <<'PY' "$VALIDATION_TIME"
+from datetime import datetime, timezone
+import sys
+dt = datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).astimezone(timezone.utc)
+print(int(dt.timestamp()))
+PY
+)"
+
+export CIR_MIRROR_ROOT="$(python3 - <<'PY' "$MIRROR_ROOT"
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
+export CIR_LOCAL_LINK_MODE=1
+env \
+  LD_PRELOAD="$FAKETIME_LIB" \
+  FAKETIME_FMT=%s \
+  FAKETIME="$FAKE_EPOCH" \
+  FAKETIME_DONT_FAKE_MONOTONIC=1 \
+  "$ROUTINATOR_BIN" \
+  --repository-dir "$WORK_REPO" \
+  --disable-rrdp \
+  --rsync-command "$WRAPPER" \
+  --no-rir-tals \
+  --extra-tals-dir "$TALS_DIR" \
+  --enable-aspa \
+  update --complete >"$RUN_LOG" 2>&1 || true
+
+env \
+  LD_PRELOAD="$FAKETIME_LIB" \
+  FAKETIME_FMT=%s \
+  FAKETIME="$FAKE_EPOCH" \
+  FAKETIME_DONT_FAKE_MONOTONIC=1 \
+  "$ROUTINATOR_BIN" \
+  --repository-dir "$WORK_REPO" \
+  --disable-rrdp \
+  --rsync-command "$WRAPPER" \
+  --no-rir-tals \
+  --extra-tals-dir "$TALS_DIR" \
+  --enable-aspa \
+  vrps --noupdate -o "$ACTUAL_VRPS" >>"$RUN_LOG" 2>&1
+
+env \
+  LD_PRELOAD="$FAKETIME_LIB" \
+  FAKETIME_FMT=%s \
+  FAKETIME="$FAKE_EPOCH" \
+  FAKETIME_DONT_FAKE_MONOTONIC=1 \
+  "$ROUTINATOR_BIN" \
+  --repository-dir "$WORK_REPO" \
+  --disable-rrdp \
+  --rsync-command "$WRAPPER" \
+  --no-rir-tals \
+  --extra-tals-dir "$TALS_DIR" \
+  --enable-aspa \
+  vrps --noupdate --format json -o "$ACTUAL_VAPS_JSON" >>"$RUN_LOG" 2>&1
+
+python3 "$JSON_TO_VAPS" --input "$ACTUAL_VAPS_JSON" --csv-out "$ACTUAL_VAPS"
+
+normalize_trust_anchor_csv() {
+  python3 - <<'PY' "$1" "$2"
+import csv
+import sys
+from pathlib import Path
+
+path = Path(sys.argv[1])
+trust_anchor = sys.argv[2]
+rows = list(csv.reader(path.open(newline="", encoding="utf-8")))
+if rows:
+    for row in rows[1:]:
+        if row:
+            row[-1] = trust_anchor
+with path.open("w", newline="", encoding="utf-8") as fh:
+    csv.writer(fh).writerows(rows)
+PY
+}
+
+normalize_trust_anchor_csv "$ACTUAL_VRPS" "$COMPARE_TRUST_ANCHOR"
+normalize_trust_anchor_csv "$ACTUAL_VAPS" "$COMPARE_TRUST_ANCHOR"
+"$CCR_TO_COMPARE_VIEWS_BIN" --ccr "$REFERENCE_CCR" --vrps-out "$REF_VRPS" --vaps-out "$REF_VAPS" --trust-anchor "$COMPARE_TRUST_ANCHOR"
+
+python3 - <<'PY' "$ACTUAL_VRPS" "$REF_VRPS" "$ACTUAL_VAPS" "$REF_VAPS" "$SUMMARY_JSON" "$META_JSON"
+import csv, json, sys
+def rows(path):
+    with open(path, newline="") as f:
+        return list(csv.reader(f))[1:]
+actual_vrps = {tuple(r) for r in rows(sys.argv[1])}
+ref_vrps = {tuple(r) for r in rows(sys.argv[2])}
+actual_vaps = {tuple(r) for r in rows(sys.argv[3])}
+ref_vaps = {tuple(r) for r in rows(sys.argv[4])}
+meta = json.load(open(sys.argv[6], encoding="utf-8"))
+summary = {
+  "compareMode": "trust-anchor-agnostic",
+  "talCount": len(meta["talFiles"]),
+  "talPaths": [item["path"] for item in meta["talFiles"]],
+  "vrps": {
+    "actual": len(actual_vrps),
+    "reference": len(ref_vrps),
+    "match": actual_vrps == ref_vrps,
+    "only_in_actual": sorted(actual_vrps - ref_vrps)[:20],
+    "only_in_reference": sorted(ref_vrps - actual_vrps)[:20],
+  },
+  "vaps": {
+    "actual": len(actual_vaps),
+    "reference": len(ref_vaps),
+    "match": actual_vaps == ref_vaps,
+    "only_in_actual": sorted(actual_vaps - ref_vaps)[:20],
+    "only_in_reference": sorted(ref_vaps - actual_vaps)[:20],
+  }
+}
+with open(sys.argv[5], "w") as f:
+    json.dump(summary, f, indent=2)
+PY
+
+if [[ "$KEEP_DB" -ne 1 ]]; then
+  rm -rf "$TMP_ROOT"
+fi
+
+echo "done: $OUT_DIR"

scripts/cir/run_cir_replay_rpki_client.sh

@@ -0,0 +1,248 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_replay_rpki_client.sh \
+    --cir <path> \
+    --repo-bytes-db <path> \
+    --out-dir <path> \
+    --reference-ccr <path> \
+    [--build-dir <path> | --rpki-client-bin <path>] \
+    [--keep-db] \
+    [--real-rsync-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+CIR=""
+REPO_BYTES_DB=""
+OUT_DIR=""
+REFERENCE_CCR=""
+BUILD_DIR=""
+RPKI_CLIENT_BIN=""
+KEEP_DB=0
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+CIR_MATERIALIZE_BIN="${CIR_MATERIALIZE_BIN:-$ROOT_DIR/target/release/cir_materialize}"
+CIR_EXTRACT_INPUTS_BIN="${CIR_EXTRACT_INPUTS_BIN:-$ROOT_DIR/target/release/cir_extract_inputs}"
+CCR_TO_COMPARE_VIEWS_BIN="${CCR_TO_COMPARE_VIEWS_BIN:-$ROOT_DIR/target/release/ccr_to_compare_views}"
+WRAPPER="$ROOT_DIR/scripts/cir/cir-rsync-wrapper"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --cir) CIR="$2"; shift 2 ;;
+    --repo-bytes-db) REPO_BYTES_DB="$2"; shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
+    --build-dir) BUILD_DIR="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --keep-db) KEEP_DB=1; shift ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$CIR" && -n "$REPO_BYTES_DB" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
+  usage >&2
+  exit 2
+}
+if [[ -z "$BUILD_DIR" && -z "$RPKI_CLIENT_BIN" ]]; then
+  usage >&2
+  exit 2
+fi
+if [[ -z "$RPKI_CLIENT_BIN" ]]; then
+  RPKI_CLIENT_BIN="$BUILD_DIR/src/rpki-client"
+fi
+if [[ ! -x "$RPKI_CLIENT_BIN" ]]; then
+  echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2
+  exit 2
+fi
+
+mkdir -p "$OUT_DIR"
+if [[ ! -x "$CIR_MATERIALIZE_BIN" || ! -x "$CIR_EXTRACT_INPUTS_BIN" || ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin cir_materialize --bin cir_extract_inputs --bin ccr_to_compare_views
+  )
+fi
+
+TMP_ROOT="$OUT_DIR/.tmp"
+TALS_DIR="$TMP_ROOT/tals"
+META_JSON="$TMP_ROOT/meta.json"
+MIRROR_ROOT="$TMP_ROOT/mirror"
+CACHE_DIR="$TMP_ROOT/cache"
+OUT_CCR_DIR="$TMP_ROOT/out"
+RUN_LOG="$OUT_DIR/rpki-client.log"
+ACTUAL_VRPS="$OUT_DIR/actual-vrps.csv"
+ACTUAL_VAPS="$OUT_DIR/actual-vaps.csv"
+ACTUAL_VAPS_META="$OUT_DIR/actual-vaps-meta.json"
+ACTUAL_VRPS_META="$OUT_DIR/actual-vrps-meta.json"
+REF_VRPS="$OUT_DIR/reference-vrps.csv"
+REF_VAPS="$OUT_DIR/reference-vaps.csv"
+SUMMARY_JSON="$OUT_DIR/compare-summary.json"
+
+rm -rf "$TMP_ROOT"
+mkdir -p "$TMP_ROOT"
+
+"$CIR_EXTRACT_INPUTS_BIN" --cir "$CIR" --tals-dir "$TALS_DIR" --meta-json "$META_JSON"
+python3 - <<'PY' "$TALS_DIR"
+from pathlib import Path
+import sys
+for tal in Path(sys.argv[1]).glob("*.tal"):
+    lines = tal.read_text(encoding="utf-8").splitlines()
+    rsync_uris = [line for line in lines if line.startswith("rsync://")]
+    base64_lines = []
+    seen_sep = False
+    for line in lines:
+        if seen_sep:
+            if line.strip():
+                base64_lines.append(line)
+        elif line.strip() == "":
+            seen_sep = True
+    tal.write_text("\n".join(rsync_uris) + "\n\n" + "\n".join(base64_lines) + "\n", encoding="utf-8")
+PY
+materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --repo-bytes-db "$REPO_BYTES_DB" --mirror-root "$MIRROR_ROOT")
+if [[ "$KEEP_DB" -eq 1 ]]; then
+  materialize_cmd+=(--keep-db)
+fi
+"${materialize_cmd[@]}"
+
+VALIDATION_EPOCH="$(python3 - <<'PY' "$META_JSON"
+from datetime import datetime, timezone
+import json, sys
+vt = json.load(open(sys.argv[1]))["validationTime"]
+dt = datetime.fromisoformat(vt.replace("Z", "+00:00")).astimezone(timezone.utc)
+print(int(dt.timestamp()))
+PY
+)"
+mapfile -t TAL_PATHS < <(python3 - <<'PY' "$META_JSON"
+import json, sys
+for item in json.load(open(sys.argv[1], encoding="utf-8"))["talFiles"]:
+    print(item["path"])
+PY
+)
+CLIENT_TAL_ARGS=()
+for tal_path in "${TAL_PATHS[@]}"; do
+  CLIENT_TAL_ARGS+=(-t "$tal_path")
+done
+COMPARE_TRUST_ANCHOR="unknown"
+
+export CIR_MIRROR_ROOT="$(python3 - <<'PY' "$MIRROR_ROOT"
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
+export CIR_LOCAL_LINK_MODE=1
+
+mkdir -p "$CACHE_DIR" "$OUT_CCR_DIR"
+chmod -R 0777 "$TMP_ROOT"
+"$RPKI_CLIENT_BIN" \
+  -R \
+  -e "$WRAPPER" \
+  -P "$VALIDATION_EPOCH" \
+  "${CLIENT_TAL_ARGS[@]}" \
+  -d "$CACHE_DIR" \
+  "$OUT_CCR_DIR" >"$RUN_LOG" 2>&1
+
+if [[ -f "$OUT_CCR_DIR/rpki.ccr" ]]; then
+  "$CCR_TO_COMPARE_VIEWS_BIN" \
+    --ccr "$OUT_CCR_DIR/rpki.ccr" \
+    --vrps-out "$ACTUAL_VRPS" \
+    --vaps-out "$ACTUAL_VAPS" \
+    --trust-anchor "$COMPARE_TRUST_ANCHOR"
+else
+  python3 - <<'PY' "$OUT_CCR_DIR/json" "$ACTUAL_VRPS" "$ACTUAL_VAPS" "$COMPARE_TRUST_ANCHOR"
+import csv
+import json
+import sys
+from pathlib import Path
+
+json_path = Path(sys.argv[1])
+vrps_out = Path(sys.argv[2])
+vaps_out = Path(sys.argv[3])
+compare_ta = sys.argv[4]
+
+if not json_path.is_file():
+    raise SystemExit(f"rpki-client output has neither rpki.ccr nor json: {json_path}")
+
+data = json.loads(json_path.read_text(encoding="utf-8"))
+
+vrps_out.parent.mkdir(parents=True, exist_ok=True)
+with vrps_out.open("w", newline="", encoding="utf-8") as fh:
+    writer = csv.writer(fh)
+    writer.writerow(["ASN", "IP Prefix", "Max Length", "Trust Anchor"])
+    for roa in data.get("roas", []):
+        writer.writerow([
+            f"AS{roa['asn']}",
+            roa["prefix"],
+            str(roa["maxLength"]),
+            compare_ta,
+        ])
+
+with vaps_out.open("w", newline="", encoding="utf-8") as fh:
+    writer = csv.writer(fh)
+    writer.writerow(["Customer ASN", "Providers", "Trust Anchor"])
+    for aspa in data.get("aspas", []):
+        providers = ";".join(f"AS{item}" for item in sorted(aspa.get("providers", [])))
+        writer.writerow([
+            f"AS{aspa['customer_asid']}",
+            providers,
+            compare_ta,
+        ])
+PY
+fi
+
+python3 - <<'PY' "$ACTUAL_VRPS" "$ACTUAL_VAPS" "$ACTUAL_VRPS_META" "$ACTUAL_VAPS_META"
+import csv, json, sys
+def count_rows(path):
+    with open(path, newline="") as f:
+        rows = list(csv.reader(f))
+    return max(len(rows) - 1, 0)
+json.dump({"count": count_rows(sys.argv[1])}, open(sys.argv[3], "w"), indent=2)
+json.dump({"count": count_rows(sys.argv[2])}, open(sys.argv[4], "w"), indent=2)
+PY
+
+"$CCR_TO_COMPARE_VIEWS_BIN" --ccr "$REFERENCE_CCR" --vrps-out "$REF_VRPS" --vaps-out "$REF_VAPS" --trust-anchor "$COMPARE_TRUST_ANCHOR"
+
+python3 - <<'PY' "$ACTUAL_VRPS" "$REF_VRPS" "$ACTUAL_VAPS" "$REF_VAPS" "$SUMMARY_JSON" "$META_JSON"
+import csv, json, sys
+def rows(path):
+    with open(path, newline="") as f:
+        return list(csv.reader(f))[1:]
+actual_vrps = {tuple(r) for r in rows(sys.argv[1])}
+ref_vrps = {tuple(r) for r in rows(sys.argv[2])}
+actual_vaps = {tuple(r) for r in rows(sys.argv[3])}
+ref_vaps = {tuple(r) for r in rows(sys.argv[4])}
+meta = json.load(open(sys.argv[6], encoding="utf-8"))
+summary = {
+  "compareMode": "trust-anchor-agnostic",
+  "talCount": len(meta["talFiles"]),
+  "talPaths": [item["path"] for item in meta["talFiles"]],
+  "vrps": {
+    "actual": len(actual_vrps),
+    "reference": len(ref_vrps),
+    "match": actual_vrps == ref_vrps,
+    "only_in_actual": sorted(actual_vrps - ref_vrps)[:20],
+    "only_in_reference": sorted(ref_vrps - actual_vrps)[:20],
+  },
+  "vaps": {
+    "actual": len(actual_vaps),
+    "reference": len(ref_vaps),
+    "match": actual_vaps == ref_vaps,
+    "only_in_actual": sorted(actual_vaps - ref_vaps)[:20],
+    "only_in_reference": sorted(ref_vaps - actual_vaps)[:20],
+  }
+}
+with open(sys.argv[5], "w") as f:
+    json.dump(summary, f, indent=2)
+PY
+
+if [[ "$KEEP_DB" -ne 1 ]]; then
+  rm -rf "$TMP_ROOT"
+fi
+
+echo "done: $OUT_DIR"

scripts/cir/run_cir_replay_sequence_ours.sh

@@ -0,0 +1,147 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_replay_sequence_ours.sh \
+    --sequence-root <path> \
+    [--rpki-bin <path>] \
+    [--real-rsync-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SEQUENCE_ROOT=""
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+STEP_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_ours.sh"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --sequence-root) SEQUENCE_ROOT="$2"; shift 2 ;;
+    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$SEQUENCE_ROOT" ]] || { usage >&2; exit 2; }
+
+SEQUENCE_ROOT="$(python3 - <<'PY' "$SEQUENCE_ROOT"
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+
+SUMMARY_JSON="$SEQUENCE_ROOT/sequence-summary.json"
+SUMMARY_MD="$SEQUENCE_ROOT/sequence-summary.md"
+DETAIL_JSON="$SEQUENCE_ROOT/sequence-detail.json"
+
+python3 - <<'PY' "$SEQUENCE_ROOT" "$SUMMARY_JSON" "$SUMMARY_MD" "$DETAIL_JSON" "$STEP_SCRIPT" "$RPKI_BIN" "$REAL_RSYNC_BIN"
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+sequence_root = Path(sys.argv[1])
+summary_json = Path(sys.argv[2])
+summary_md = Path(sys.argv[3])
+detail_json = Path(sys.argv[4])
+step_script = Path(sys.argv[5])
+rpki_bin = sys.argv[6]
+real_rsync_bin = sys.argv[7]
+
+sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
+repo_bytes_db = sequence_root / sequence["repoBytesDbPath"]
+steps = sequence["steps"]
+
+results = []
+all_match = True
+for step in steps:
+    step_id = step["stepId"]
+    out_dir = sequence_root / "replay-ours" / step_id
+    out_dir.parent.mkdir(parents=True, exist_ok=True)
+    cmd = [
+        str(step_script),
+        "--cir",
+        str(sequence_root / step["cirPath"]),
+        "--out-dir",
+        str(out_dir),
+        "--reference-ccr",
+        str(sequence_root / step["ccrPath"]),
+        "--rpki-bin",
+        rpki_bin,
+        "--real-rsync-bin",
+        real_rsync_bin,
+    ]
+    cmd.extend(["--repo-bytes-db", str(repo_bytes_db)])
+    proc = subprocess.run(cmd, capture_output=True, text=True)
+    if proc.returncode != 0:
+        raise SystemExit(
+            f"ours sequence replay failed for {step_id}: stdout={proc.stdout} stderr={proc.stderr}"
+        )
+    compare = json.loads((out_dir / "compare-summary.json").read_text(encoding="utf-8"))
+    timing = json.loads((out_dir / "timing.json").read_text(encoding="utf-8")) if (out_dir / "timing.json").exists() else {}
+    record = {
+        "stepId": step_id,
+        "kind": step["kind"],
+        "validationTime": step["validationTime"],
+        "outDir": str(out_dir),
+        "comparePath": str(out_dir / "compare-summary.json"),
+        "timingPath": str(out_dir / "timing.json"),
+        "compareMode": compare.get("compareMode"),
+        "talCount": compare.get("talCount"),
+        "talPaths": compare.get("talPaths", []),
+        "compare": compare,
+        "timing": timing,
+        "match": bool(compare["vrps"]["match"]) and bool(compare["vaps"]["match"]),
+    }
+    all_match = all_match and record["match"]
+    results.append(record)
+
+summary = {
+    "version": 1,
+    "participant": "ours",
+    "sequenceRoot": str(sequence_root),
+    "stepCount": len(results),
+    "allMatch": all_match,
+    "steps": results,
+}
+summary_json.write_text(json.dumps(summary, indent=2), encoding="utf-8")
+detail_json.write_text(json.dumps(results, indent=2), encoding="utf-8")
+
+lines = [
+    "# Ours CIR Sequence Replay Summary",
+    "",
+    f"- `sequence_root`: `{sequence_root}`",
+    f"- `step_count`: `{len(results)}`",
+    f"- `all_match`: `{all_match}`",
+    "",
+    "| Step | Kind | TALs | Compare mode | VRP actual/ref | VRP match | VAP actual/ref | VAP match | Duration (ms) |",
+    "| --- | --- | ---: | --- | --- | --- | --- | --- | ---: |",
+]
+for item in results:
+    compare = item["compare"]
+    timing = item.get("timing") or {}
+    lines.append(
+        "| {step} | {kind} | {tal_count} | {compare_mode} | {va}/{vr} | {vm} | {aa}/{ar} | {am} | {dur} |".format(
+            step=item["stepId"],
+            kind=item["kind"],
+            tal_count=item.get("talCount") if item.get("talCount") is not None else "-",
+            compare_mode=item.get("compareMode") or "-",
+            va=compare["vrps"]["actual"],
+            vr=compare["vrps"]["reference"],
+            vm=compare["vrps"]["match"],
+            aa=compare["vaps"]["actual"],
+            ar=compare["vaps"]["reference"],
+            am=compare["vaps"]["match"],
+            dur=timing.get("durationMs", "-"),
+        )
+    )
+summary_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
+PY
+
+echo "done: $SEQUENCE_ROOT"

scripts/cir/run_cir_replay_sequence_routinator.sh

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_replay_sequence_routinator.sh \
+    --sequence-root <path> \
+    [--routinator-root <path>] \
+    [--routinator-bin <path>] \
+    [--real-rsync-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SEQUENCE_ROOT=""
+ROUTINATOR_ROOT="${ROUTINATOR_ROOT:-/home/yuyr/dev/rust_playground/routinator}"
+ROUTINATOR_BIN="${ROUTINATOR_BIN:-$ROUTINATOR_ROOT/target/debug/routinator}"
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+STEP_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_routinator.sh"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --sequence-root) SEQUENCE_ROOT="$2"; shift 2 ;;
+    --routinator-root) ROUTINATOR_ROOT="$2"; shift 2 ;;
+    --routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$SEQUENCE_ROOT" ]] || { usage >&2; exit 2; }
+
+SEQUENCE_ROOT="$(python3 - <<'PY' "$SEQUENCE_ROOT"
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+
+SUMMARY_JSON="$SEQUENCE_ROOT/sequence-summary-routinator.json"
+SUMMARY_MD="$SEQUENCE_ROOT/sequence-summary-routinator.md"
+
+python3 - <<'PY' "$SEQUENCE_ROOT" "$SUMMARY_JSON" "$SUMMARY_MD" "$STEP_SCRIPT" "$ROUTINATOR_ROOT" "$ROUTINATOR_BIN" "$REAL_RSYNC_BIN"
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+sequence_root = Path(sys.argv[1])
+summary_json = Path(sys.argv[2])
+summary_md = Path(sys.argv[3])
+step_script = Path(sys.argv[4])
+routinator_root = sys.argv[5]
+routinator_bin = sys.argv[6]
+real_rsync_bin = sys.argv[7]
+
+sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
+repo_bytes_db = sequence_root / sequence["repoBytesDbPath"]
+steps = sequence["steps"]
+results = []
+all_match = True
+
+for step in steps:
+    step_id = step["stepId"]
+    out_dir = sequence_root / "replay-routinator" / step_id
+    out_dir.parent.mkdir(parents=True, exist_ok=True)
+    cmd = [
+        str(step_script),
+        "--cir",
+        str(sequence_root / step["cirPath"]),
+        "--out-dir",
+        str(out_dir),
+        "--reference-ccr",
+        str(sequence_root / step["ccrPath"]),
+        "--routinator-root",
+        routinator_root,
+        "--routinator-bin",
+        routinator_bin,
+        "--real-rsync-bin",
+        real_rsync_bin,
+    ]
+    cmd.extend(["--repo-bytes-db", str(repo_bytes_db)])
+    proc = subprocess.run(cmd, capture_output=True, text=True)
+    if proc.returncode != 0:
+        raise SystemExit(
+            f"routinator sequence replay failed for {step_id}: stdout={proc.stdout} stderr={proc.stderr}"
+        )
+    compare = json.loads((out_dir / "compare-summary.json").read_text(encoding="utf-8"))
+    match = bool(compare["vrps"]["match"]) and bool(compare["vaps"]["match"])
+    all_match = all_match and match
+    results.append(
+        {
+            "stepId": step_id,
+            "kind": step["kind"],
+            "validationTime": step["validationTime"],
+            "outDir": str(out_dir),
+            "comparePath": str(out_dir / "compare-summary.json"),
+            "compareMode": compare.get("compareMode"),
+            "talCount": compare.get("talCount"),
+            "talPaths": compare.get("talPaths", []),
+            "match": match,
+            "compare": compare,
+        }
+    )
+
+summary = {
+    "version": 1,
+    "participant": "routinator",
+    "sequenceRoot": str(sequence_root),
+    "stepCount": len(results),
+    "allMatch": all_match,
+    "steps": results,
+}
+summary_json.write_text(json.dumps(summary, indent=2), encoding="utf-8")
+lines = [
+    "# Routinator CIR Sequence Replay Summary",
+    "",
+    f"- `sequence_root`: `{sequence_root}`",
+    f"- `step_count`: `{len(results)}`",
+    f"- `all_match`: `{all_match}`",
+    "",
+    "| Step | Kind | TALs | Compare mode | VRP actual/ref | VRP match | VAP actual/ref | VAP match |",
+    "| --- | --- | ---: | --- | --- | --- | --- | --- |",
+]
+for item in results:
+    compare = item["compare"]
+    lines.append(
+        "| {step} | {kind} | {tal_count} | {compare_mode} | {va}/{vr} | {vm} | {aa}/{ar} | {am} |".format(
+            step=item["stepId"],
+            kind=item["kind"],
+            tal_count=item.get("talCount") if item.get("talCount") is not None else "-",
+            compare_mode=item.get("compareMode") or "-",
+            va=compare["vrps"]["actual"],
+            vr=compare["vrps"]["reference"],
+            vm=compare["vrps"]["match"],
+            aa=compare["vaps"]["actual"],
+            ar=compare["vaps"]["reference"],
+            am=compare["vaps"]["match"],
+        )
+    )
+summary_md.write_text("\n".join(lines), encoding="utf-8")
+PY
+
+echo "done: $SEQUENCE_ROOT"

scripts/cir/run_cir_replay_sequence_rpki_client.sh

@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_replay_sequence_rpki_client.sh \
+    --sequence-root <path> \
+    [--build-dir <path> | --rpki-client-bin <path>] \
+    [--real-rsync-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SEQUENCE_ROOT=""
+BUILD_DIR=""
+RPKI_CLIENT_BIN=""
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+STEP_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_rpki_client.sh"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --sequence-root) SEQUENCE_ROOT="$2"; shift 2 ;;
+    --build-dir) BUILD_DIR="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$SEQUENCE_ROOT" && ( -n "$BUILD_DIR" || -n "$RPKI_CLIENT_BIN" ) ]] || { usage >&2; exit 2; }
+
+SEQUENCE_ROOT="$(python3 - <<'PY' "$SEQUENCE_ROOT"
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+
+SUMMARY_JSON="$SEQUENCE_ROOT/sequence-summary-rpki-client.json"
+SUMMARY_MD="$SEQUENCE_ROOT/sequence-summary-rpki-client.md"
+
+python3 - <<'PY' "$SEQUENCE_ROOT" "$SUMMARY_JSON" "$SUMMARY_MD" "$STEP_SCRIPT" "$BUILD_DIR" "$RPKI_CLIENT_BIN" "$REAL_RSYNC_BIN"
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+sequence_root = Path(sys.argv[1])
+summary_json = Path(sys.argv[2])
+summary_md = Path(sys.argv[3])
+step_script = Path(sys.argv[4])
+build_dir = sys.argv[5]
+rpki_client_bin = sys.argv[6]
+real_rsync_bin = sys.argv[7]
+
+sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
+repo_bytes_db = sequence_root / sequence["repoBytesDbPath"]
+steps = sequence["steps"]
+results = []
+all_match = True
+
+for step in steps:
+    step_id = step["stepId"]
+    out_dir = sequence_root / "replay-rpki-client" / step_id
+    out_dir.parent.mkdir(parents=True, exist_ok=True)
+    cmd = [
+        str(step_script),
+        "--cir",
+        str(sequence_root / step["cirPath"]),
+        "--out-dir",
+        str(out_dir),
+        "--reference-ccr",
+        str(sequence_root / step["ccrPath"]),
+        "--real-rsync-bin",
+        real_rsync_bin,
+    ]
+    if rpki_client_bin:
+        cmd.extend(["--rpki-client-bin", rpki_client_bin])
+    else:
+        cmd.extend(["--build-dir", build_dir])
+    cmd.extend(["--repo-bytes-db", str(repo_bytes_db)])
+    proc = subprocess.run(cmd, capture_output=True, text=True)
+    if proc.returncode != 0:
+        raise SystemExit(
+            f"rpki-client sequence replay failed for {step_id}: stdout={proc.stdout} stderr={proc.stderr}"
+        )
+    compare = json.loads((out_dir / "compare-summary.json").read_text(encoding="utf-8"))
+    match = bool(compare["vrps"]["match"]) and bool(compare["vaps"]["match"])
+    all_match = all_match and match
+    results.append(
+        {
+            "stepId": step_id,
+            "kind": step["kind"],
+            "validationTime": step["validationTime"],
+            "outDir": str(out_dir),
+            "comparePath": str(out_dir / "compare-summary.json"),
+            "compareMode": compare.get("compareMode"),
+            "talCount": compare.get("talCount"),
+            "talPaths": compare.get("talPaths", []),
+            "match": match,
+            "compare": compare,
+        }
+    )
+
+summary = {
+    "version": 1,
+    "participant": "rpki-client",
+    "sequenceRoot": str(sequence_root),
+    "stepCount": len(results),
+    "allMatch": all_match,
+    "steps": results,
+}
+summary_json.write_text(json.dumps(summary, indent=2), encoding="utf-8")
+lines = [
+    "# rpki-client CIR Sequence Replay Summary",
+    "",
+    f"- `sequence_root`: `{sequence_root}`",
+    f"- `step_count`: `{len(results)}`",
+    f"- `all_match`: `{all_match}`",
+    "",
+    "| Step | Kind | TALs | Compare mode | VRP actual/ref | VRP match | VAP actual/ref | VAP match |",
+    "| --- | --- | ---: | --- | --- | --- | --- | --- |",
+]
+for item in results:
+    compare = item["compare"]
+    lines.append(
+        "| {step} | {kind} | {tal_count} | {compare_mode} | {va}/{vr} | {vm} | {aa}/{ar} | {am} |".format(
+            step=item["stepId"],
+            kind=item["kind"],
+            tal_count=item.get("talCount") if item.get("talCount") is not None else "-",
+            compare_mode=item.get("compareMode") or "-",
+            va=compare["vrps"]["actual"],
+            vr=compare["vrps"]["reference"],
+            vm=compare["vrps"]["match"],
+            aa=compare["vaps"]["actual"],
+            ar=compare["vaps"]["reference"],
+            am=compare["vaps"]["match"],
+        )
+    )
+summary_md.write_text("\n".join(lines), encoding="utf-8")
+PY
+
+echo "done: $SEQUENCE_ROOT"

scripts/cir/run_cir_sequence_matrix_multi_rir.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_sequence_matrix_multi_rir.sh \
+    --root <path> \
+    [--rir <afrinic,apnic,arin,lacnic,ripe>] \
+    [--rpki-bin <path>] \
+    [--routinator-root <path>] \
+    [--routinator-bin <path>] \
+    [--rpki-client-build-dir <path>] \
+    [--drop-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+ROOT=""
+RIRS="afrinic,apnic,arin,lacnic,ripe"
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+ROUTINATOR_ROOT="${ROUTINATOR_ROOT:-/home/yuyr/dev/rust_playground/routinator}"
+ROUTINATOR_BIN="${ROUTINATOR_BIN:-$ROUTINATOR_ROOT/target/debug/routinator}"
+RPKI_CLIENT_BUILD_DIR="${RPKI_CLIENT_BUILD_DIR:-/home/yuyr/dev/rpki-client-9.7/build-m5}"
+DROP_BIN="${DROP_BIN:-$ROOT_DIR/target/release/cir_drop_report}"
+
+OURS_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_sequence_ours.sh"
+ROUTINATOR_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_sequence_routinator.sh"
+RPKIC_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_sequence_rpki_client.sh"
+DROP_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_drop_sequence.sh"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --root) ROOT="$2"; shift 2 ;;
+    --rir) RIRS="$2"; shift 2 ;;
+    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
+    --routinator-root) ROUTINATOR_ROOT="$2"; shift 2 ;;
+    --routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
+    --rpki-client-build-dir) RPKI_CLIENT_BUILD_DIR="$2"; shift 2 ;;
+    --drop-bin) DROP_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$ROOT" ]] || { usage >&2; exit 2; }
+
+SUMMARY_JSON="$ROOT/final-summary.json"
+SUMMARY_MD="$ROOT/final-summary.md"
+IFS=',' read -r -a ITEMS <<< "$RIRS"
+
+results=()
+for rir in "${ITEMS[@]}"; do
+  seq_root="$ROOT/$rir"
+  "$OURS_SCRIPT" --sequence-root "$seq_root" --rpki-bin "$RPKI_BIN"
+  "$ROUTINATOR_SCRIPT" --sequence-root "$seq_root" --routinator-root "$ROUTINATOR_ROOT" --routinator-bin "$ROUTINATOR_BIN"
+  "$RPKIC_SCRIPT" --sequence-root "$seq_root" --build-dir "$RPKI_CLIENT_BUILD_DIR"
+  "$DROP_SCRIPT" --sequence-root "$seq_root" --drop-bin "$DROP_BIN"
+done
+
+python3 - <<'PY' "$ROOT" "$RIRS" "$SUMMARY_JSON" "$SUMMARY_MD"
+import json, sys
+from pathlib import Path
+from collections import Counter
+
+root = Path(sys.argv[1]).resolve()
+rirs = [item for item in sys.argv[2].split(',') if item]
+summary_json = Path(sys.argv[3])
+summary_md = Path(sys.argv[4])
+items = []
+total_steps = 0
+total_dropped_vrps = 0
+total_dropped_objects = 0
+reason_counter = Counter()
+for rir in rirs:
+    seq_root = root / rir
+    ours = json.loads((seq_root / "sequence-summary.json").read_text(encoding="utf-8"))
+    routinator = json.loads((seq_root / "sequence-summary-routinator.json").read_text(encoding="utf-8"))
+    rpki_client = json.loads((seq_root / "sequence-summary-rpki-client.json").read_text(encoding="utf-8"))
+    drop = json.loads((seq_root / "drop-summary.json").read_text(encoding="utf-8"))
+    step_count = len(ours["steps"])
+    total_steps += step_count
+    rir_dropped_vrps = 0
+    rir_dropped_objects = 0
+    for step in drop["steps"]:
+        drop_path = Path(step["reportPath"])
+        detail = json.loads(drop_path.read_text(encoding="utf-8"))
+        summary = detail.get("summary", {})
+        rir_dropped_vrps += int(summary.get("droppedVrpCount", 0))
+        rir_dropped_objects += int(summary.get("droppedObjectCount", 0))
+        total_dropped_vrps += int(summary.get("droppedVrpCount", 0))
+        total_dropped_objects += int(summary.get("droppedObjectCount", 0))
+        for reason, count in summary.get("droppedByReason", {}).items():
+            reason_counter[reason] += int(count)
+    items.append({
+        "rir": rir,
+        "stepCount": step_count,
+        "oursAllMatch": ours["allMatch"],
+        "routinatorAllMatch": routinator["allMatch"],
+        "rpkiClientAllMatch": rpki_client["allMatch"],
+        "dropSummary": drop["steps"],
+        "droppedVrpCount": rir_dropped_vrps,
+        "droppedObjectCount": rir_dropped_objects,
+    })
+summary = {
+    "version": 1,
+    "totalStepCount": total_steps,
+    "totalDroppedVrpCount": total_dropped_vrps,
+    "totalDroppedObjectCount": total_dropped_objects,
+    "topReasons": [{"reason": reason, "count": count} for reason, count in reason_counter.most_common(10)],
+    "rirs": items,
+}
+summary_json.write_text(json.dumps(summary, indent=2), encoding="utf-8")
+lines = ["# Multi-RIR CIR Sequence Matrix Summary", ""]
+lines.append(f"- `total_step_count`: `{total_steps}`")
+lines.append(f"- `total_dropped_vrps`: `{total_dropped_vrps}`")
+lines.append(f"- `total_dropped_objects`: `{total_dropped_objects}`")
+lines.append("")
+if reason_counter:
+    lines.append("## Top Drop Reasons")
+    lines.append("")
+    for reason, count in reason_counter.most_common(10):
+        lines.append(f"- `{reason}`: `{count}`")
+    lines.append("")
+for item in items:
+    lines.append(
+        f"- `{item['rir']}`: `steps={item['stepCount']}` `ours={item['oursAllMatch']}` `routinator={item['routinatorAllMatch']}` `rpki-client={item['rpkiClientAllMatch']}` `drop_vrps={item['droppedVrpCount']}` `drop_objects={item['droppedObjectCount']}`"
+    )
+summary_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
+PY
+
+echo "done: $ROOT"

scripts/compare/run_perf_compare_quick_remote.sh

@@ -0,0 +1,502 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/compare/run_perf_compare_quick_remote.sh \
+    --run-root <path> \
+    --remote-root <path> \
+    [--rir-set <mixed2|all5>] \
+    [--ssh-target <user@host>] \
+    [--rpki-client-bin <path>] \
+    [--libtls-path <path>] \
+    [--rp-run-mode <serial|parallel>] \
+    [--copy-rpki-client-cache] \
+    [--probe-rpki-client-cache] \
+    [--ours-extra-args '<args>'] \
+    [--dry-run]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+first_existing_executable() {
+  local fallback="$1"
+  shift
+  local candidate
+  for candidate in "$@"; do
+    if [[ -x "$candidate" ]]; then
+      printf '%s' "$candidate"
+      return
+    fi
+  done
+  printf '%s' "$fallback"
+}
+
+first_existing_file() {
+  local fallback="$1"
+  shift
+  local candidate
+  for candidate in "$@"; do
+    if [[ -f "$candidate" ]]; then
+      printf '%s' "$candidate"
+      return
+    fi
+  done
+  printf '%s' "$fallback"
+}
+
+RUN_ROOT=""
+REMOTE_ROOT=""
+SSH_TARGET="${SSH_TARGET:-root@47.251.56.108}"
+RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-$(first_existing_executable \
+  "/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client" \
+  "$ROOT_DIR/../../.cache/rpki-client-9.7-build/bin/rpki-client" \
+  "$ROOT_DIR/../../.cache/rpki-client-9.7-build/src/rpki-client-9.7/src/rpki-client" \
+  "$ROOT_DIR/../../.cache/rpki-client-remote9.0/rpki-client" \
+  "/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client")}"
+LIBTLS_PATH="${LIBTLS_PATH:-$(first_existing_file \
+  "/home/yuyr/dev/rpki-client-9.7/.deps/libtls/root/usr/lib/x86_64-linux-gnu/libtls.so.28.0.0" \
+  "$ROOT_DIR/../../.cache/rpki-client-9.7-build/runlib/libtls.so.28" \
+  "$ROOT_DIR/../../.cache/rpki-client-9.7-build/sysroot/usr/lib/x86_64-linux-gnu/libtls.so.28.0.0" \
+  "$ROOT_DIR/../../.cache/rpki-client-remote9.0/libtls.so.28" \
+  "/home/yuyr/dev/rpki-client-9.7/.deps/libtls/root/usr/lib/x86_64-linux-gnu/libtls.so.28.0.0")}"
+RP_RUN_MODE="${RP_RUN_MODE:-serial}"
+RIR_SET="${RIR_SET:-mixed2}"
+OURS_EXTRA_ARGS="${OURS_EXTRA_ARGS:-}"
+COPY_RPKI_CLIENT_CACHE="${COPY_RPKI_CLIENT_CACHE:-0}"
+PROBE_RPKI_CLIENT_CACHE="${PROBE_RPKI_CLIENT_CACHE:-0}"
+DRY_RUN=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --run-root) RUN_ROOT="$2"; shift 2 ;;
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --rir-set) RIR_SET="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --libtls-path) LIBTLS_PATH="$2"; shift 2 ;;
+    --rp-run-mode) RP_RUN_MODE="$2"; shift 2 ;;
+    --copy-rpki-client-cache) COPY_RPKI_CLIENT_CACHE=1; shift ;;
+    --probe-rpki-client-cache) PROBE_RPKI_CLIENT_CACHE=1; shift ;;
+    --ours-extra-args) OURS_EXTRA_ARGS="$2"; shift 2 ;;
+    --dry-run) DRY_RUN=1; shift ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$RUN_ROOT" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
+[[ "$RP_RUN_MODE" == "serial" || "$RP_RUN_MODE" == "parallel" ]] || { echo "invalid --rp-run-mode: $RP_RUN_MODE" >&2; usage; exit 2; }
+[[ "$RIR_SET" == "mixed2" || "$RIR_SET" == "all5" ]] || { echo "invalid --rir-set: $RIR_SET" >&2; usage; exit 2; }
+[[ "$DRY_RUN" -eq 1 || -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
+[[ "$DRY_RUN" -eq 1 || -f "$LIBTLS_PATH" ]] || { echo "libtls not found: $LIBTLS_PATH" >&2; exit 2; }
+
+RUN_ROOT="$(python3 - <<'PY' "$RUN_ROOT"
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+
+mkdir -p "$RUN_ROOT/steps/step-001/ours" "$RUN_ROOT/steps/step-001/rpki-client" "$RUN_ROOT/steps/step-001/compare"
+mkdir -p "$RUN_ROOT/steps/step-002/ours" "$RUN_ROOT/steps/step-002/rpki-client" "$RUN_ROOT/steps/step-002/compare"
+
+tal_path_for_rir() {
+  case "$1" in
+    afrinic) printf '%s' "$ROOT_DIR/tests/fixtures/tal/afrinic.tal" ;;
+    apnic) printf '%s' "$ROOT_DIR/tests/fixtures/tal/apnic-rfc7730-https.tal" ;;
+    arin) printf '%s' "$ROOT_DIR/tests/fixtures/tal/arin.tal" ;;
+    lacnic) printf '%s' "$ROOT_DIR/tests/fixtures/tal/lacnic.tal" ;;
+    ripe) printf '%s' "$ROOT_DIR/tests/fixtures/tal/ripe-ncc.tal" ;;
+    *) echo "unknown rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+ta_path_for_rir() {
+  case "$1" in
+    afrinic) printf '%s' "$ROOT_DIR/tests/fixtures/ta/afrinic-ta.cer" ;;
+    apnic) printf '%s' "$ROOT_DIR/tests/fixtures/ta/apnic-ta.cer" ;;
+    arin) printf '%s' "$ROOT_DIR/tests/fixtures/ta/arin-ta.cer" ;;
+    lacnic) printf '%s' "$ROOT_DIR/tests/fixtures/ta/lacnic-ta.cer" ;;
+    ripe) printf '%s' "$ROOT_DIR/tests/fixtures/ta/ripe-ncc-ta.cer" ;;
+    *) echo "unknown rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+case "$RIR_SET" in
+  mixed2)
+    RIRS=(apnic arin)
+    SCOPE_LABEL="APNIC+ARIN mixed release two-step synchronized compare"
+    ;;
+  all5)
+    RIRS=(afrinic apnic arin lacnic ripe)
+    SCOPE_LABEL="all-five-RIR mixed release two-step synchronized compare"
+    ;;
+esac
+
+COPY_FILES=()
+for rir in "${RIRS[@]}"; do
+  COPY_FILES+=("$(tal_path_for_rir "$rir")" "$(ta_path_for_rir "$rir")")
+done
+
+if [[ "$DRY_RUN" -eq 1 ]]; then
+  cat <<EOF2
+workflow_name=性能对比测试快速版
+scope=$SCOPE_LABEL
+rir_set=$RIR_SET
+rirs=${RIRS[*]}
+run_root=$RUN_ROOT
+remote_root=$REMOTE_ROOT
+ssh_target=$SSH_TARGET
+rp_run_mode=$RP_RUN_MODE
+ours_extra_args=$OURS_EXTRA_ARGS
+EOF2
+  exit 0
+fi
+
+cleanup_remote() {
+  if [[ "${KEEP_REMOTE:-0}" != "1" ]]; then
+    ssh "$SSH_TARGET" "rm -rf '$REMOTE_ROOT'" >/dev/null 2>&1 || true
+  fi
+}
+trap cleanup_remote EXIT
+
+(
+  cd "$ROOT_DIR"
+  cargo build --release --bin rpki --bin ccr_to_compare_views --bin ccr_state_compare --bin cir_state_compare --bin cir_probe_rpki_client_cache
+)
+
+ssh "$SSH_TARGET" "set -e; systemctl disable --now rpki-client.timer >/dev/null 2>&1 || true; systemctl stop rpki-client.service >/dev/null 2>&1 || true; pkill -f '[/]rpki-client([[:space:]]|$)' >/dev/null 2>&1 || true; pkill -f '[/]routinator([[:space:]]|$)' >/dev/null 2>&1 || true; id -u _rpki-client >/dev/null 2>&1 || useradd -r -M -s /usr/sbin/nologin _rpki-client || true; rm -rf '$REMOTE_ROOT'; mkdir -p '$REMOTE_ROOT/bin' '$REMOTE_ROOT/lib' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client' '$REMOTE_ROOT/steps/step-001/ours' '$REMOTE_ROOT/steps/step-001/rpki-client' '$REMOTE_ROOT/steps/step-002/ours' '$REMOTE_ROOT/steps/step-002/rpki-client'"
+scp "$ROOT_DIR/target/release/rpki" "${COPY_FILES[@]}" "$SSH_TARGET:$REMOTE_ROOT/"
+if [[ "$PROBE_RPKI_CLIENT_CACHE" == "1" ]]; then
+  scp "$ROOT_DIR/target/release/cir_probe_rpki_client_cache" "$SSH_TARGET:$REMOTE_ROOT/bin/"
+fi
+scp "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_ROOT/bin/rpki-client"
+scp "$LIBTLS_PATH" "$SSH_TARGET:$REMOTE_ROOT/lib/libtls.so.28"
+printf '%s' "$OURS_EXTRA_ARGS" | ssh "$SSH_TARGET" "cat > '$REMOTE_ROOT/ours-extra-args.txt'"
+printf '%s' "$RP_RUN_MODE" | ssh "$SSH_TARGET" "cat > '$REMOTE_ROOT/rp-run-mode.txt'"
+printf '%s' "$RIR_SET" | ssh "$SSH_TARGET" "cat > '$REMOTE_ROOT/rir-set.txt'"
+
+run_step() {
+  local step_id="$1"
+  local kind="$2"
+  local local_step="$RUN_ROOT/steps/$step_id"
+
+  ssh "$SSH_TARGET" bash -s -- "$REMOTE_ROOT" "$step_id" "$kind" <<'EOS'
+set -euo pipefail
+REMOTE_ROOT="$1"
+STEP_ID="$2"
+KIND="$3"
+
+cd "$REMOTE_ROOT"
+mkdir -p "steps/$STEP_ID/ours" "steps/$STEP_ID/rpki-client"
+touch rpki-client-skiplist
+chmod 0644 rpki-client-skiplist
+OURS_EXTRA_ARGS="$(cat ours-extra-args.txt)"
+RP_RUN_MODE="$(cat rp-run-mode.txt)"
+RIR_SET="$(cat rir-set.txt)"
+OURS_EXTRA_ARGV=()
+if [[ -n "$OURS_EXTRA_ARGS" ]]; then
+  # shellcheck disable=SC2206
+  OURS_EXTRA_ARGV=($OURS_EXTRA_ARGS)
+fi
+
+case "$RIR_SET" in
+  mixed2) RIRS=(apnic arin) ;;
+  all5) RIRS=(afrinic apnic arin lacnic ripe) ;;
+  *) echo "invalid rir set: $RIR_SET" >&2; exit 2 ;;
+esac
+
+tal_file_for_rir() {
+  case "$1" in
+    afrinic) printf '%s' "afrinic.tal" ;;
+    apnic) printf '%s' "apnic-rfc7730-https.tal" ;;
+    arin) printf '%s' "arin.tal" ;;
+    lacnic) printf '%s' "lacnic.tal" ;;
+    ripe) printf '%s' "ripe-ncc.tal" ;;
+    *) echo "unknown rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+tal_uri_for_rir() {
+  case "$1" in
+    afrinic) printf '%s' "https://rpki.afrinic.net/repository/AfriNIC.cer" ;;
+    apnic) printf '%s' "https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer" ;;
+    arin) printf '%s' "https://rrdp.arin.net/arin-rpki-ta.cer" ;;
+    lacnic) printf '%s' "https://rrdp.lacnic.net/ta/rta-lacnic-rpki.cer" ;;
+    ripe) printf '%s' "https://rpki.ripe.net/ta/ripe-ncc-ta.cer" ;;
+    *) echo "unknown rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+ta_file_for_rir() {
+  case "$1" in
+    afrinic) printf '%s' "afrinic-ta.cer" ;;
+    apnic) printf '%s' "apnic-ta.cer" ;;
+    arin) printf '%s' "arin-ta.cer" ;;
+    lacnic) printf '%s' "lacnic-ta.cer" ;;
+    ripe) printf '%s' "ripe-ncc-ta.cer" ;;
+    *) echo "unknown rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+refresh_ta_file_for_rir() {
+  local rir="$1"
+  local uri
+  local file
+  uri="$(tal_uri_for_rir "$rir")"
+  file="$(ta_file_for_rir "$rir")"
+  python3 - <<'PY' "$uri" "$file"
+import sys
+import urllib.request
+uri, path = sys.argv[1:]
+request = urllib.request.Request(uri, headers={"User-Agent": "rpki-dev/compare-fast-path"})
+with urllib.request.urlopen(request, timeout=30) as response:
+    data = response.read()
+if not data:
+    raise SystemExit(f"empty TA certificate response: {uri}")
+with open(path, "wb") as output:
+    output.write(data)
+PY
+}
+
+for rir in "${RIRS[@]}"; do
+  refresh_ta_file_for_rir "$rir"
+done
+
+OURS_TAL_ARGS=()
+CLIENT_TAL_ARGS=()
+OURS_CIR_TAL_ARGS=()
+for rir in "${RIRS[@]}"; do
+  tal_file="$(tal_file_for_rir "$rir")"
+  ta_file="$(ta_file_for_rir "$rir")"
+  tal_uri="$(tal_uri_for_rir "$rir")"
+  OURS_TAL_ARGS+=(--tal-path "$tal_file" --ta-path "$ta_file")
+  OURS_CIR_TAL_ARGS+=(--cir-tal-uri "$tal_uri")
+  CLIENT_TAL_ARGS+=(-t "../../$tal_file")
+done
+
+if [[ "$KIND" == "snapshot" ]]; then
+  rm -rf state/ours/work-db state/ours/raw-store.db state/ours/repo-bytes.db state/rpki-client/cache state/rpki-client/out state/rpki-client/ta state/rpki-client/.ta
+fi
+mkdir -p state/ours/work-db state/ours/raw-store.db state/ours/repo-bytes.db state/rpki-client/cache state/rpki-client/out state/rpki-client/ta state/rpki-client/.ta
+chmod 0777 state/ours/work-db state/ours/raw-store.db state/ours/repo-bytes.db
+chmod -R 0777 state/rpki-client
+touch state/rpki-client/rpki-client-skiplist
+chmod 0644 state/rpki-client/rpki-client-skiplist
+
+START_EPOCH="$(python3 - <<'PY'
+import time
+print(time.time() + 3.0)
+PY
+)"
+
+run_ours() {
+  python3 - <<'PY' "$START_EPOCH"
+import sys, time
+x = float(sys.argv[1])
+d = x - time.time()
+if d > 0:
+    time.sleep(d)
+PY
+  started_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  set +e
+  env RPKI_PROGRESS_LOG=1 RPKI_PROGRESS_SLOW_SECS=0 ./rpki \
+    --db state/ours/work-db \
+    --raw-store-db state/ours/raw-store.db \
+    --repo-bytes-db state/ours/repo-bytes.db \
+    "${OURS_TAL_ARGS[@]}" \
+    "${OURS_EXTRA_ARGV[@]}" \
+    --ccr-out "steps/$STEP_ID/ours/result.ccr" \
+    --cir-enable \
+    --cir-out "steps/$STEP_ID/ours/result.cir" \
+    "${OURS_CIR_TAL_ARGS[@]}" \
+    --report-json "steps/$STEP_ID/ours/report.json" \
+    > "steps/$STEP_ID/ours/run.log" 2>&1
+  exit_code=$?
+  set -e
+  finished_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  python3 - <<'PY' "steps/$STEP_ID/ours/round-result.json" "$STEP_ID" "$KIND" "$started_ms" "$finished_ms" "$exit_code"
+import json, sys
+path, step_id, kind, started_ms, finished_ms, exit_code = sys.argv[1:]
+json.dump(
+    {
+        "stepId": step_id,
+        "kind": kind,
+        "durationMs": int(finished_ms) - int(started_ms),
+        "exitCode": int(exit_code),
+    },
+    open(path, "w"),
+    indent=2,
+)
+PY
+}
+
+run_client() {
+  cd state/rpki-client
+  python3 - <<'PY' "$START_EPOCH"
+import sys, time
+x = float(sys.argv[1])
+d = x - time.time()
+if d > 0:
+    time.sleep(d)
+PY
+  started_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  set +e
+  LD_LIBRARY_PATH="$REMOTE_ROOT/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" "$REMOTE_ROOT/bin/rpki-client" \
+    -vv \
+    -S rpki-client-skiplist \
+    "${CLIENT_TAL_ARGS[@]}" \
+    -d cache out \
+    > "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/run.log" 2>&1
+  exit_code=$?
+  set -e
+  cp out/rpki.ccr "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/result.ccr" 2>/dev/null || true
+  cp out/rpki.cir "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/result.cir" 2>/dev/null || true
+  cp out/openbgpd "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/openbgpd" 2>/dev/null || true
+  finished_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  python3 - <<'PY' "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/round-result.json" "$STEP_ID" "$KIND" "$started_ms" "$finished_ms" "$exit_code"
+import json, sys
+path, step_id, kind, started_ms, finished_ms, exit_code = sys.argv[1:]
+json.dump(
+    {
+        "stepId": step_id,
+        "kind": kind,
+        "durationMs": int(finished_ms) - int(started_ms),
+        "exitCode": int(exit_code),
+    },
+    open(path, "w"),
+    indent=2,
+)
+PY
+}
+
+if [[ "$RP_RUN_MODE" == "parallel" ]]; then
+  run_ours &
+  OURS_PID=$!
+  run_client &
+  CLIENT_PID=$!
+  wait "$OURS_PID"
+  wait "$CLIENT_PID"
+else
+  run_ours
+  run_client
+fi
+EOS
+
+  for rel in result.ccr result.cir round-result.json run.log stage-timing.json; do
+    scp -C "$SSH_TARGET:$REMOTE_ROOT/steps/$step_id/ours/$rel" "$local_step/ours/"
+  done
+  for rel in result.ccr result.cir round-result.json run.log openbgpd; do
+    scp -C "$SSH_TARGET:$REMOTE_ROOT/steps/$step_id/rpki-client/$rel" "$local_step/rpki-client/" || true
+  done
+  if [[ "$COPY_RPKI_CLIENT_CACHE" == "1" ]]; then
+    mkdir -p "$local_step/rpki-client/cache"
+    rsync -a --delete "$SSH_TARGET:$REMOTE_ROOT/state/rpki-client/cache/" "$local_step/rpki-client/cache/"
+  fi
+
+  if [[ -f "$local_step/ours/result.cir" && -f "$local_step/rpki-client/result.cir" ]]; then
+    "$ROOT_DIR/scripts/periodic/compare_ccr_cir_round.sh" \
+      --ours-ccr "$local_step/ours/result.ccr" \
+      --rpki-client-ccr "$local_step/rpki-client/result.ccr" \
+      --ours-cir "$local_step/ours/result.cir" \
+      --rpki-client-cir "$local_step/rpki-client/result.cir" \
+      --out-dir "$local_step/compare" \
+      --trust-anchor unknown >/dev/null
+    if [[ "$PROBE_RPKI_CLIENT_CACHE" == "1" ]]; then
+      ssh "$SSH_TARGET" "set -e; mkdir -p '$REMOTE_ROOT/steps/$step_id/compare/cir'; '$REMOTE_ROOT/bin/cir_probe_rpki_client_cache' --ours-cir '$REMOTE_ROOT/steps/$step_id/ours/result.cir' --rpki-client-cir '$REMOTE_ROOT/steps/$step_id/rpki-client/result.cir' --cache-root '$REMOTE_ROOT/state/rpki-client/cache' --rpki-client-log '$REMOTE_ROOT/steps/$step_id/rpki-client/run.log' --out-json '$REMOTE_ROOT/steps/$step_id/compare/cir/rpki-client-cache-probe.json' --sample-limit 50 >/dev/null"
+      scp -C "$SSH_TARGET:$REMOTE_ROOT/steps/$step_id/compare/cir/rpki-client-cache-probe.json" "$local_step/compare/cir/"
+    fi
+    if [[ "$COPY_RPKI_CLIENT_CACHE" == "1" ]]; then
+      "$ROOT_DIR/target/release/cir_probe_rpki_client_cache" \
+        --ours-cir "$local_step/ours/result.cir" \
+        --rpki-client-cir "$local_step/rpki-client/result.cir" \
+        --cache-root "$local_step/rpki-client/cache" \
+        --rpki-client-log "$local_step/rpki-client/run.log" \
+        --out-json "$local_step/compare/cir/rpki-client-cache-probe.json" \
+        --sample-limit 50 >/dev/null
+    fi
+  else
+    "$ROOT_DIR/scripts/periodic/compare_ccr_round.sh" \
+      --ours-ccr "$local_step/ours/result.ccr" \
+      --rpki-client-ccr "$local_step/rpki-client/result.ccr" \
+      --out-dir "$local_step/compare" \
+      --trust-anchor unknown >/dev/null
+  fi
+
+  python3 - <<'PY' "$local_step/ours/round-result.json" "$local_step/rpki-client/round-result.json" "$local_step/ours/stage-timing.json" "$local_step/compare/summary.json" "$local_step/compare/compare-summary.json" "$local_step/step-summary.json" "$OURS_EXTRA_ARGS"
+import json, sys
+ours = json.load(open(sys.argv[1]))
+client = json.load(open(sys.argv[2]))
+stage = json.load(open(sys.argv[3]))
+compare_path = sys.argv[4] if __import__('pathlib').Path(sys.argv[4]).exists() else sys.argv[5]
+compare = json.load(open(compare_path))
+ours_extra_args = sys.argv[7]
+json.dump(
+    {
+        "stepId": ours["stepId"],
+        "kind": ours["kind"],
+        "oursExtraArgs": ours_extra_args,
+        "oursDurationMs": ours["durationMs"],
+        "rpkiClientDurationMs": client["durationMs"],
+        "oursExitCode": ours["exitCode"],
+        "rpkiClientExitCode": client["exitCode"],
+        "oursTotalMs": stage["total_ms"],
+        "oursRepoSyncMsTotal": stage["repo_sync_ms_total"],
+        "oursPublicationPointRepoSyncMsTotal": stage.get("publication_point_repo_sync_ms_total"),
+        "oursDownloadEventCount": stage.get("download_event_count"),
+        "oursRrdpDownloadMsTotal": stage.get("rrdp_download_ms_total"),
+        "oursRsyncDownloadMsTotal": stage.get("rsync_download_ms_total"),
+        "oursDownloadBytesTotal": stage.get("download_bytes_total"),
+        "oursVrps": compare["vrps"]["ours"],
+        "rpkiClientVrps": compare["vrps"]["rpkiClient"],
+        "oursVaps": compare["vaps"]["ours"],
+        "rpkiClientVaps": compare["vaps"]["rpkiClient"],
+        "vrpMatch": compare["vrps"]["match"],
+        "vapMatch": compare["vaps"]["match"],
+        "allMatch": compare["allMatch"],
+        "onlyInOurs": len(compare["vrps"]["onlyInOurs"]),
+        "onlyInRpkiClient": len(compare["vrps"]["onlyInRpkiClient"]),
+    },
+    open(sys.argv[6], "w"),
+    indent=2,
+)
+PY
+}
+
+run_step step-001 snapshot
+run_step step-002 delta
+
+python3 - <<'PY' "$RUN_ROOT/steps/step-001/step-summary.json" "$RUN_ROOT/steps/step-002/step-summary.json" "$RUN_ROOT/summary.json" "$RP_RUN_MODE" "$OURS_EXTRA_ARGS" "$RIR_SET" "$SCOPE_LABEL" "${RIRS[@]}"
+import json, sys
+steps = [json.load(open(p)) for p in sys.argv[1:3]]
+summary = {
+    "workflowName": "性能对比测试快速版",
+    "scope": sys.argv[7],
+    "rpRunMode": sys.argv[4],
+    "oursExtraArgs": sys.argv[5],
+    "rirSet": sys.argv[6],
+    "rirs": sys.argv[8:],
+    "steps": steps,
+}
+json.dump(summary, open(sys.argv[3], "w"), indent=2, ensure_ascii=False)
+print(json.dumps(summary, indent=2, ensure_ascii=False))
+PY

scripts/coverage.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Requires:
+#   rustup component add llvm-tools-preview
+#   cargo install cargo-llvm-cov --locked
+
+# Optional:
+#   COVERAGE_FORCE_CLEAN=1  Force `cargo llvm-cov clean --workspace` before the run.
+#                           Default behavior is to reuse existing llvm-cov build artifacts.
+#   RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS=1  Skip replay/matrix integration tests that
+#                           spawn shell pipelines and can trigger separate release builds.
+#                           coverage.sh enables this by default.
+#   RPKI_SKIP_HEAVY_BLACKBOX_TESTS=1  Skip slower blackbox CLI/script integration tests
+#                           that provide low incremental coverage per wall-clock second.
+#                           coverage.sh enables this by default.
+#   RPKI_SKIP_HEAVY_CRYPTO_TESTS=1  Skip slower OpenSSL-heavy certificate generation tests
+#                           that provide low incremental coverage per wall-clock second.
+#                           coverage.sh enables this by default.
+
+run_out="$(mktemp)"
+text_out="$(mktemp)"
+html_out="$(mktemp)"
+
+cleanup() {
+  rm -f "$run_out" "$text_out" "$html_out"
+}
+trap cleanup EXIT
+
+IGNORE_REGEX='repository_view_stats\.rs|db_stats\.rs|rrdp_state_dump\.rs|ccr_dump\.rs|ccr_verify\.rs|ccr_to_routinator_csv\.rs|ccr_to_compare_views\.rs|cir_materialize\.rs|cir_extract_inputs\.rs|cir_drop_report\.rs|cir_ta_only_fixture\.rs|cir_dump_reject_list\.rs|rpki_object_parse\.rs|rpki_query_indexer\.rs|rpki_query_service\.rs|triage_ccr_cir_pair\.rs|rpki_artifact_metrics|rpki_inter_rp_metrics|rpki_daemon\.rs|sequence_triage_ccr_cir|ccr_state_compare\.rs|cir_state_compare\.rs|cir_probe_rpki_client_cache\.rs|ccr/compare_view\.rs|progress_log\.rs|cli\.rs|validation/run_tree_from_tal\.rs|validation/tree_parallel\.rs|validation/tree_runner|validation/from_tal\.rs|sync/store_projection\.rs|sync/repo\.rs|sync/rrdp|(^|/)storage(/|\.rs$)|cir/materialize\.rs'
+
+# Preserve colored output even though we post-process output by running under a pseudo-TTY.
+# We run tests only once, then generate both CLI text + HTML reports without rerunning tests.
+set +e
+
+if [ "${COVERAGE_FORCE_CLEAN:-0}" = "1" ]; then
+  cargo llvm-cov clean --workspace >/dev/null 2>&1
+  echo "coverage mode: clean build (COVERAGE_FORCE_CLEAN=1)"
+else
+  echo "coverage mode: reuse existing llvm-cov artifacts (default)"
+fi
+
+export RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS="${RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS:-1}"
+export RPKI_SKIP_HEAVY_BLACKBOX_TESTS="${RPKI_SKIP_HEAVY_BLACKBOX_TESTS:-1}"
+export RPKI_SKIP_HEAVY_CRYPTO_TESTS="${RPKI_SKIP_HEAVY_CRYPTO_TESTS:-1}"
+
+# 1) Run tests once to collect coverage data (no report).
+script -q -e -c "CARGO_TERM_COLOR=always cargo llvm-cov --no-report" "$run_out" >/dev/null 2>&1
+run_status="$?"
+
+# 2) CLI summary report + fail-under gate (no test rerun).
+script -q -e -c "CARGO_TERM_COLOR=always cargo llvm-cov report --fail-under-lines 90 --ignore-filename-regex '$IGNORE_REGEX'" "$text_out" >/dev/null 2>&1
+text_status="$?"
+
+# 3) HTML report (no test rerun).
+script -q -e -c "CARGO_TERM_COLOR=always cargo llvm-cov report --html --ignore-filename-regex '$IGNORE_REGEX'" "$html_out" >/dev/null 2>&1
+html_status="$?"
+
+set -e
+
+strip_script_noise() {
+  tr -d '\r' | sed '/^Script \(started\|done\) on /d'
+}
+
+strip_ansi_for_parse() {
+  awk '
+    {
+      line = $0
+      gsub(/\033\[[0-9;]*[A-Za-z]/, "", line) # CSI escapes
+      gsub(/\033\([A-Za-z]/, "", line)        # charset escapes (e.g., ESC(B)
+      gsub(/\r/, "", line)
+      print line
+    }
+  '
+}
+
+cat "$run_out" | strip_script_noise
+cat "$text_out" | strip_script_noise
+cat "$html_out" | strip_script_noise
+
+cat "$run_out" | strip_ansi_for_parse | awk '
+  BEGIN {
+    passed=0; failed=0; ignored=0; measured=0; filtered=0;
+  }
+  /^test result: / {
+    if (match($0, /([0-9]+) passed; ([0-9]+) failed; ([0-9]+) ignored; ([0-9]+) measured; ([0-9]+) filtered out;/, m)) {
+      passed += m[1]; failed += m[2]; ignored += m[3]; measured += m[4]; filtered += m[5];
+    }
+  }
+  END {
+    executed = passed + failed;
+    total = passed + failed + ignored + measured;
+    printf("\nTEST SUMMARY (all suites): passed=%d failed=%d ignored=%d measured=%d filtered_out=%d executed=%d total=%d\n",
+           passed, failed, ignored, measured, filtered, executed, total);
+  }
+'
+
+echo
+echo "HTML report: target/llvm-cov/html/index.html"
+
+status="$text_status"
+if [ "$run_status" -ne 0 ]; then status="$run_status"; fi
+if [ "$html_status" -ne 0 ]; then status="$html_status"; fi
+exit "$status"

scripts/docker/build_arm64_installer_package.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+exec "$SCRIPT_DIR/build_docker_installer_package.sh" --arch arm64 "$@"

scripts/docker/build_arm64_metrics_image.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+exec "$SCRIPT_DIR/build_docker_metrics_image.sh" --arch arm64 "$@"

scripts/docker/build_arm64_runtime_image.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+exec "$SCRIPT_DIR/build_docker_runtime_image.sh" --arch arm64 "$@"

scripts/docker/build_docker_installer_package.sh

@@ -0,0 +1,465 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+TARGET_ARCH="${TARGET_ARCH:-}"
+IMAGE_TAG=""
+IMAGE_TAR=""
+METRICS_IMAGE=""
+METRICS_IMAGE_TAR=""
+PROMETHEUS_IMAGE="${PROMETHEUS_IMAGE:-prom/prometheus:v2.55.1}"
+PROMETHEUS_IMAGE_TAR="${PROMETHEUS_IMAGE_TAR:-}"
+GRAFANA_IMAGE="${GRAFANA_IMAGE:-grafana/grafana:11.3.1}"
+GRAFANA_IMAGE_TAR="${GRAFANA_IMAGE_TAR:-}"
+OUT_DIR="${OUT_DIR:-}"
+PACKAGE_PREFIX="${PACKAGE_PREFIX:-}"
+TEMPLATE_DIR="${TEMPLATE_DIR:-$REPO_ROOT/deploy/docker-installer}"
+ALLOW_DIRTY=0
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/docker/build_docker_installer_package.sh --arch amd64|arm64 [options]
+
+Options:
+  --arch <arch>       Target architecture: amd64|arm64
+  --prometheus-image <tag>
+                      Prometheus image tag to record and package.
+  --prometheus-image-tar <path>
+                      Existing Prometheus docker save tar/tar.gz to include.
+  --grafana-image <tag>
+                      Grafana image tag to record and package.
+  --grafana-image-tar <path>
+                      Existing Grafana docker save tar/tar.gz to include.
+  --out-dir <path>    Output directory.
+  --prefix <name>     Package directory/tar prefix.
+  --template-dir <path>
+                      Package template directory.
+  --allow-dirty       Allow a development build from a dirty worktree and mark
+                      image tags, package name and manifest as dirty.
+  -h, --help          Show help.
+
+Every package rebuilds runtime and metrics images from the current HEAD. Their
+tags are ours-rp-runtime-<arch>:<git8> and ours-rp-metrics-<arch>:<git8>.
+External runtime/metrics image tags and tar archives are intentionally rejected.
+USAGE
+}
+
+normalize_arch() {
+  case "$1" in
+    amd64|x86_64)
+      printf 'amd64\n'
+      ;;
+    arm64|aarch64)
+      printf 'arm64\n'
+      ;;
+    *)
+      return 1
+      ;;
+  esac
+}
+
+platform_for_arch() {
+  printf 'linux/%s\n' "$1"
+}
+
+safe_tag_name() {
+  printf '%s' "$1" | tr '/:' '--'
+}
+
+runtime_image_tag() {
+  local arch="$1"
+  local revision="$2"
+  printf 'ours-rp-runtime-%s:%s\n' "$arch" "$revision"
+}
+
+metrics_image_tag() {
+  local arch="$1"
+  local revision="$2"
+  printf 'ours-rp-metrics-%s:%s\n' "$arch" "$revision"
+}
+
+default_package_prefix() {
+  printf 'ours-rp-installer-%s\n' "$1"
+}
+
+default_host_data_dir() {
+  printf '/var/lib/ours-rp-%s-installer\n' "$1"
+}
+
+default_compose_project() {
+  printf 'ours-rp-%s-installer\n' "$1"
+}
+
+default_metrics_instance() {
+  printf '%s-installer\n' "$1"
+}
+
+replace_or_append_env() {
+  local env_path="$1"
+  local key="$2"
+  local value="$3"
+  local tmp_env
+  tmp_env="${env_path}.tmp"
+  awk -v key="$key" -v value="$value" '
+    BEGIN { done=0 }
+    $0 ~ "^" key "=" { print key "=" value; done=1; next }
+    { print }
+    END { if (!done) print key "=" value }
+  ' "$env_path" > "$tmp_env"
+  mv "$tmp_env" "$env_path"
+}
+
+replace_text_in_file() {
+  local file_path="$1"
+  local old_text="$2"
+  local new_text="$3"
+  OLD_TEXT="$old_text" NEW_TEXT="$new_text" perl -0pi -e 's/\Q$ENV{OLD_TEXT}\E/$ENV{NEW_TEXT}/g' "$file_path"
+}
+
+ensure_image_for_platform() {
+  local image="$1"
+  local expected_platform="$2"
+  local role="$3"
+  local actual_platform
+  if ! docker image inspect "$image" >/dev/null 2>&1; then
+    echo "pulling $role image for $expected_platform: $image" >&2
+    docker pull --platform "$expected_platform" "$image" >&2
+  fi
+  actual_platform="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image" 2>/dev/null || echo unknown)"
+  if [[ "$actual_platform" != "$expected_platform" ]]; then
+    echo "re-pulling $role image for $expected_platform: $image (current=$actual_platform)" >&2
+    docker pull --platform "$expected_platform" "$image" >&2
+    actual_platform="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image" 2>/dev/null || echo unknown)"
+  fi
+  [[ "$actual_platform" == "$expected_platform" ]] || {
+    cat >&2 <<EOF
+wrong platform for $role image: $image
+  expected: $expected_platform
+  actual:   $actual_platform
+EOF
+    exit 2
+  }
+}
+
+save_image_if_needed() {
+  local image="$1"
+  local existing_tar="$2"
+  local out_dir="$3"
+  local role="$4"
+  local expected_platform="$5"
+  if [[ -n "$existing_tar" ]]; then
+    [[ -f "$existing_tar" ]] || {
+      echo "missing $role image tar: $existing_tar" >&2
+      exit 2
+    }
+    printf '%s\n' "$existing_tar"
+    return 0
+  fi
+  ensure_image_for_platform "$image" "$expected_platform" "$role"
+  local tar_path="$out_dir/$(safe_tag_name "$image")-${TARGET_ARCH}.tar.gz"
+  echo "saving $role image to $tar_path" >&2
+  docker save "$image" | gzip -c > "$tar_path"
+  printf '%s\n' "$tar_path"
+}
+
+image_label() {
+  local image="$1"
+  local key="$2"
+  docker image inspect --format "{{ index .Config.Labels \"$key\" }}" "$image"
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --arch)
+      TARGET_ARCH="$(normalize_arch "$2")" || {
+        echo "unsupported target architecture: $2" >&2
+        exit 2
+      }
+      shift 2
+      ;;
+    --image|--image-tar|--metrics-image|--metrics-image-tar)
+      echo "$1 is no longer supported: installer packages always rebuild runtime and metrics images from current HEAD" >&2
+      exit 2
+      ;;
+    --prometheus-image)
+      PROMETHEUS_IMAGE="$2"
+      shift 2
+      ;;
+    --prometheus-image-tar)
+      PROMETHEUS_IMAGE_TAR="$2"
+      shift 2
+      ;;
+    --grafana-image)
+      GRAFANA_IMAGE="$2"
+      shift 2
+      ;;
+    --grafana-image-tar)
+      GRAFANA_IMAGE_TAR="$2"
+      shift 2
+      ;;
+    --out-dir)
+      OUT_DIR="$2"
+      shift 2
+      ;;
+    --prefix)
+      PACKAGE_PREFIX="$2"
+      shift 2
+      ;;
+    --template-dir)
+      TEMPLATE_DIR="$2"
+      shift 2
+      ;;
+    --allow-dirty)
+      ALLOW_DIRTY=1
+      shift
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "unknown option: $1" >&2
+      usage >&2
+      exit 2
+      ;;
+  esac
+done
+
+[[ -n "$TARGET_ARCH" ]] || {
+  echo "--arch is required" >&2
+  usage >&2
+  exit 2
+}
+
+[[ -d "$TEMPLATE_DIR" ]] || {
+  echo "missing template dir: $TEMPLATE_DIR" >&2
+  exit 2
+}
+
+SOURCE_COMMIT_FULL="$(git -C "$REPO_ROOT" rev-parse HEAD)"
+SOURCE_COMMIT_SHORT="$(git -C "$REPO_ROOT" rev-parse --short=8 HEAD)"
+SOURCE_DIRTY="false"
+if [[ -n "$(git -C "$REPO_ROOT" status --short)" ]]; then
+  SOURCE_DIRTY="true"
+fi
+if [[ "$SOURCE_DIRTY" == "true" && "$ALLOW_DIRTY" != "1" ]]; then
+  cat >&2 <<EOF
+refusing to build a release installer from a dirty worktree.
+  source_commit: $SOURCE_COMMIT_FULL
+Use --allow-dirty only for development verification; the package and image tags
+will be explicitly marked dirty.
+EOF
+  exit 2
+fi
+TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)"
+REVISION_TOKEN="$SOURCE_COMMIT_SHORT"
+if [[ "$SOURCE_DIRTY" == "true" ]]; then
+  REVISION_TOKEN="${REVISION_TOKEN}-dirty"
+fi
+IMAGE_TAG="$(runtime_image_tag "$TARGET_ARCH" "$REVISION_TOKEN")"
+METRICS_IMAGE="$(metrics_image_tag "$TARGET_ARCH" "$REVISION_TOKEN")"
+if [[ -z "$OUT_DIR" ]]; then
+  OUT_DIR="$REPO_ROOT/target/${TARGET_ARCH}-installer"
+fi
+if [[ -z "$PACKAGE_PREFIX" ]]; then
+  PACKAGE_PREFIX="$(default_package_prefix "$TARGET_ARCH")"
+fi
+
+TARGET_PLATFORM="$(platform_for_arch "$TARGET_ARCH")"
+HOST_DATA_DIR_DEFAULT="$(default_host_data_dir "$TARGET_ARCH")"
+COMPOSE_PROJECT_DEFAULT="$(default_compose_project "$TARGET_ARCH")"
+METRICS_INSTANCE_DEFAULT="$(default_metrics_instance "$TARGET_ARCH")"
+IMAGE_OUT_DIR="$REPO_ROOT/target/${TARGET_ARCH}-docker"
+
+mkdir -p "$OUT_DIR"
+echo "rebuilding runtime image: $IMAGE_TAG"
+"$SCRIPT_DIR/build_docker_runtime_image.sh" \
+  --arch "$TARGET_ARCH" \
+  --image "$IMAGE_TAG" \
+  --out-dir "$IMAGE_OUT_DIR"
+echo "rebuilding metrics image: $METRICS_IMAGE"
+"$SCRIPT_DIR/build_docker_metrics_image.sh" \
+  --arch "$TARGET_ARCH" \
+  --image "$METRICS_IMAGE" \
+  --out-dir "$IMAGE_OUT_DIR"
+
+IMAGE_TAR="$IMAGE_OUT_DIR/$(safe_tag_name "$IMAGE_TAG").tar.gz"
+METRICS_IMAGE_TAR="$IMAGE_OUT_DIR/$(safe_tag_name "$METRICS_IMAGE").tar.gz"
+[[ -f "$IMAGE_TAR" ]] || { echo "runtime image build did not produce $IMAGE_TAR" >&2; exit 2; }
+[[ -f "$METRICS_IMAGE_TAR" ]] || { echo "metrics image build did not produce $METRICS_IMAGE_TAR" >&2; exit 2; }
+
+runtime_image_revision="$(image_label "$IMAGE_TAG" "org.opencontainers.image.revision")"
+metrics_image_revision="$(image_label "$METRICS_IMAGE" "org.opencontainers.image.revision")"
+runtime_image_dirty="$(image_label "$IMAGE_TAG" "org.opencontainers.image.source-dirty")"
+metrics_image_dirty="$(image_label "$METRICS_IMAGE" "org.opencontainers.image.source-dirty")"
+[[ "$runtime_image_revision" == "$SOURCE_COMMIT_FULL" ]] || { echo "runtime image revision mismatch: $runtime_image_revision != $SOURCE_COMMIT_FULL" >&2; exit 2; }
+[[ "$metrics_image_revision" == "$SOURCE_COMMIT_FULL" ]] || { echo "metrics image revision mismatch: $metrics_image_revision != $SOURCE_COMMIT_FULL" >&2; exit 2; }
+[[ "$runtime_image_dirty" == "$SOURCE_DIRTY" ]] || { echo "runtime image dirty flag mismatch" >&2; exit 2; }
+[[ "$metrics_image_dirty" == "$SOURCE_DIRTY" ]] || { echo "metrics image dirty flag mismatch" >&2; exit 2; }
+
+package_name="${PACKAGE_PREFIX}-${REVISION_TOKEN}-${TIMESTAMP}"
+stage="$OUT_DIR/$package_name"
+tar_path="$OUT_DIR/$package_name.tar.gz"
+
+rm -rf "$stage"
+rsync -a --delete "$TEMPLATE_DIR"/ "$stage"/
+mkdir -p "$stage/images"
+cp "$IMAGE_TAR" "$stage/images/"
+cp "$METRICS_IMAGE_TAR" "$stage/images/"
+
+monitor_image_stage="$OUT_DIR/.monitor-images-$TARGET_ARCH-$TIMESTAMP"
+rm -rf "$monitor_image_stage"
+mkdir -p "$monitor_image_stage"
+prometheus_tar="$(save_image_if_needed "$PROMETHEUS_IMAGE" "$PROMETHEUS_IMAGE_TAR" "$monitor_image_stage" "prometheus" "$TARGET_PLATFORM")"
+grafana_tar="$(save_image_if_needed "$GRAFANA_IMAGE" "$GRAFANA_IMAGE_TAR" "$monitor_image_stage" "grafana" "$TARGET_PLATFORM")"
+cp "$prometheus_tar" "$stage/images/"
+cp "$grafana_tar" "$stage/images/"
+
+if [[ -f "$stage/.env.example" ]]; then
+  replace_or_append_env "$stage/.env.example" "PACKAGE_ARCH" "$TARGET_ARCH"
+  replace_or_append_env "$stage/.env.example" "PACKAGE_PLATFORM" "$TARGET_PLATFORM"
+  replace_or_append_env "$stage/.env.example" "COMPOSE_PROJECT_NAME" "$COMPOSE_PROJECT_DEFAULT"
+  replace_or_append_env "$stage/.env.example" "RPKI_IMAGE" "$IMAGE_TAG"
+  replace_or_append_env "$stage/.env.example" "RPKI_PLATFORM" "$TARGET_PLATFORM"
+  replace_or_append_env "$stage/.env.example" "METRICS_IMAGE" "$METRICS_IMAGE"
+  replace_or_append_env "$stage/.env.example" "METRICS_PLATFORM" "$TARGET_PLATFORM"
+  replace_or_append_env "$stage/.env.example" "HOST_DATA_DIR" "$HOST_DATA_DIR_DEFAULT"
+  replace_or_append_env "$stage/.env.example" "RTR_REPORT_DIR" "$HOST_DATA_DIR_DEFAULT/empty-rtr-report"
+  replace_or_append_env "$stage/.env.example" "RTR_REPORT_CONTAINER_DIR" "/var/lib/ours-rp/rtr-report"
+  replace_or_append_env "$stage/.env.example" "ALLOW_CROSS_ARCH" "0"
+  replace_or_append_env "$stage/.env.example" "METRICS_INSTANCE" "$METRICS_INSTANCE_DEFAULT"
+  replace_or_append_env "$stage/.env.example" "MONITOR_PLATFORM" "$TARGET_PLATFORM"
+  replace_or_append_env "$stage/.env.example" "PROMETHEUS_IMAGE" "$PROMETHEUS_IMAGE"
+  replace_or_append_env "$stage/.env.example" "GRAFANA_IMAGE" "$GRAFANA_IMAGE"
+fi
+
+replace_text_in_file "$stage/docs/README.zh-CN.md" "__PACKAGE_ARCH__" "$TARGET_ARCH"
+replace_text_in_file "$stage/docs/README.en.md" "__PACKAGE_ARCH__" "$TARGET_ARCH"
+replace_text_in_file "$stage/docs/operations.zh-CN.md" "__PACKAGE_ARCH__" "$TARGET_ARCH"
+replace_text_in_file "$stage/docs/operations.en.md" "__PACKAGE_ARCH__" "$TARGET_ARCH"
+replace_text_in_file "$stage/docs/troubleshooting.zh-CN.md" "__PACKAGE_ARCH__" "$TARGET_ARCH"
+replace_text_in_file "$stage/docs/troubleshooting.en.md" "__PACKAGE_ARCH__" "$TARGET_ARCH"
+replace_text_in_file "$stage/docs/README.zh-CN.md" "__PACKAGE_PLATFORM__" "$TARGET_PLATFORM"
+replace_text_in_file "$stage/docs/README.en.md" "__PACKAGE_PLATFORM__" "$TARGET_PLATFORM"
+replace_text_in_file "$stage/docs/troubleshooting.zh-CN.md" "__PACKAGE_PLATFORM__" "$TARGET_PLATFORM"
+replace_text_in_file "$stage/docs/troubleshooting.en.md" "__PACKAGE_PLATFORM__" "$TARGET_PLATFORM"
+replace_text_in_file "$stage/docs/README.zh-CN.md" "__HOST_DATA_DIR__" "$HOST_DATA_DIR_DEFAULT"
+replace_text_in_file "$stage/docs/README.en.md" "__HOST_DATA_DIR__" "$HOST_DATA_DIR_DEFAULT"
+replace_text_in_file "$stage/docs/operations.zh-CN.md" "__HOST_DATA_DIR__" "$HOST_DATA_DIR_DEFAULT"
+replace_text_in_file "$stage/docs/operations.en.md" "__HOST_DATA_DIR__" "$HOST_DATA_DIR_DEFAULT"
+replace_text_in_file "$stage/docs/troubleshooting.zh-CN.md" "__HOST_DATA_DIR__" "$HOST_DATA_DIR_DEFAULT"
+replace_text_in_file "$stage/docs/troubleshooting.en.md" "__HOST_DATA_DIR__" "$HOST_DATA_DIR_DEFAULT"
+replace_text_in_file "$stage/docs/README.zh-CN.md" "__RUNTIME_IMAGE__" "$IMAGE_TAG"
+replace_text_in_file "$stage/docs/README.en.md" "__RUNTIME_IMAGE__" "$IMAGE_TAG"
+replace_text_in_file "$stage/docs/README.zh-CN.md" "__METRICS_IMAGE__" "$METRICS_IMAGE"
+replace_text_in_file "$stage/docs/README.en.md" "__METRICS_IMAGE__" "$METRICS_IMAGE"
+
+runtime_tar_sha256="$(sha256sum "$IMAGE_TAR" | awk '{print $1}')"
+metrics_tar_sha256="$(sha256sum "$METRICS_IMAGE_TAR" | awk '{print $1}')"
+prometheus_tar_sha256="$(sha256sum "$prometheus_tar" | awk '{print $1}')"
+grafana_tar_sha256="$(sha256sum "$grafana_tar" | awk '{print $1}')"
+
+cat > "$stage/PACKAGE-MANIFEST.env" <<EOF
+package_schema_version=2
+package_name=$package_name
+created_at_utc=$TIMESTAMP
+build_timestamp_utc=$TIMESTAMP
+source_commit=$SOURCE_COMMIT_FULL
+source_commit_short=$SOURCE_COMMIT_SHORT
+source_dirty=$SOURCE_DIRTY
+git_commit=$SOURCE_COMMIT_FULL
+git_status_count=$(git -C "$REPO_ROOT" status --short 2>/dev/null | wc -l | tr -d ' ')
+PACKAGE_ARCH=$TARGET_ARCH
+PACKAGE_PLATFORM=$TARGET_PLATFORM
+package_arch=$TARGET_ARCH
+package_platform=$TARGET_PLATFORM
+cross_arch_default=deny
+cross_arch_enable_var=ALLOW_CROSS_ARCH
+RPKI_IMAGE=$IMAGE_TAG
+RPKI_PLATFORM=$TARGET_PLATFORM
+image_tag=$IMAGE_TAG
+image_tar=$(basename "$IMAGE_TAR")
+image_tar_size_bytes=$(wc -c < "$IMAGE_TAR")
+image_tar_sha256=$runtime_tar_sha256
+runtime_image=$IMAGE_TAG
+runtime_image_revision=$runtime_image_revision
+runtime_image_dirty=$runtime_image_dirty
+rpki_image=$IMAGE_TAG
+rpki_platform=$TARGET_PLATFORM
+METRICS_IMAGE=$METRICS_IMAGE
+METRICS_PLATFORM=$TARGET_PLATFORM
+metrics_image=$METRICS_IMAGE
+metrics_image_tar=$(basename "$METRICS_IMAGE_TAR")
+metrics_image_tar_size_bytes=$(wc -c < "$METRICS_IMAGE_TAR")
+metrics_image_tar_sha256=$metrics_tar_sha256
+metrics_image_revision=$metrics_image_revision
+metrics_image_dirty=$metrics_image_dirty
+PROMETHEUS_IMAGE=$PROMETHEUS_IMAGE
+prometheus_image=$PROMETHEUS_IMAGE
+prometheus_image_tar=$(basename "$prometheus_tar")
+prometheus_image_tar_size_bytes=$(wc -c < "$prometheus_tar")
+prometheus_image_tar_sha256=$prometheus_tar_sha256
+GRAFANA_IMAGE=$GRAFANA_IMAGE
+grafana_image=$GRAFANA_IMAGE
+grafana_image_tar=$(basename "$grafana_tar")
+grafana_image_tar_size_bytes=$(wc -c < "$grafana_tar")
+grafana_image_tar_sha256=$grafana_tar_sha256
+target_platform=$TARGET_PLATFORM
+MONITOR_PLATFORM=$TARGET_PLATFORM
+rpki_image_tar=$(basename "$IMAGE_TAR")
+monitor_platform=$TARGET_PLATFORM
+metrics_platform=$TARGET_PLATFORM
+default_host_data_dir=$HOST_DATA_DIR_DEFAULT
+default_compose_project_name=$COMPOSE_PROJECT_DEFAULT
+default_metrics_instance=$METRICS_INSTANCE_DEFAULT
+EOF
+
+cat > "$stage/PACKAGE-SUMMARY.txt" <<EOF
+package_name: $package_name
+source_commit: $SOURCE_COMMIT_FULL
+source_commit_short: $SOURCE_COMMIT_SHORT
+source_dirty: $SOURCE_DIRTY
+build_timestamp_utc: $TIMESTAMP
+package_arch: $TARGET_ARCH
+package_platform: $TARGET_PLATFORM
+runtime_image: $IMAGE_TAG
+runtime_image_revision: $runtime_image_revision
+metrics_image: $METRICS_IMAGE
+metrics_image_revision: $metrics_image_revision
+prometheus_image: $PROMETHEUS_IMAGE
+grafana_image: $GRAFANA_IMAGE
+default_host_data_dir: $HOST_DATA_DIR_DEFAULT
+default_compose_project_name: $COMPOSE_PROJECT_DEFAULT
+cross_arch_default: deny
+cross_arch_enable_var: ALLOW_CROSS_ARCH
+EOF
+
+chmod +x "$stage/scripts"/*.sh
+
+if find "$stage" -maxdepth 1 -type f -name '*.sh' -print -quit | grep -q .; then
+  echo "error: operational scripts must be installed under scripts/ only" >&2
+  exit 1
+fi
+tar -C "$OUT_DIR" -czf "$tar_path" "$package_name"
+package_sha256="$(sha256sum "$tar_path" | awk '{print $1}')"
+rm -rf "$monitor_image_stage"
+
+{
+  echo "package=$tar_path"
+  echo "package_dir=$stage"
+  echo "package_size_bytes=$(wc -c < "$tar_path")"
+  echo "package_sha256=$package_sha256"
+  echo "source_commit=$SOURCE_COMMIT_FULL"
+  echo "source_commit_short=$SOURCE_COMMIT_SHORT"
+  echo "source_dirty=$SOURCE_DIRTY"
+  echo "build_timestamp_utc=$TIMESTAMP"
+  echo "package_arch=$TARGET_ARCH"
+  echo "package_platform=$TARGET_PLATFORM"
+  echo "manifest=$stage/PACKAGE-MANIFEST.env"
+  echo "summary=$stage/PACKAGE-SUMMARY.txt"
+} > "$OUT_DIR/$package_name.summary.env"
+
+echo "package built: $tar_path"

scripts/docker/build_docker_metrics_image.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+normalize_arch() {
+  case "$1" in
+    amd64|x86_64)
+      printf 'amd64\n'
+      ;;
+    arm64|aarch64)
+      printf 'arm64\n'
+      ;;
+    *)
+      return 1
+      ;;
+  esac
+}
+
+TARGET_ARCH="${TARGET_ARCH:-}"
+IMAGE_TAG="${IMAGE_TAG:-}"
+DOCKERFILE="${DOCKERFILE:-$REPO_ROOT/docker/ours-rp-metrics.Dockerfile}"
+args=()
+image_explicit=0
+dockerfile_explicit=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --arch)
+      TARGET_ARCH="$(normalize_arch "$2")" || {
+        echo "unsupported target architecture: $2" >&2
+        exit 2
+      }
+      args+=("$1" "$TARGET_ARCH")
+      shift 2
+      ;;
+    --image)
+      IMAGE_TAG="$2"
+      image_explicit=1
+      args+=("$1" "$2")
+      shift 2
+      ;;
+    --dockerfile)
+      DOCKERFILE="$2"
+      dockerfile_explicit=1
+      args+=("$1" "$2")
+      shift 2
+      ;;
+    *)
+      args+=("$1")
+      shift
+      ;;
+  esac
+done
+
+if [[ -z "$TARGET_ARCH" ]]; then
+  cat >&2 <<'EOF'
+missing target architecture.
+
+Usage:
+  scripts/docker/build_docker_metrics_image.sh --arch amd64|arm64 [options]
+EOF
+  exit 2
+fi
+
+if [[ -z "$IMAGE_TAG" ]]; then
+  SOURCE_COMMIT_SHORT="$(git -C "$REPO_ROOT" rev-parse --short=8 HEAD 2>/dev/null || echo unknown)"
+  if [[ -n "$(git -C "$REPO_ROOT" status --short 2>/dev/null)" ]]; then
+    SOURCE_COMMIT_SHORT="${SOURCE_COMMIT_SHORT}-dirty"
+  fi
+  IMAGE_TAG="ours-rp-metrics-${TARGET_ARCH}:${SOURCE_COMMIT_SHORT}"
+fi
+
+if [[ "$dockerfile_explicit" != "1" ]]; then
+  args=(--dockerfile "$DOCKERFILE" "${args[@]}")
+fi
+if [[ "$image_explicit" != "1" ]]; then
+  args=(--image "$IMAGE_TAG" "${args[@]}")
+fi
+
+exec "$SCRIPT_DIR/build_docker_runtime_image.sh" "${args[@]}"

scripts/docker/build_docker_runtime_image.sh

@@ -0,0 +1,295 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+TARGET_ARCH="${TARGET_ARCH:-}"
+IMAGE_TAG="${IMAGE_TAG:-}"
+BUILDER_IMAGE="${BUILDER_IMAGE:-rust:1-bookworm}"
+RUNTIME_IMAGE="${RUNTIME_IMAGE:-debian:bookworm-slim}"
+OUT_DIR="${OUT_DIR:-}"
+DOCKERFILE="${DOCKERFILE:-$REPO_ROOT/docker/ours-rp-runtime.Dockerfile}"
+BUILDER_NAME="${BUILDER_NAME:-default}"
+INSTALL_BINFMT="${INSTALL_BINFMT:-1}"
+SAVE_IMAGE="${SAVE_IMAGE:-1}"
+LOAD_IMAGE="${LOAD_IMAGE:-1}"
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/docker/build_docker_runtime_image.sh --arch amd64|arm64 [options]
+
+Options:
+  --arch <arch>       Target architecture: amd64|arm64
+  --image <tag>       Docker image tag (default: ours-rp-runtime-<arch>:<git8>)
+  --out-dir <path>    Directory for docker save tar.gz (default: target/<arch>-docker)
+  --dockerfile <path> Dockerfile path
+  --builder <name>    buildx builder name
+  --builder-image <tag>
+                      Builder base image (default: rust:1-bookworm)
+  --runtime-image <tag>
+                      Runtime base image (default: debian:bookworm-slim)
+  --no-binfmt         Do not install binfmt/qemu for cross-architecture builds
+  --no-save           Build image but do not docker save it
+  --no-load           Use buildx output tar instead of --load
+  -h, --help          Show this help
+USAGE
+}
+
+normalize_arch() {
+  case "$1" in
+    amd64|x86_64)
+      printf 'amd64\n'
+      ;;
+    arm64|aarch64)
+      printf 'arm64\n'
+      ;;
+    *)
+      return 1
+      ;;
+  esac
+}
+
+safe_tag_name() {
+  printf '%s' "$1" | tr '/:' '--'
+}
+
+local_base_image_tag() {
+  local role="$1"
+  local arch="$2"
+  local source_image="$3"
+  printf 'ours-rp-base-%s-%s:%s\n' "$role" "$arch" "$(safe_tag_name "$source_image")"
+}
+
+platform_for_arch() {
+  printf 'linux/%s\n' "$1"
+}
+
+default_image_tag() {
+  local arch="$1"
+  local commit_short="$2"
+  printf 'ours-rp-runtime-%s:%s\n' "$arch" "$commit_short"
+}
+
+require_command() {
+  command -v "$1" >/dev/null 2>&1 || {
+    echo "missing required command: $1" >&2
+    exit 2
+  }
+}
+
+host_arch() {
+  local raw_arch
+  raw_arch="$(uname -m)"
+  normalize_arch "$raw_arch" || {
+    echo "unsupported host architecture: $raw_arch" >&2
+    exit 2
+  }
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --arch)
+      TARGET_ARCH="$(normalize_arch "$2")" || {
+        echo "unsupported target architecture: $2" >&2
+        exit 2
+      }
+      shift 2
+      ;;
+    --image)
+      IMAGE_TAG="$2"
+      shift 2
+      ;;
+    --out-dir)
+      OUT_DIR="$2"
+      shift 2
+      ;;
+    --dockerfile)
+      DOCKERFILE="$2"
+      shift 2
+      ;;
+    --builder)
+      BUILDER_NAME="$2"
+      shift 2
+      ;;
+    --builder-image)
+      BUILDER_IMAGE="$2"
+      shift 2
+      ;;
+    --runtime-image)
+      RUNTIME_IMAGE="$2"
+      shift 2
+      ;;
+    --no-binfmt)
+      INSTALL_BINFMT=0
+      shift
+      ;;
+    --no-save)
+      SAVE_IMAGE=0
+      shift
+      ;;
+    --no-load)
+      LOAD_IMAGE=0
+      shift
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "unknown option: $1" >&2
+      usage >&2
+      exit 2
+      ;;
+  esac
+done
+
+[[ -n "$TARGET_ARCH" ]] || {
+  echo "--arch is required" >&2
+  usage >&2
+  exit 2
+}
+
+SOURCE_COMMIT_FULL="$(git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null || echo unknown)"
+SOURCE_COMMIT_SHORT="$(git -C "$REPO_ROOT" rev-parse --short=8 HEAD 2>/dev/null || echo unknown)"
+SOURCE_DIRTY="false"
+if [[ -n "$(git -C "$REPO_ROOT" status --short 2>/dev/null)" ]]; then
+  SOURCE_DIRTY="true"
+fi
+BUILD_TIMESTAMP_UTC="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+if [[ -z "$IMAGE_TAG" ]]; then
+  default_revision="$SOURCE_COMMIT_SHORT"
+  if [[ "$SOURCE_DIRTY" == "true" ]]; then
+    default_revision="${default_revision}-dirty"
+  fi
+  IMAGE_TAG="$(default_image_tag "$TARGET_ARCH" "$default_revision")"
+fi
+if [[ -z "$OUT_DIR" ]]; then
+  OUT_DIR="$REPO_ROOT/target/${TARGET_ARCH}-docker"
+fi
+
+require_command docker
+mkdir -p "$OUT_DIR"
+
+target_platform="$(platform_for_arch "$TARGET_ARCH")"
+local_host_arch="$(host_arch)"
+builder_platform="$(platform_for_arch "$local_host_arch")"
+builder_source_image="$BUILDER_IMAGE"
+runtime_source_image="$RUNTIME_IMAGE"
+builder_local_image="$(local_base_image_tag builder "$local_host_arch" "$builder_source_image")"
+runtime_local_image="$(local_base_image_tag runtime "$TARGET_ARCH" "$runtime_source_image")"
+
+if [[ "$INSTALL_BINFMT" == "1" && "$local_host_arch" != "$TARGET_ARCH" ]]; then
+  echo "installing binfmt/qemu for $TARGET_ARCH"
+  docker run --rm --privileged tonistiigi/binfmt --install "$TARGET_ARCH"
+fi
+
+# Pulling through the Docker daemon uses its configured proxy. Buildx then consumes
+# these architecture-pinned local aliases without issuing separate registry metadata
+# requests for the Dockerfile FROM instructions.
+echo "pulling builder base image: $builder_source_image ($builder_platform)"
+docker pull --platform "$builder_platform" "$builder_source_image"
+docker tag "$builder_source_image" "$builder_local_image"
+echo "pulling runtime base image: $runtime_source_image ($target_platform)"
+docker pull --platform "$target_platform" "$runtime_source_image"
+docker tag "$runtime_source_image" "$runtime_local_image"
+BUILDER_IMAGE="$builder_local_image"
+RUNTIME_IMAGE="$runtime_local_image"
+
+if [[ "$BUILDER_NAME" != "default" ]] && ! docker buildx inspect "$BUILDER_NAME" >/dev/null 2>&1; then
+  docker buildx create --name "$BUILDER_NAME" --driver docker-container --use >/dev/null
+else
+  docker buildx use "$BUILDER_NAME" >/dev/null
+fi
+docker buildx inspect --bootstrap >/dev/null
+
+metadata_path="$OUT_DIR/$(safe_tag_name "$IMAGE_TAG").build-metadata.json"
+tar_path="$OUT_DIR/$(safe_tag_name "$IMAGE_TAG").tar.gz"
+build_log="$OUT_DIR/$(safe_tag_name "$IMAGE_TAG").build.log"
+
+echo "building $target_platform image: $IMAGE_TAG"
+echo "repo: $REPO_ROOT"
+echo "dockerfile: $DOCKERFILE"
+echo "builder_source_image: $builder_source_image"
+echo "runtime_source_image: $runtime_source_image"
+echo "builder_local_image: $BUILDER_IMAGE"
+echo "runtime_local_image: $RUNTIME_IMAGE"
+echo "source_commit: $SOURCE_COMMIT_FULL"
+echo "source_dirty: $SOURCE_DIRTY"
+echo "build_timestamp_utc: $BUILD_TIMESTAMP_UTC"
+start_epoch="$(date +%s)"
+
+if [[ "$LOAD_IMAGE" == "1" ]]; then
+  docker buildx build \
+    --platform "$target_platform" \
+    --builder "$BUILDER_NAME" \
+    --pull=false \
+    --load \
+    --build-arg BUILDKIT_INLINE_CACHE=1 \
+    --build-arg "BUILDER_IMAGE=$BUILDER_IMAGE" \
+    --build-arg "RUNTIME_IMAGE=$RUNTIME_IMAGE" \
+    --build-arg "SOURCE_COMMIT=$SOURCE_COMMIT_FULL" \
+    --build-arg "SOURCE_DIRTY=$SOURCE_DIRTY" \
+    --build-arg "BUILD_TIMESTAMP_UTC=$BUILD_TIMESTAMP_UTC" \
+    --metadata-file "$metadata_path" \
+    -t "$IMAGE_TAG" \
+    -f "$DOCKERFILE" \
+    "$REPO_ROOT" 2>&1 | tee "$build_log"
+else
+  raw_tar_path="$OUT_DIR/$(safe_tag_name "$IMAGE_TAG").tar"
+  docker buildx build \
+    --platform "$target_platform" \
+    --builder "$BUILDER_NAME" \
+    --pull=false \
+    --output "type=docker,dest=$raw_tar_path" \
+    --build-arg BUILDKIT_INLINE_CACHE=1 \
+    --build-arg "BUILDER_IMAGE=$BUILDER_IMAGE" \
+    --build-arg "RUNTIME_IMAGE=$RUNTIME_IMAGE" \
+    --build-arg "SOURCE_COMMIT=$SOURCE_COMMIT_FULL" \
+    --build-arg "SOURCE_DIRTY=$SOURCE_DIRTY" \
+    --build-arg "BUILD_TIMESTAMP_UTC=$BUILD_TIMESTAMP_UTC" \
+    --metadata-file "$metadata_path" \
+    -t "$IMAGE_TAG" \
+    -f "$DOCKERFILE" \
+    "$REPO_ROOT" 2>&1 | tee "$build_log"
+  gzip -f "$raw_tar_path"
+  tar_path="${raw_tar_path}.gz"
+fi
+
+elapsed_secs=$(( $(date +%s) - start_epoch ))
+
+if [[ "$SAVE_IMAGE" == "1" && "$LOAD_IMAGE" == "1" ]]; then
+  echo "saving image to $tar_path"
+  docker save "$IMAGE_TAG" | gzip -c > "$tar_path"
+fi
+
+tar_sha256=""
+if [[ -f "$tar_path" ]]; then
+  tar_sha256="$(sha256sum "$tar_path" | awk '{print $1}')"
+fi
+
+{
+  echo "image=$IMAGE_TAG"
+  echo "platform=$target_platform"
+  echo "arch=$TARGET_ARCH"
+  echo "builder_image=$builder_source_image"
+  echo "runtime_image=$runtime_source_image"
+  echo "builder_local_image=$BUILDER_IMAGE"
+  echo "runtime_local_image=$RUNTIME_IMAGE"
+  echo "elapsed_secs=$elapsed_secs"
+  echo "metadata=$metadata_path"
+  echo "tar=$tar_path"
+  echo "tar_size_bytes=$(wc -c < "$tar_path" 2>/dev/null || echo 0)"
+  echo "tar_sha256=$tar_sha256"
+  echo "source_commit=$SOURCE_COMMIT_FULL"
+  echo "source_commit_short=$SOURCE_COMMIT_SHORT"
+  echo "source_dirty=$SOURCE_DIRTY"
+  echo "build_timestamp_utc=$BUILD_TIMESTAMP_UTC"
+  echo "git_commit=$SOURCE_COMMIT_FULL"
+  echo "git_status_count=$(git -C "$REPO_ROOT" status --short 2>/dev/null | wc -l | tr -d ' ')"
+  echo "built_at_utc=$BUILD_TIMESTAMP_UTC"
+} > "$OUT_DIR/$(safe_tag_name "$IMAGE_TAG").build-summary.env"
+
+echo "build complete: elapsed=${elapsed_secs}s tar=$tar_path"