20260609 增加三RP持续对比监控

replay 对比，五个RIR 输出vrp与routinator一致

.gitignore

@@ -1,2 +1,4 @@
 target/
 Cargo.lock
+perf.*
+specs/* copy.excalidraw

Cargo.toml

@@ -3,13 +3,34 @@ name = "rpki"
 version = "0.1.0"
 edition = "2024"
 
+[features]
+default = ["full"]
+# Full build used by the main RP implementation (includes RocksDB-backed storage).
+full = ["dep:rocksdb"]
+profile = ["dep:pprof", "dep:flate2"]
+
 [dependencies]
-der-parser = "10.0.0"
+asn1-rs = "0.7.1"
+der-parser = { version = "10.0.0", features = ["serialize"] }
 hex = "0.4.3"
+base64 = "0.22.1"
+sha2 = "0.10.8"
 thiserror = "2.0.18"
 time = "0.3.45"
+ring = "0.17.14"
 x509-parser = { version = "0.18.0", features = ["verify"] }
 url = "2.5.8"
-asn1-rs = "0.7.1"
-asn1-rs-derive = "0.6.0"
-asn1 = "0.23.0"
+serde = { version = "1.0.218", features = ["derive"] }
+serde_json = "1.0.140"
+toml = "0.8.20"
+rocksdb = { version = "0.22.0", optional = true, default-features = false, features = ["lz4"] }
+serde_cbor = "0.11.2"
+roxmltree = "0.20.0"
+quick-xml = "0.37.2"
+uuid = { version = "1.7.0", features = ["v4"] }
+reqwest = { version = "0.12.12", default-features = false, features = ["blocking", "rustls-tls", "gzip", "brotli", "deflate"] }
+pprof = { version = "0.14.1", optional = true, features = ["flamegraph", "prost-codec"] }
+flate2 = { version = "1.0.35", optional = true }
+tempfile = "3.16.0"
+
+[dev-dependencies]

README.md

@@ -9,3 +9,55 @@ cargo test
 cargo test -- --nocapture
 ```
 
+# 覆盖率（cargo-llvm-cov）
+
+安装工具：
+
+```
+rustup component add llvm-tools-preview
+cargo install cargo-llvm-cov --locked
+```
+
+统计行覆盖率并要求 >=90%：
+
+```
+./scripts/coverage.sh
+# 或
+cargo llvm-cov --fail-under-lines 90
+```
+
+默认会复用现有插桩产物，不会先 clean。需要强制全量重编译时：
+
+```
+COVERAGE_FORCE_CLEAN=1 ./scripts/coverage.sh
+```
+
+说明：
+- 默认行为适合本地重复确认覆盖率，避免每次都重编译整套插桩目标；
+- 默认还会设置 `RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS=1`，跳过会拉起 shell replay pipeline 的重型集成测试，避免 coverage 期间额外触发 `target/release` 构建；
+- 默认还会设置 `RPKI_SKIP_HEAVY_BLACKBOX_TESTS=1`，跳过更慢的 blackbox CLI / CIR record 脚本测试，进一步降低日常 coverage 成本；
+- 默认还会设置 `RPKI_SKIP_HEAVY_CRYPTO_TESTS=1`，跳过需要大量 OpenSSL 生成证书/CRL 的重型密码学测试，进一步压缩日常 coverage 时长；
+- 如需把这批脚本回放测试也纳入 coverage，可显式关闭该开关：
+
+```
+RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS=0 ./scripts/coverage.sh
+```
+
+如需连同第二批 blackbox 测试一起跑：
+
+```
+RPKI_SKIP_HEAVY_BLACKBOX_TESTS=0 ./scripts/coverage.sh
+```
+
+如需连同重型 OpenSSL 证书路径测试一起跑：
+
+```
+RPKI_SKIP_HEAVY_CRYPTO_TESTS=0 ./scripts/coverage.sh
+```
+
+- replay 脚本现在也支持通过环境变量注入现成二进制，避免找不到二进制时自动 `cargo build --release`：
+  - `RPKI_BIN`
+  - `CIR_MATERIALIZE_BIN`
+  - `CIR_EXTRACT_INPUTS_BIN`
+  - `CCR_TO_COMPARE_VIEWS_BIN`
+- `COVERAGE_FORCE_CLEAN=1` 适合需要完全从零重建插桩目标时使用。

benchmark/ours_manifest_bench/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "ours-manifest-bench"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+rpki = { path = "../..", default-features = false }
+

benchmark/ours_manifest_bench/src/main.rs

@@ -0,0 +1,145 @@
+use rpki::data_model::manifest::ManifestObject;
+use std::hint::black_box;
+use std::path::PathBuf;
+use std::time::Instant;
+
+#[derive(Debug, Clone)]
+struct Config {
+    sample: Option<String>,
+    manifest_path: Option<PathBuf>,
+    iterations: u64,
+    warmup_iterations: u64,
+    repeats: u32,
+}
+
+fn usage_and_exit() -> ! {
+    eprintln!(
+        "Usage:\n  ours-manifest-bench (--sample <name> | --manifest <path>) [--iterations N] [--warmup-iterations N] [--repeats N]\n\nExamples:\n  cargo run --release -- --sample small-01 --iterations 20000 --warmup-iterations 2000 --repeats 3\n  cargo run --release -- --manifest ../../tests/benchmark/selected_der/small-01.mft"
+    );
+    std::process::exit(2);
+}
+
+fn parse_args() -> Config {
+    let mut sample: Option<String> = None;
+    let mut manifest_path: Option<PathBuf> = None;
+    let mut iterations: u64 = 20_000;
+    let mut warmup_iterations: u64 = 2_000;
+    let mut repeats: u32 = 3;
+
+    let mut args = std::env::args().skip(1);
+    while let Some(arg) = args.next() {
+        match arg.as_str() {
+            "--sample" => sample = Some(args.next().unwrap_or_else(|| usage_and_exit())),
+            "--manifest" => {
+                manifest_path = Some(PathBuf::from(args.next().unwrap_or_else(|| usage_and_exit())))
+            }
+            "--iterations" => {
+                iterations = args
+                    .next()
+                    .unwrap_or_else(|| usage_and_exit())
+                    .parse()
+                    .unwrap_or_else(|_| usage_and_exit())
+            }
+            "--warmup-iterations" => {
+                warmup_iterations = args
+                    .next()
+                    .unwrap_or_else(|| usage_and_exit())
+                    .parse()
+                    .unwrap_or_else(|_| usage_and_exit())
+            }
+            "--repeats" => {
+                repeats = args
+                    .next()
+                    .unwrap_or_else(|| usage_and_exit())
+                    .parse()
+                    .unwrap_or_else(|_| usage_and_exit())
+            }
+            "-h" | "--help" => usage_and_exit(),
+            _ => usage_and_exit(),
+        }
+    }
+
+    if sample.is_none() && manifest_path.is_none() {
+        usage_and_exit();
+    }
+    if sample.is_some() && manifest_path.is_some() {
+        usage_and_exit();
+    }
+
+    Config {
+        sample,
+        manifest_path,
+        iterations,
+        warmup_iterations,
+        repeats,
+    }
+}
+
+fn derive_manifest_path(sample: &str) -> PathBuf {
+    // Assumes current working directory is `rpki/benchmark/ours_manifest_bench`.
+    PathBuf::from(format!("../../tests/benchmark/selected_der/{sample}.mft"))
+}
+
+fn main() {
+    let cfg = parse_args();
+    let manifest_path = cfg
+        .manifest_path
+        .clone()
+        .unwrap_or_else(|| derive_manifest_path(cfg.sample.as_deref().unwrap()));
+
+    let bytes = std::fs::read(&manifest_path).unwrap_or_else(|e| {
+        eprintln!("read manifest fixture failed: {e}; path={}", manifest_path.display());
+        std::process::exit(1);
+    });
+
+    let decoded_once = ManifestObject::decode_der(&bytes).unwrap_or_else(|e| {
+        eprintln!("decode failed: {e}; path={}", manifest_path.display());
+        std::process::exit(1);
+    });
+    let file_count = decoded_once.manifest.file_count();
+
+    let mut round_ns_per_op: Vec<f64> = Vec::with_capacity(cfg.repeats as usize);
+    let mut round_ops_per_s: Vec<f64> = Vec::with_capacity(cfg.repeats as usize);
+
+    for _round in 0..cfg.repeats {
+        for _ in 0..cfg.warmup_iterations {
+            let obj = ManifestObject::decode_der(black_box(&bytes)).expect("warmup decode");
+            black_box(obj);
+        }
+
+        let start = Instant::now();
+        for _ in 0..cfg.iterations {
+            let obj = ManifestObject::decode_der(black_box(&bytes)).expect("timed decode");
+            black_box(obj);
+        }
+        let elapsed = start.elapsed();
+
+        let ns_per_op = (elapsed.as_secs_f64() * 1e9) / (cfg.iterations as f64);
+        let ops_per_s = (cfg.iterations as f64) / elapsed.as_secs_f64();
+        round_ns_per_op.push(ns_per_op);
+        round_ops_per_s.push(ops_per_s);
+    }
+
+    let avg_ns_per_op = round_ns_per_op.iter().sum::<f64>() / (round_ns_per_op.len() as f64);
+    let avg_ops_per_s = round_ops_per_s.iter().sum::<f64>() / (round_ops_per_s.len() as f64);
+
+    let sample_name = cfg.sample.clone().unwrap_or_else(|| {
+        manifest_path
+            .file_name()
+            .map(|s| s.to_string_lossy().to_string())
+            .unwrap_or_else(|| manifest_path.display().to_string())
+    });
+    let sample_name = sample_name
+        .strip_suffix(".mft")
+        .unwrap_or(&sample_name)
+        .to_string();
+
+    println!("fixture: {}", manifest_path.display());
+    println!();
+    println!("| sample | avg ns/op | ops/s | file count |");
+    println!("|---|---:|---:|---:|");
+    println!(
+        "| {} | {:.2} | {:.2} | {} |",
+        sample_name, avg_ns_per_op, avg_ops_per_s, file_count
+    );
+}

benchmark/routinator_object_bench/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "routinator-object-bench"
+version = "0.1.0"
+edition = "2024"
+publish = false
+
+[dependencies]
+rpki = { version = "=0.19.1", features = ["repository"] }

benchmark/routinator_object_bench/src/main.rs

@@ -0,0 +1,552 @@
+use rpki::repository::cert::Cert;
+use rpki::repository::crl::Crl;
+use rpki::repository::manifest::Manifest;
+use rpki::repository::roa::Roa;
+use rpki::repository::aspa::Aspa;
+use rpki::repository::resources::{AsResources, IpResources};
+
+use std::hint::black_box;
+use std::path::{Path, PathBuf};
+use std::time::Instant;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+enum ObjType {
+    Cer,
+    Crl,
+    Manifest,
+    Roa,
+    Aspa,
+}
+
+impl ObjType {
+    fn parse(s: &str) -> Result<Self, String> {
+        match s {
+            "cer" => Ok(Self::Cer),
+            "crl" => Ok(Self::Crl),
+            "manifest" => Ok(Self::Manifest),
+            "roa" => Ok(Self::Roa),
+            "aspa" => Ok(Self::Aspa),
+            _ => Err("type must be one of: cer, crl, manifest, roa, aspa".into()),
+        }
+    }
+
+    fn as_str(self) -> &'static str {
+        match self {
+            ObjType::Cer => "cer",
+            ObjType::Crl => "crl",
+            ObjType::Manifest => "manifest",
+            ObjType::Roa => "roa",
+            ObjType::Aspa => "aspa",
+        }
+    }
+
+    fn ext(self) -> &'static str {
+        match self {
+            ObjType::Cer => "cer",
+            ObjType::Crl => "crl",
+            ObjType::Manifest => "mft",
+            ObjType::Roa => "roa",
+            ObjType::Aspa => "asa",
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+struct Sample {
+    obj_type: ObjType,
+    name: String,
+    path: PathBuf,
+}
+
+#[derive(Clone, Debug)]
+struct Config {
+    dir: PathBuf,
+    type_filter: Option<ObjType>,
+    sample_filter: Option<String>,
+    fixed_iters: Option<u64>,
+    warmup_iters: u64,
+    rounds: u64,
+    min_round_ms: u64,
+    max_adaptive_iters: u64,
+    strict: bool,
+    cert_inspect: bool,
+    out_csv: Option<PathBuf>,
+    out_md: Option<PathBuf>,
+}
+
+fn usage_and_exit(err: Option<&str>) -> ! {
+    if let Some(err) = err {
+        eprintln!("error: {err}");
+        eprintln!();
+    }
+    eprintln!(
+        "Usage:\n\
+  cargo run --release --manifest-path rpki/benchmark/routinator_object_bench/Cargo.toml -- [OPTIONS]\n\
+\n\
+Options:\n\
+  --dir <PATH>            Fixtures root dir (default: ../../tests/benchmark/selected_der_v2)\n\
+  --type <cer|crl|manifest|roa|aspa>    Filter by type\n\
+  --sample <NAME>         Filter by sample name (e.g. p50)\n\
+  --iters <N>             Fixed iterations per round (optional; otherwise adaptive)\n\
+  --warmup-iters <N>      Warmup iterations (default: 50)\n\
+  --rounds <N>            Rounds (default: 5)\n\
+  --min-round-ms <MS>     Adaptive: minimum round time (default: 200)\n\
+  --max-iters <N>         Adaptive: maximum iters (default: 1_000_000)\n\
+  --strict <true|false>   Strict DER where applicable (default: true)\n\
+  --cert-inspect          Also run Cert::inspect_ca/inspect_ee where applicable (default: false)\n\
+  --out-csv <PATH>        Write CSV output\n\
+  --out-md <PATH>         Write Markdown output\n\
+"
+    );
+    std::process::exit(2);
+}
+
+fn parse_bool(s: &str, name: &str) -> bool {
+    match s {
+        "1" | "true" | "TRUE" | "yes" | "YES" => true,
+        "0" | "false" | "FALSE" | "no" | "NO" => false,
+        _ => usage_and_exit(Some(&format!("{name} must be true/false"))),
+    }
+}
+
+fn parse_u64(s: &str, name: &str) -> u64 {
+    s.parse::<u64>()
+        .unwrap_or_else(|_| usage_and_exit(Some(&format!("{name} must be an integer"))))
+}
+
+fn default_samples_dir() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/benchmark/selected_der_v2")
+}
+
+fn parse_args() -> Config {
+    let mut dir: PathBuf = default_samples_dir();
+    let mut type_filter: Option<ObjType> = None;
+    let mut sample_filter: Option<String> = None;
+    let mut fixed_iters: Option<u64> = None;
+    let mut warmup_iters: u64 = 50;
+    let mut rounds: u64 = 5;
+    let mut min_round_ms: u64 = 200;
+    let mut max_adaptive_iters: u64 = 1_000_000;
+    let mut strict: bool = true;
+    let mut cert_inspect: bool = false;
+    let mut out_csv: Option<PathBuf> = None;
+    let mut out_md: Option<PathBuf> = None;
+
+    let mut args = std::env::args().skip(1);
+    while let Some(arg) = args.next() {
+        match arg.as_str() {
+            "--dir" => dir = PathBuf::from(args.next().unwrap_or_else(|| usage_and_exit(None))),
+            "--type" => {
+                type_filter = Some(ObjType::parse(
+                    &args.next().unwrap_or_else(|| usage_and_exit(None)),
+                )
+                .unwrap_or_else(|e| usage_and_exit(Some(&e))))
+            }
+            "--sample" => {
+                sample_filter = Some(args.next().unwrap_or_else(|| usage_and_exit(None)))
+            }
+            "--iters" => {
+                fixed_iters = Some(parse_u64(
+                    &args.next().unwrap_or_else(|| usage_and_exit(None)),
+                    "--iters",
+                ))
+            }
+            "--warmup-iters" => {
+                warmup_iters = parse_u64(
+                    &args.next().unwrap_or_else(|| usage_and_exit(None)),
+                    "--warmup-iters",
+                )
+            }
+            "--rounds" => {
+                rounds = parse_u64(&args.next().unwrap_or_else(|| usage_and_exit(None)), "--rounds")
+            }
+            "--min-round-ms" => {
+                min_round_ms = parse_u64(
+                    &args.next().unwrap_or_else(|| usage_and_exit(None)),
+                    "--min-round-ms",
+                )
+            }
+            "--max-iters" => {
+                max_adaptive_iters = parse_u64(
+                    &args.next().unwrap_or_else(|| usage_and_exit(None)),
+                    "--max-iters",
+                )
+            }
+            "--strict" => {
+                strict = parse_bool(
+                    &args.next().unwrap_or_else(|| usage_and_exit(None)),
+                    "--strict",
+                )
+            }
+            "--cert-inspect" => cert_inspect = true,
+            "--out-csv" => out_csv = Some(PathBuf::from(args.next().unwrap_or_else(|| usage_and_exit(None)))),
+            "--out-md" => out_md = Some(PathBuf::from(args.next().unwrap_or_else(|| usage_and_exit(None)))),
+            "-h" | "--help" => usage_and_exit(None),
+            _ => usage_and_exit(Some(&format!("unknown argument: {arg}"))),
+        }
+    }
+
+    if warmup_iters == 0 {
+        usage_and_exit(Some("--warmup-iters must be > 0"));
+    }
+    if rounds == 0 {
+        usage_and_exit(Some("--rounds must be > 0"));
+    }
+    if min_round_ms == 0 {
+        usage_and_exit(Some("--min-round-ms must be > 0"));
+    }
+    if max_adaptive_iters == 0 {
+        usage_and_exit(Some("--max-iters must be > 0"));
+    }
+    if let Some(n) = fixed_iters {
+        if n == 0 {
+            usage_and_exit(Some("--iters must be > 0"));
+        }
+    }
+
+    Config {
+        dir,
+        type_filter,
+        sample_filter,
+        fixed_iters,
+        warmup_iters,
+        rounds,
+        min_round_ms,
+        max_adaptive_iters,
+        strict,
+        cert_inspect,
+        out_csv,
+        out_md,
+    }
+}
+
+fn read_samples(root: &Path) -> Vec<Sample> {
+    let mut out = Vec::new();
+    for obj_type in [
+        ObjType::Cer,
+        ObjType::Crl,
+        ObjType::Manifest,
+        ObjType::Roa,
+        ObjType::Aspa,
+    ] {
+        let dir = root.join(obj_type.as_str());
+        let rd = match std::fs::read_dir(&dir) {
+            Ok(rd) => rd,
+            Err(_) => continue,
+        };
+        for ent in rd.flatten() {
+            let path = ent.path();
+            if path.extension().and_then(|s| s.to_str()) != Some(obj_type.ext()) {
+                continue;
+            }
+            let name = path
+                .file_stem()
+                .and_then(|s| s.to_str())
+                .unwrap_or("unknown")
+                .to_string();
+            out.push(Sample { obj_type, name, path });
+        }
+    }
+    out.sort_by(|a, b| a.obj_type.cmp(&b.obj_type).then_with(|| a.name.cmp(&b.name)));
+    out
+}
+
+fn choose_iters_adaptive<F: FnMut()>(mut op: F, min_round_ms: u64, max_iters: u64) -> u64 {
+    let min_secs = (min_round_ms as f64) / 1e3;
+    let mut iters: u64 = 1;
+    loop {
+        let start = Instant::now();
+        for _ in 0..iters {
+            op();
+        }
+        let elapsed = start.elapsed().as_secs_f64();
+        if elapsed >= min_secs {
+            return iters;
+        }
+        if iters >= max_iters {
+            return iters;
+        }
+        iters = (iters.saturating_mul(2)).min(max_iters);
+    }
+}
+
+fn count_ip(res: &IpResources) -> u64 {
+    if res.is_inherited() {
+        return 1;
+    }
+    let Ok(blocks) = res.to_blocks() else {
+        return 0;
+    };
+    blocks.iter().count() as u64
+}
+
+fn count_as(res: &AsResources) -> u64 {
+    if res.is_inherited() {
+        return 1;
+    }
+    let Ok(blocks) = res.to_blocks() else {
+        return 0;
+    };
+    blocks.iter().count() as u64
+}
+
+fn complexity(obj_type: ObjType, bytes: &[u8], strict: bool, cert_inspect: bool) -> u64 {
+    match obj_type {
+        ObjType::Cer => {
+            let cert = Cert::decode(bytes).expect("decode cert");
+            if cert_inspect {
+                if cert.is_ca() {
+                    cert.inspect_ca(strict).expect("inspect ca");
+                } else {
+                    cert.inspect_ee(strict).expect("inspect ee");
+                }
+            }
+            count_ip(cert.v4_resources())
+                .saturating_add(count_ip(cert.v6_resources()))
+                .saturating_add(count_as(cert.as_resources()))
+        }
+        ObjType::Crl => {
+            let crl = Crl::decode(bytes).expect("decode crl");
+            crl.revoked_certs().iter().count() as u64
+        }
+        ObjType::Manifest => {
+            let mft = Manifest::decode(bytes, strict).expect("decode manifest");
+            if cert_inspect {
+                mft.cert().inspect_ee(strict).expect("inspect ee");
+            }
+            mft.content().len() as u64
+        }
+        ObjType::Roa => {
+            let roa = Roa::decode(bytes, strict).expect("decode roa");
+            if cert_inspect {
+                roa.cert().inspect_ee(strict).expect("inspect ee");
+            }
+            roa.content().iter().count() as u64
+        }
+        ObjType::Aspa => {
+            let asa = Aspa::decode(bytes, strict).expect("decode aspa");
+            if cert_inspect {
+                asa.cert().inspect_ee(strict).expect("inspect ee");
+            }
+            asa.content().provider_as_set().len() as u64
+        }
+    }
+}
+
+fn decode_profile(obj_type: ObjType, bytes: &[u8], strict: bool, cert_inspect: bool) {
+    match obj_type {
+        ObjType::Cer => {
+            let cert = Cert::decode(black_box(bytes)).expect("decode cert");
+            if cert_inspect {
+                if cert.is_ca() {
+                    cert.inspect_ca(strict).expect("inspect ca");
+                } else {
+                    cert.inspect_ee(strict).expect("inspect ee");
+                }
+            }
+            black_box(cert);
+        }
+        ObjType::Crl => {
+            let crl = Crl::decode(black_box(bytes)).expect("decode crl");
+            black_box(crl);
+        }
+        ObjType::Manifest => {
+            let mft = Manifest::decode(black_box(bytes), strict).expect("decode manifest");
+            if cert_inspect {
+                mft.cert().inspect_ee(strict).expect("inspect ee");
+            }
+            black_box(mft);
+        }
+        ObjType::Roa => {
+            let roa = Roa::decode(black_box(bytes), strict).expect("decode roa");
+            if cert_inspect {
+                roa.cert().inspect_ee(strict).expect("inspect ee");
+            }
+            black_box(roa);
+        }
+        ObjType::Aspa => {
+            let asa = Aspa::decode(black_box(bytes), strict).expect("decode aspa");
+            if cert_inspect {
+                asa.cert().inspect_ee(strict).expect("inspect ee");
+            }
+            black_box(asa);
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+struct ResultRow {
+    obj_type: String,
+    sample: String,
+    size_bytes: usize,
+    complexity: u64,
+    avg_ns_per_op: f64,
+    ops_per_sec: f64,
+}
+
+fn render_markdown(title: &str, rows: &[ResultRow]) -> String {
+    let mut out = String::new();
+    out.push_str(&format!("# {title}\n\n"));
+    out.push_str("| type | sample | size_bytes | complexity | avg ns/op | ops/s |\n");
+    out.push_str("|---|---|---:|---:|---:|---:|\n");
+    for r in rows {
+        out.push_str(&format!(
+            "| {} | {} | {} | {} | {:.2} | {:.2} |\n",
+            r.obj_type, r.sample, r.size_bytes, r.complexity, r.avg_ns_per_op, r.ops_per_sec
+        ));
+    }
+    out
+}
+
+fn render_csv(rows: &[ResultRow]) -> String {
+    let mut out = String::new();
+    out.push_str("type,sample,size_bytes,complexity,avg_ns_per_op,ops_per_sec\n");
+    for r in rows {
+        let sample = r.sample.replace('"', "\"\"");
+        out.push_str(&format!(
+            "{},{},{},{},{:.6},{:.6}\n",
+            r.obj_type,
+            format!("\"{}\"", sample),
+            r.size_bytes,
+            r.complexity,
+            r.avg_ns_per_op,
+            r.ops_per_sec
+        ));
+    }
+    out
+}
+
+fn create_parent_dirs(path: &Path) {
+    if let Some(parent) = path.parent() {
+        std::fs::create_dir_all(parent).unwrap_or_else(|e| {
+            panic!("create_dir_all {}: {e}", parent.display());
+        });
+    }
+}
+
+fn write_text_file(path: &Path, content: &str) {
+    create_parent_dirs(path);
+    std::fs::write(path, content).unwrap_or_else(|e| panic!("write {}: {e}", path.display()));
+}
+
+fn main() {
+    let cfg = parse_args();
+    let mut samples = read_samples(&cfg.dir);
+    if samples.is_empty() {
+        usage_and_exit(Some(&format!(
+            "no samples found under: {}",
+            cfg.dir.display()
+        )));
+    }
+
+    if let Some(t) = cfg.type_filter {
+        samples.retain(|s| s.obj_type == t);
+        if samples.is_empty() {
+            usage_and_exit(Some(&format!("no sample matched --type {}", t.as_str())));
+        }
+    }
+    if let Some(filter) = cfg.sample_filter.as_deref() {
+        samples.retain(|s| s.name == filter);
+        if samples.is_empty() {
+            usage_and_exit(Some(&format!("no sample matched --sample {filter}")));
+        }
+    }
+
+    println!("# Routinator baseline (rpki crate) decode benchmark (selected_der_v2)");
+    println!();
+    println!("- dir: {}", cfg.dir.display());
+    println!("- strict: {}", cfg.strict);
+    println!("- cert_inspect: {}", cfg.cert_inspect);
+    if let Some(t) = cfg.type_filter {
+        println!("- type: {}", t.as_str());
+    }
+    if let Some(s) = cfg.sample_filter.as_deref() {
+        println!("- sample: {}", s);
+    }
+    if let Some(n) = cfg.fixed_iters {
+        println!("- iters: {} (fixed)", n);
+    } else {
+        println!(
+            "- warmup: {} iters, rounds: {}, min_round: {}ms (adaptive iters, max {})",
+            cfg.warmup_iters, cfg.rounds, cfg.min_round_ms, cfg.max_adaptive_iters
+        );
+    }
+    if let Some(p) = cfg.out_csv.as_ref() {
+        println!("- out_csv: {}", p.display());
+    }
+    if let Some(p) = cfg.out_md.as_ref() {
+        println!("- out_md: {}", p.display());
+    }
+    println!();
+
+    println!("| type | sample | size_bytes | complexity | avg ns/op | ops/s |");
+    println!("|---|---|---:|---:|---:|---:|");
+
+    let mut rows: Vec<ResultRow> = Vec::with_capacity(samples.len());
+    for sample in &samples {
+        let bytes = std::fs::read(&sample.path)
+            .unwrap_or_else(|e| panic!("read {}: {e}", sample.path.display()));
+        let size_bytes = bytes.len();
+        let complexity = complexity(sample.obj_type, bytes.as_slice(), cfg.strict, cfg.cert_inspect);
+
+        for _ in 0..cfg.warmup_iters {
+            decode_profile(sample.obj_type, bytes.as_slice(), cfg.strict, cfg.cert_inspect);
+        }
+
+        let mut per_round_ns_per_op = Vec::with_capacity(cfg.rounds as usize);
+        for _round in 0..cfg.rounds {
+            let iters = if let Some(n) = cfg.fixed_iters {
+                n
+            } else {
+                choose_iters_adaptive(
+                    || decode_profile(sample.obj_type, bytes.as_slice(), cfg.strict, cfg.cert_inspect),
+                    cfg.min_round_ms,
+                    cfg.max_adaptive_iters,
+                )
+            };
+            let start = Instant::now();
+            for _ in 0..iters {
+                decode_profile(sample.obj_type, bytes.as_slice(), cfg.strict, cfg.cert_inspect);
+            }
+            let elapsed = start.elapsed();
+            let total_ns = elapsed.as_secs_f64() * 1e9;
+            per_round_ns_per_op.push(total_ns / (iters as f64));
+        }
+
+        let avg_ns = per_round_ns_per_op.iter().sum::<f64>() / (per_round_ns_per_op.len() as f64);
+        let ops_per_sec = 1e9_f64 / avg_ns;
+
+        println!(
+            "| {} | {} | {} | {} | {:.2} | {:.2} |",
+            sample.obj_type.as_str(),
+            sample.name,
+            size_bytes,
+            complexity,
+            avg_ns,
+            ops_per_sec
+        );
+
+        rows.push(ResultRow {
+            obj_type: sample.obj_type.as_str().to_string(),
+            sample: sample.name.clone(),
+            size_bytes,
+            complexity,
+            avg_ns_per_op: avg_ns,
+            ops_per_sec,
+        });
+    }
+
+    if let Some(path) = cfg.out_md.as_ref() {
+        let md = render_markdown(
+            "Routinator baseline (rpki crate) decode+inspect (selected_der_v2)",
+            &rows,
+        );
+        write_text_file(path, &md);
+        eprintln!("Wrote {}", path.display());
+    }
+    if let Some(path) = cfg.out_csv.as_ref() {
+        let csv = render_csv(&rows);
+        write_text_file(path, &csv);
+        eprintln!("Wrote {}", path.display());
+    }
+}

monitor/README.md

@@ -0,0 +1,220 @@
+# Ours RP Prometheus / Grafana Monitor
+
+本目录提供本地开发监控栈，用于采集 `rpki_artifact_metrics` 暴露的 ours RP soak 指标。
+
+## 前置条件
+
+1. Docker + Docker Compose v2；
+2. 宿主机已启动 `rpki_artifact_metrics`，并监听 Docker 网桥可访问的地址，例如 `0.0.0.0:9556`；
+3. Prometheus 容器通过 `host.docker.internal:9556` 访问宿主 sidecar。
+
+Linux Docker 下 compose 已配置：
+
+```yaml
+extra_hosts:
+  - host.docker.internal:host-gateway
+```
+
+## 启动
+
+```bash
+cd rpki_2/rpki/monitor
+docker compose up -d
+```
+
+默认镜像使用官方 Docker Hub 镜像：
+
+```text
+prom/prometheus:v2.55.1
+grafana/grafana:11.3.1
+```
+
+如需切到其它镜像源：
+
+```bash
+PROMETHEUS_IMAGE=<mirror>/prom/prometheus:v2.55.1 \
+GRAFANA_IMAGE=<mirror>/grafana/grafana:11.3.1 \
+docker compose up -d
+```
+
+默认端口：
+
+- Prometheus: <http://localhost:9090>
+- Grafana: <http://localhost:3000>
+- Grafana 默认账号密码：`admin` / `admin`
+
+如端口冲突：
+
+```bash
+PROMETHEUS_PORT=19090 GRAFANA_PORT=13000 docker compose up -d
+```
+
+Prometheus 默认保留 7 天数据；可通过 `PROMETHEUS_RETENTION` 覆盖：
+
+```bash
+PROMETHEUS_RETENTION=7d docker compose up -d
+```
+
+## 长期稳定性测试
+
+portable soak package 内置 `run_24h_soak_with_metrics.sh`，用于连续运行 ours RP、启动 metrics sidecar、启动本监控栈，并每小时生成报告：
+
+```bash
+cd /path/to/portable-soak
+SOAK_DURATION_SECS=0 \
+HOURLY_REPORT_INTERVAL_SECS=3600 \
+SOAK_RETAIN_RUNS=100 \
+CLEAN_TMP_AFTER_RUN=1 \
+PROMETHEUS_RETENTION=7d \
+STOP_MONITOR_STACK_ON_EXIT=0 \
+FEISHU_WEBHOOK_SCRIPT=/home/yuyr/.codex/skills/user/feishu-webhook/scripts/send_feishu_text.py \
+./run_24h_soak_with_metrics.sh
+```
+
+`SOAK_DURATION_SECS=0` 表示持续运行不自动停止；如需 24 小时自然停止，可设置为 `86400`，脚本会等当前 run 完成后退出，不会直接 kill 半轮验证。
+
+关键产物：
+
+- `runs/run_xxxx/`：最近 100 个 run 原始产物；
+- `hourly_reports/hour_*.md`：小时级报告；
+- `hourly_reports/hourly_summary.jsonl`：小时级结构化汇总；
+- `incident_runs/run_xxxx/`：异常 run 固化副本；
+- `logs/metrics.*`、`logs/24h-soak.*`、`logs/hourly-reporter.*`：运行日志。
+
+短周期联调可把 `SOAK_DURATION_SECS` 和 `HOURLY_REPORT_INTERVAL_SECS` 调小，并设置 `FEISHU_DRY_RUN=1` 避免真实飞书通知。
+
+## 停止
+
+```bash
+cd rpki_2/rpki/monitor
+docker compose down
+```
+
+保留数据 volume。若要清理数据：
+
+```bash
+docker compose down -v
+```
+
+## 典型本地联调命令
+
+先启动 APNIC soak 和 metrics sidecar，例如：
+
+```bash
+# soak .env 关键配置
+MAX_RUNS=-1
+RIRS=apnic
+RETAIN_RUNS=5
+INTERVAL_SECS=0
+
+# metrics sidecar
+rpki_artifact_metrics \
+  --run-root /path/to/portable-soak \
+  --listen 0.0.0.0:9556 \
+  --poll-secs 5 \
+  --instance local-apnic-continuous
+```
+
+再启动监控栈：
+
+```bash
+cd rpki_2/rpki/monitor
+docker compose up -d
+```
+
+## 验证
+
+Prometheus target：
+
+```bash
+curl -s 'http://localhost:9090/api/v1/targets' | python3 -m json.tool
+```
+
+Prometheus query：
+
+```bash
+curl -G 'http://localhost:9090/api/v1/query' \
+  --data-urlencode 'query=up{job="ours-rp-artifact-metrics"}'
+
+curl -G 'http://localhost:9090/api/v1/query' \
+  --data-urlencode 'query=ours_rp_run_completed_total{status="success"}'
+```
+
+Grafana health：
+
+```bash
+curl -s http://localhost:3000/api/health | python3 -m json.tool
+```
+
+Grafana dashboard：
+
+- 打开 <http://localhost:3000/d/ours-rp-soak-overview/ours-rp-soak-overview>
+
+## 主要指标
+
+- `ours_rp_metrics_service_up`
+- `ours_rp_run_completed_total`
+- `ours_rp_run_duration_seconds`
+- `ours_rp_run_max_rss_bytes`
+- `ours_rp_vrps{kind="total|unique"}`：`total` 为去重前 VRP 条目数，`unique` 按 `(ASN, IP Prefix, Max Length)` 去重。
+- `ours_rp_vaps`
+- `ours_rp_publication_points`
+- `ours_rp_repo_sync_phase_count`
+- `ours_rp_large_publication_points{object_count_gt="10|50|100|..."}`
+- `ours_rp_cir_objects`
+- `ours_rp_ccr_state_items`
+
+## Inter-RP 持续对比监控
+
+`rpki_inter_rp_metrics` 用于汇总三方 RP 的最新产物：
+
+- ours RP：读取当前 portable soak 的 `runs/run_xxxx/run-summary.json`、`result.ccr`、CSV 产物；
+- Routinator：读取远端200同步来的 `routinator/latest/run-meta.json`、`vrps.csv`、`vaps.csv`；
+- rpki-client 9.8：读取远端200同步来的 `rpki-client/latest/run-meta.json`、`vrps.csv`、`vaps.csv`、`result.ccr`。
+
+远端231 启动 sidecar 示例：
+
+```bash
+rpki_inter_rp_metrics \
+  --ours-run-root /root/rpki_20260608_2_feature062_24h_20260608T075547Z/portable-soak \
+  --peer-root /root/inter-rp-aggregator/synced-from-200 \
+  --listen 0.0.0.0:9557 \
+  --poll-secs 30 \
+  --instance remote231-inter-rp
+```
+
+Prometheus 已新增 `ours-rp-inter-rp-metrics` scrape job，默认访问 `host.docker.internal:9557`。
+
+远端200 runner 与远端231同步脚本位于：
+
+```text
+scripts/inter_rp/run_remote200_rp_loops.sh
+scripts/inter_rp/run_single_rp_with_rss.sh
+scripts/inter_rp/sync_remote200_to_231.sh
+scripts/inter_rp/run_inter_rp_metrics_sidecar.sh
+scripts/inter_rp/inter-rp.env.example
+```
+
+如需从本机独立开关远端200上的 Routinator 或 rpki-client，使用：
+
+```bash
+scripts/inter_rp/control_remote200_rp.sh status all
+scripts/inter_rp/control_remote200_rp.sh stop routinator
+scripts/inter_rp/control_remote200_rp.sh start routinator
+scripts/inter_rp/control_remote200_rp.sh restart rpki-client
+```
+
+默认远端为 `root@43.110.128.200`，可通过 `REMOTE_HOST=...` 覆盖；脚本只管理指定 RP 的 loop 和当前子进程，不会自动影响另一个 RP。
+
+关键指标：
+
+- `inter_rp_run_wall_seconds{rp="ours-rp|routinator|rpki-client"}`
+- `inter_rp_run_max_rss_bytes{rp="...",kind="aggregate_peak"}`
+- `inter_rp_vrps{rp="..."}`：按 `(ASN, IP Prefix, Max Length)` 去重。
+- `inter_rp_vaps{rp="..."}`：按 `(Customer ASN, Providers)` 去重，Routinator 使用 `--enable-aspa` JSON 输出转换，rpki-client 使用 `-j` JSON 输出转换。
+- `inter_rp_ccr_digest_match{left="ours-rp",right="rpki-client",state="overall|mfts|vrps|vaps|tas|rks"}`
+- `inter_rp_sync_age_seconds`
+
+Grafana dashboard：
+
+- <http://localhost:3000/d/ours-rp-inter-rp/ours-rp-inter-rp>

monitor/docker-compose.yml

@@ -0,0 +1,38 @@
+services:
+  prometheus:
+    image: ${PROMETHEUS_IMAGE:-prom/prometheus:v2.55.1}
+    container_name: ours-rp-prometheus
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-7d}
+      - --web.enable-lifecycle
+    extra_hosts:
+      - host.docker.internal:host-gateway
+    ports:
+      - "${PROMETHEUS_PORT:-9090}:9090"
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    restart: unless-stopped
+
+  grafana:
+    image: ${GRAFANA_IMAGE:-grafana/grafana:11.3.1}
+    container_name: ours-rp-grafana
+    depends_on:
+      - prometheus
+    ports:
+      - "${GRAFANA_PORT:-3000}:3000"
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+    volumes:
+      - grafana-data:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+    restart: unless-stopped
+
+volumes:
+  prometheus-data:
+  grafana-data:

monitor/grafana/dashboards/ours-rp-inter-rp.json

@@ -0,0 +1,505 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "inter_rp_service_last_reload_success",
+          "legendFormat": "reload",
+          "refId": "A"
+        }
+      ],
+      "title": "Metrics Reload OK",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "inter_rp_sync_age_seconds",
+          "legendFormat": "sync age",
+          "refId": "A"
+        }
+      ],
+      "title": "Remote200 Sync Age",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "inter_rp_parse_errors",
+          "legendFormat": "errors",
+          "refId": "A"
+        }
+      ],
+      "title": "Parse Errors",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "inter_rp_ccr_digest_match{state=\"overall\"}",
+          "legendFormat": "overall",
+          "refId": "A"
+        }
+      ],
+      "title": "Ours vs rpki-client CCR Match",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_run_wall_seconds",
+          "legendFormat": "{{rp}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Wall Time by RP",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_run_max_rss_bytes{kind=\"aggregate_peak\"}",
+          "legendFormat": "{{rp}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Max RSS Aggregate Peak by RP",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0,
+          "min": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 12
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_vrps",
+          "legendFormat": "{{rp}}",
+          "refId": "A"
+        }
+      ],
+      "title": "VRPs by RP (unique ASN/Prefix/MaxLen)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0,
+          "min": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 12
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_vaps",
+          "legendFormat": "{{rp}}",
+          "refId": "A"
+        }
+      ],
+      "title": "VAPs / ASPAs by RP (unique Customer/Providers)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 20
+      },
+      "id": 9,
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_ccr_digest_match{left=\"ours-rp\",right=\"rpki-client\"}",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{state}}",
+          "refId": "A"
+        }
+      ],
+      "title": "CCR Digest Match States",
+      "type": "table"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 20
+      },
+      "id": 10,
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_vrps_diff",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "vrps {{left}}-{{right}}",
+          "refId": "A"
+        },
+        {
+          "expr": "inter_rp_vaps_diff",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "vaps {{left}}-{{right}}",
+          "refId": "B"
+        }
+      ],
+      "title": "Output Count Diffs (VRP/VAP unique)",
+      "type": "table"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 28
+      },
+      "id": 11,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_artifact_age_seconds",
+          "legendFormat": "{{rp}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Artifact Age by RP",
+      "type": "timeseries"
+    }
+  ],
+  "preload": false,
+  "refresh": "10s",
+  "schemaVersion": 40,
+  "tags": [
+    "rpki",
+    "inter-rp"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Ours RP Inter-RP",
+  "uid": "ours-rp-inter-rp",
+  "version": 1
+}

monitor/grafana/dashboards/ours-rp-repo-sync.json

@@ -0,0 +1,761 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_publication_points",
+          "legendFormat": "publication points",
+          "refId": "A"
+        }
+      ],
+      "title": "Publication Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_ok\"}",
+          "legendFormat": "rrdp ok",
+          "refId": "A"
+        }
+      ],
+      "title": "RRDP OK Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}",
+          "legendFormat": "fallback",
+          "refId": "A"
+        }
+      ],
+      "title": "Rsync Fallback Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache",
+          "refId": "A"
+        }
+      ],
+      "title": "Failed No Cache Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 4
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"repo_sync_total\"}",
+          "legendFormat": "repo sync total",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"rrdp_download_total\"}",
+          "legendFormat": "rrdp download",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"rsync_download_total\"}",
+          "legendFormat": "rsync download",
+          "refId": "C"
+        }
+      ],
+      "title": "Repo Sync Download Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 12,
+        "w": 12,
+        "h": 8
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count",
+          "legendFormat": "{{phase}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repo Sync Phase Counts",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 12,
+        "w": 12,
+        "h": 8
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}",
+          "legendFormat": "rrdp failed, rsync ok",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_failed\"}",
+          "legendFormat": "rrdp failed, rsync failed",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache",
+          "refId": "C"
+        },
+        {
+          "expr": "ours_rp_tree_instances{state=\"failed\"}",
+          "legendFormat": "tree failed",
+          "refId": "D"
+        }
+      ],
+      "title": "Repo Failure / Fallback Counts",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 20,
+        "w": 12,
+        "h": 8
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_ok\"}",
+          "legendFormat": "rsync fallback duration",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_failed\"}",
+          "legendFormat": "failed duration",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_repo_terminal_state_duration_seconds_total{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache duration",
+          "refId": "C"
+        }
+      ],
+      "title": "Repo Failure / Fallback Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 29,
+        "w": 12,
+        "h": 9
+      },
+      "id": 9,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_rrdp_rsync_failed_repository_duration_seconds",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "RRDP + Rsync Failed Repositories",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "terminal_state": true,
+              "rank": true,
+              "transport": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "phase": 2,
+              "uri": 3,
+              "Value": 4
+            },
+            "renameByName": {
+              "Value": "duration"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 29,
+        "w": 12,
+        "h": 9
+      },
+      "id": 11,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "topk(20, ours_rp_top_repository_sync_duration_seconds)",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Top 20 Repositories by Sync Duration",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "terminal_state": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "phase": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "rank": 2,
+              "transport": 3,
+              "uri": 4,
+              "Value": 5
+            },
+            "renameByName": {
+              "Value": "value"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 38,
+        "w": 24,
+        "h": 9
+      },
+      "id": 10,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "topk(20, ours_rp_top_publication_point_object_count)",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Top Publication Points by Objects",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "phase": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "rank": 2,
+              "terminal_state": 3,
+              "transport": 4,
+              "uri": 5,
+              "Value": 6
+            },
+            "renameByName": {}
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository sync success in the latest successful run; 1 means successful, 0 means failed or failed_no_cache.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bool"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 47
+      },
+      "id": 12,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_sync_success",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Sync Success by Repo",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository total sync duration aggregated from publication point repo_sync_duration_ms.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 55
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_sync_duration_seconds{stat=\"sum\"}",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Sync Duration by Repo",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository downloaded bytes attributed from report.json downloads events.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 63
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_download_bytes",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Download Bytes by Repo",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": [
+    "ours-rp",
+    "rpki",
+    "soak",
+    "repo-sync"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Ours RP Repo Sync",
+  "uid": "ours-rp-repo-sync",
+  "version": 3,
+  "weekStart": ""
+}

monitor/grafana/dashboards/ours-rp-soak-overview.json

@@ -0,0 +1,639 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_cir_trust_anchors",
+          "legendFormat": "RIRs",
+          "refId": "A"
+        }
+      ],
+      "title": "Current Run RIRs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 6,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_run_duration_seconds",
+          "legendFormat": "wall",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest Wall Time",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_run_max_rss_bytes",
+          "legendFormat": "rss",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest Max RSS",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 18,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_publication_points",
+          "legendFormat": "publication points",
+          "refId": "A"
+        }
+      ],
+      "title": "Publication Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 8,
+        "w": 12,
+        "h": 8
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_duration_seconds",
+          "legendFormat": "wall",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_stage_duration_seconds{stage=\"validation\"}",
+          "legendFormat": "validation",
+          "refId": "B"
+        }
+      ],
+      "title": "Run / Validation Duration",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 8,
+        "w": 12,
+        "h": 8
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_vrps{kind=\"total\"}",
+          "legendFormat": "VRPs raw",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_vrps{kind=\"unique\"}",
+          "legendFormat": "VRPs unique",
+          "refId": "D"
+        },
+        {
+          "expr": "ours_rp_vaps",
+          "legendFormat": "VAPs",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_cir_objects",
+          "legendFormat": "CIR objects",
+          "refId": "C"
+        }
+      ],
+      "title": "Output and Input Sizes",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 16,
+        "w": 12,
+        "h": 8
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_large_publication_points",
+          "legendFormat": "> {{object_count_gt}} objects",
+          "refId": "A"
+        }
+      ],
+      "title": "Large Publication Points by Object Count",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 9,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_run_sequence",
+          "legendFormat": "seq",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest Run Sequence",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 6,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 10,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_run_success",
+          "legendFormat": "success",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest Run Success",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 11,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_vrps{kind=\"total\"}",
+          "legendFormat": "VRPs raw",
+          "refId": "A"
+        }
+      ],
+      "title": "VRPs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 18,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 12,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_vaps",
+          "legendFormat": "VAPs",
+          "refId": "A"
+        }
+      ],
+      "title": "VAPs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 16,
+        "w": 12,
+        "h": 8
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"validation\"}",
+          "legendFormat": "validation",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"report_write\"}",
+          "legendFormat": "report write",
+          "refId": "E"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"ccr_write\"}",
+          "legendFormat": "ccr write",
+          "refId": "F"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"cir_write\"}",
+          "legendFormat": "cir write",
+          "refId": "G"
+        }
+      ],
+      "title": "Output Stage Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "decimals": 2
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 24,
+        "w": 24,
+        "h": 8
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_max_rss_bytes",
+          "legendFormat": "Max RSS",
+          "refId": "A"
+        }
+      ],
+      "title": "Max RSS Over Time",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": [
+    "ours-rp",
+    "rpki",
+    "soak"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Ours RP Soak Overview",
+  "uid": "ours-rp-soak-overview",
+  "version": 4,
+  "weekStart": ""
+}

monitor/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: ours-rp
+    orgId: 1
+    folder: Ours RP
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards

monitor/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    uid: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: true

monitor/prometheus/prometheus.yml

@@ -0,0 +1,20 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+
+scrape_configs:
+  - job_name: ours-rp-artifact-metrics
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - host.docker.internal:9556
+        labels:
+          rp: ours-rp
+          source: artifact-sidecar
+  - job_name: ours-rp-inter-rp-metrics
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - host.docker.internal:9557
+        labels:
+          source: inter-rp-sidecar

scripts/benchmark/README.md

@@ -0,0 +1,70 @@
+# RPKI Benchmarks (Stage2, selected_der_v2)
+
+This directory contains a reproducible, one-click benchmark to measure **decode + profile validate**
+performance for all supported object types and compare **OURS** against the **Routinator baseline**
+(`rpki` crate `=0.19.1` with `repository` feature).
+
+## What it measures
+
+Dataset:
+
+- Fixtures: `rpki/tests/benchmark/selected_der_v2/`
+- Objects: `cer`, `crl`, `manifest` (`.mft`), `roa`, `aspa` (`.asa`)
+- Samples: 10 quantiles per type (`min/p01/p10/p25/p50/p75/p90/p95/p99/max`) → 50 files total
+
+Metrics:
+
+- **decode+validate**: `decode_der` (parse + profile validate) for each object file
+- **landing** (OURS only): `PackFile::from_bytes_compute_sha256` + CBOR encode + `RocksDB put_raw`
+- **compare**: ratio `ours_ns/op ÷ rout_ns/op` for decode+validate
+
+## Default benchmark settings
+
+Both OURS and Routinator baseline use the same run settings:
+
+- warmup: `10` iterations
+- rounds: `3`
+- adaptive loop target: `min_round_ms=200` (with an internal max of `1_000_000` iters)
+- strict DER: `true` (baseline)
+- cert inspect: `false` (baseline)
+
+You can override the settings via environment variables in the runner script:
+
+- `BENCH_WARMUP_ITERS` (default `10`)
+- `BENCH_ROUNDS` (default `3`)
+- `BENCH_MIN_ROUND_MS` (default `200`)
+
+## One-click run (OURS + Routinator compare)
+
+From the `rpki/` crate directory:
+
+```bash
+./scripts/benchmark/run_stage2_selected_der_v2_release.sh
+```
+
+Outputs are written under:
+
+- `rpki/target/bench/`
+  - OURS decode+validate: `stage2_selected_der_v2_decode_release_<TS>.{md,csv}`
+  - OURS landing: `stage2_selected_der_v2_landing_release_<TS>.{md,csv}`
+  - Routinator: `stage2_selected_der_v2_routinator_decode_release_<TS>.{md,csv}`
+  - Compare: `stage2_selected_der_v2_compare_ours_vs_routinator_decode_release_<TS>.{md,csv}`
+  - Summary: `stage2_selected_der_v2_compare_summary_<TS>.md`
+
+### Why decode and landing are separated
+
+The underlying benchmark can run in `BENCH_MODE=both`, but the **landing** part writes to RocksDB
+and may trigger background work (e.g., compactions) that can **skew subsequent decode timings**.
+For a fair OURS-vs-Routinator comparison, the runner script:
+
+- runs `BENCH_MODE=decode_validate` for comparison, and
+- runs `BENCH_MODE=landing` separately for landing-only numbers.
+
+## Notes
+
+- The Routinator baseline benchmark is implemented in-repo under:
+  - `rpki/benchmark/routinator_object_bench/`
+  - It pins `rpki = "=0.19.1"` in its `Cargo.toml`.
+- This benchmark is implemented as an `#[ignore]` integration test:
+  - `rpki/tests/bench_stage2_decode_profile_selected_der_v2.rs`
+  - The runner script invokes it with `cargo test --release ... -- --ignored --nocapture`.

scripts/benchmark/run_stage2_selected_der_v2_release.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Stage2 (selected_der_v2) decode+profile validate benchmark.
+# Runs:
+#  1) OURS decode+validate benchmark and writes MD/CSV.
+#  2) OURS landing benchmark and writes MD/CSV.
+#  3) Routinator baseline decode benchmark (rpki crate =0.19.1).
+#  4) Produces a joined compare CSV/MD and a short geomean summary.
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+OUT_DIR="$ROOT_DIR/target/bench"
+mkdir -p "$OUT_DIR"
+
+TS="$(date -u +%Y%m%dT%H%M%SZ)"
+
+WARMUP_ITERS="${BENCH_WARMUP_ITERS:-10}"
+ROUNDS="${BENCH_ROUNDS:-3}"
+MIN_ROUND_MS="${BENCH_MIN_ROUND_MS:-200}"
+
+OURS_MD="$OUT_DIR/stage2_selected_der_v2_decode_release_${TS}.md"
+OURS_CSV="$OUT_DIR/stage2_selected_der_v2_decode_release_${TS}.csv"
+
+OURS_LANDING_MD="$OUT_DIR/stage2_selected_der_v2_landing_release_${TS}.md"
+OURS_LANDING_CSV="$OUT_DIR/stage2_selected_der_v2_landing_release_${TS}.csv"
+
+ROUT_MD="$OUT_DIR/stage2_selected_der_v2_routinator_decode_release_${TS}.md"
+ROUT_CSV="$OUT_DIR/stage2_selected_der_v2_routinator_decode_release_${TS}.csv"
+
+COMPARE_MD="$OUT_DIR/stage2_selected_der_v2_compare_ours_vs_routinator_decode_release_${TS}.md"
+COMPARE_CSV="$OUT_DIR/stage2_selected_der_v2_compare_ours_vs_routinator_decode_release_${TS}.csv"
+
+SUMMARY_MD="$OUT_DIR/stage2_selected_der_v2_compare_summary_${TS}.md"
+
+echo "[1/4] OURS: decode+validate benchmark (release)..." >&2
+BENCH_MODE="decode_validate" \
+BENCH_WARMUP_ITERS="$WARMUP_ITERS" \
+BENCH_ROUNDS="$ROUNDS" \
+BENCH_MIN_ROUND_MS="$MIN_ROUND_MS" \
+BENCH_OUT_MD="$OURS_MD" \
+BENCH_OUT_CSV="$OURS_CSV" \
+  cargo test --release --test bench_stage2_decode_profile_selected_der_v2 -- --ignored --nocapture >/dev/null
+
+echo "[2/4] OURS: landing benchmark (release)..." >&2
+BENCH_MODE="landing" \
+BENCH_WARMUP_ITERS="$WARMUP_ITERS" \
+BENCH_ROUNDS="$ROUNDS" \
+BENCH_MIN_ROUND_MS="$MIN_ROUND_MS" \
+BENCH_OUT_MD_LANDING="$OURS_LANDING_MD" \
+BENCH_OUT_CSV_LANDING="$OURS_LANDING_CSV" \
+  cargo test --release --test bench_stage2_decode_profile_selected_der_v2 -- --ignored --nocapture >/dev/null
+
+echo "[3/4] Routinator baseline + compare join..." >&2
+OURS_CSV="$OURS_CSV" \
+ROUT_CSV="$ROUT_CSV" \
+ROUT_MD="$ROUT_MD" \
+COMPARE_CSV="$COMPARE_CSV" \
+COMPARE_MD="$COMPARE_MD" \
+WARMUP_ITERS="$WARMUP_ITERS" \
+ROUNDS="$ROUNDS" \
+MIN_ROUND_MS="$MIN_ROUND_MS" \
+  scripts/stage2_perf_compare_m4.sh >/dev/null
+
+echo "[4/4] Summary (geomean ratios)..." >&2
+python3 - "$COMPARE_CSV" "$SUMMARY_MD" <<'PY'
+import csv
+import math
+import sys
+from pathlib import Path
+from datetime import datetime, timezone
+
+in_csv = Path(sys.argv[1])
+out_md = Path(sys.argv[2])
+
+rows = list(csv.DictReader(in_csv.open(newline="")))
+ratios = {}
+for r in rows:
+    ratios.setdefault(r["type"], []).append(float(r["ratio_ours_over_rout"]))
+
+def geomean(vals):
+    return math.exp(sum(math.log(v) for v in vals) / len(vals))
+
+def p50(vals):
+    v = sorted(vals)
+    n = len(v)
+    if n % 2 == 1:
+        return v[n // 2]
+    return (v[n // 2 - 1] + v[n // 2]) / 2.0
+
+all_vals = [float(r["ratio_ours_over_rout"]) for r in rows]
+types = ["all"] + sorted(ratios.keys())
+
+now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+lines = []
+lines.append("# Stage2 selected_der_v2 compare summary (release)\n\n")
+lines.append(f"- recorded_at_utc: `{now}`\n")
+lines.append(f"- inputs_csv: `{in_csv}`\n\n")
+lines.append("| type | n | min | p50 | geomean | max | >1 count |\n")
+lines.append("|---|---:|---:|---:|---:|---:|---:|\n")
+
+for t in types:
+    vals = all_vals if t == "all" else ratios[t]
+    vals_sorted = sorted(vals)
+    lines.append(
+        f"| {t} | {len(vals_sorted)} | {vals_sorted[0]:.4f} | {p50(vals_sorted):.4f} | "
+        f"{geomean(vals_sorted):.4f} | {vals_sorted[-1]:.4f} | {sum(1 for v in vals_sorted if v>1.0)} |\n"
+    )
+
+out_md.write_text("".join(lines), encoding="utf-8")
+print(out_md)
+PY
+
+echo "Done." >&2
+echo "- OURS decode MD:   $OURS_MD" >&2
+echo "- OURS decode CSV:  $OURS_CSV" >&2
+echo "- OURS landing MD:  $OURS_LANDING_MD" >&2
+echo "- OURS landing CSV: $OURS_LANDING_CSV" >&2
+echo "- Routinator:       $ROUT_MD" >&2
+echo "- Compare MD:       $COMPARE_MD" >&2
+echo "- Compare CSV:      $COMPARE_CSV" >&2
+echo "- Summary MD:       $SUMMARY_MD" >&2

scripts/cir/README.md

@@ -0,0 +1,56 @@
+# CIR Scripts
+
+## `cir-rsync-wrapper`
+
+一个用于 CIR 黑盒 replay 的 rsync wrapper。
+
+### 环境变量
+
+- `REAL_RSYNC_BIN`
+  - 真实 rsync 二进制路径
+  - 默认优先 `/usr/bin/rsync`
+- `CIR_MIRROR_ROOT`
+  - 本地镜像树根目录
+  - 当命令行中出现 `rsync://...` source 时必需
+
+### 语义
+
+- 仅改写 `rsync://host/path` 类型参数
+- 其它参数原样透传给真实 rsync
+- 改写目标：
+  - `rsync://example.net/repo/a.roa`
+  - →
+  - `<CIR_MIRROR_ROOT>/example.net/repo/a.roa`
+
+### 兼容目标
+
+- Routinator `--rsync-command`
+- `rpki-client -e rsync_prog`
+
+## 其它脚本
+
+- `run_cir_replay_ours.sh`
+- `run_cir_replay_routinator.sh`
+- `run_cir_replay_rpki_client.sh`
+- `run_cir_replay_matrix.sh`
+
+## `cir-local-link-sync.py`
+
+当 `CIR_LOCAL_LINK_MODE=1` 且 wrapper 检测到 source 已经被改写为本地 mirror 路径时，
+wrapper 不再调用真实 `rsync`，而是调用这个 helper 完成：
+
+- `hardlink` 优先的本地树同步
+- 失败时回退到 copy
+- 支持 `--delete`
+
+`run_cir_replay_matrix.sh` 会顺序执行：
+
+- `ours`
+- Routinator
+- `rpki-client`
+
+并汇总生成：
+
+- `summary.json`
+- `summary.md`
+- `detail.md`

scripts/cir/cir-local-link-sync.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+import argparse
+import errno
+import os
+import shutil
+from pathlib import Path
+
+
+def _same_inode(src: Path, dst: Path) -> bool:
+    try:
+        src_stat = src.stat()
+        dst_stat = dst.stat()
+    except FileNotFoundError:
+        return False
+    return (src_stat.st_dev, src_stat.st_ino) == (dst_stat.st_dev, dst_stat.st_ino)
+
+
+def _remove_path(path: Path) -> None:
+    if not path.exists() and not path.is_symlink():
+        return
+    if path.is_dir() and not path.is_symlink():
+        shutil.rmtree(path)
+    else:
+        path.unlink()
+
+
+def _prune_empty_dirs(root: Path) -> None:
+    if not root.exists():
+        return
+    for path in sorted((p for p in root.rglob("*") if p.is_dir()), key=lambda p: len(p.parts), reverse=True):
+        try:
+            path.rmdir()
+        except OSError:
+            pass
+
+
+def _link_or_copy(src: Path, dst: Path) -> str:
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    if dst.exists() or dst.is_symlink():
+        if _same_inode(src, dst):
+            return "reused"
+        _remove_path(dst)
+    try:
+        os.link(src, dst)
+        return "linked"
+    except OSError as err:
+        if err.errno not in (errno.EXDEV, errno.EPERM, errno.EMLINK, errno.ENOTSUP, errno.EACCES):
+            raise
+    shutil.copy2(src, dst)
+    return "copied"
+
+
+def _file_map(src_arg: str, dest_arg: str) -> tuple[Path, dict[str, Path]]:
+    src = Path(src_arg.rstrip(os.sep))
+    if not src.exists():
+        raise FileNotFoundError(src)
+    mapping: dict[str, Path] = {}
+    if src.is_dir():
+        copy_contents = src_arg.endswith(os.sep)
+        if copy_contents:
+            root = src
+            for path in root.rglob("*"):
+                if path.is_file():
+                    mapping[path.relative_to(root).as_posix()] = path
+        else:
+            root = src
+            base = src.name
+            for path in root.rglob("*"):
+                if path.is_file():
+                    rel = Path(base) / path.relative_to(root)
+                    mapping[rel.as_posix()] = path
+    else:
+        dest_path = Path(dest_arg)
+        if dest_arg.endswith(os.sep) or dest_path.is_dir():
+            mapping[src.name] = src
+        else:
+            mapping[dest_path.name] = src
+    return Path(dest_arg), mapping
+
+
+def sync_local_tree(src_arg: str, dst_arg: str, delete: bool) -> dict[str, int]:
+    dst_root, mapping = _file_map(src_arg, dst_arg)
+    dst_root.mkdir(parents=True, exist_ok=True)
+
+    expected = {dst_root / rel for rel in mapping.keys()}
+
+    deleted = 0
+    if delete and dst_root.exists():
+        for path in sorted(dst_root.rglob("*"), key=lambda p: len(p.parts), reverse=True):
+            if path.is_dir():
+                continue
+            if path not in expected:
+                _remove_path(path)
+                deleted += 1
+        _prune_empty_dirs(dst_root)
+
+    linked = 0
+    copied = 0
+    reused = 0
+    for rel, src in mapping.items():
+        dst = dst_root / rel
+        result = _link_or_copy(src, dst)
+        if result == "linked":
+            linked += 1
+        elif result == "copied":
+            copied += 1
+        else:
+            reused += 1
+
+    return {
+        "files": len(mapping),
+        "linked": linked,
+        "copied": copied,
+        "reused": reused,
+        "deleted": deleted,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Sync a local CIR mirror tree using hardlinks when possible.")
+    parser.add_argument("--delete", action="store_true", help="Delete target files not present in source")
+    parser.add_argument("source")
+    parser.add_argument("dest")
+    args = parser.parse_args()
+
+    summary = sync_local_tree(args.source, args.dest, args.delete)
+    print(
+        "local-link-sync files={files} linked={linked} copied={copied} reused={reused} deleted={deleted}".format(
+            **summary
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/cir/cir-rsync-wrapper

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+import os
+import shutil
+import sys
+from pathlib import Path
+from urllib.parse import urlparse
+
+
+def real_rsync_bin() -> str:
+    env = os.environ.get("REAL_RSYNC_BIN")
+    if env:
+        return env
+    default = "/usr/bin/rsync"
+    if Path(default).exists():
+        return default
+    found = shutil.which("rsync")
+    if found:
+        return found
+    raise SystemExit("cir-rsync-wrapper: REAL_RSYNC_BIN is not set and rsync was not found")
+
+
+def rewrite_arg(arg: str, mirror_root: str | None) -> str:
+    if not arg.startswith("rsync://"):
+        return arg
+    if not mirror_root:
+        raise SystemExit(
+            "cir-rsync-wrapper: CIR_MIRROR_ROOT is required when an rsync:// source is present"
+        )
+    parsed = urlparse(arg)
+    if parsed.scheme != "rsync" or not parsed.hostname:
+        raise SystemExit(f"cir-rsync-wrapper: invalid rsync URI: {arg}")
+    path = parsed.path.lstrip("/")
+    local = Path(mirror_root).resolve() / parsed.hostname
+    if path:
+        local = local / path
+    local_str = str(local)
+    if local.exists() and local.is_dir() and not local_str.endswith("/"):
+        local_str += "/"
+    elif arg.endswith("/") and not local_str.endswith("/"):
+        local_str += "/"
+    return local_str
+
+
+def filter_args(args: list[str]) -> list[str]:
+    mirror_root = os.environ.get("CIR_MIRROR_ROOT")
+    rewritten_any = any(arg.startswith("rsync://") for arg in args)
+    out: list[str] = []
+    i = 0
+    while i < len(args):
+        arg = args[i]
+        if rewritten_any:
+            if arg == "--address":
+                i += 2
+                continue
+            if arg.startswith("--address="):
+                i += 1
+                continue
+            if arg == "--contimeout":
+                i += 2
+                continue
+            if arg.startswith("--contimeout="):
+                i += 1
+                continue
+        out.append(rewrite_arg(arg, mirror_root))
+        i += 1
+    return out
+
+
+def local_link_mode_enabled() -> bool:
+    value = os.environ.get("CIR_LOCAL_LINK_MODE", "")
+    return value.lower() in {"1", "true", "yes", "on"}
+
+
+def extract_source_and_dest(args: list[str]) -> tuple[str, str]:
+    expects_value = {
+        "--timeout",
+        "--min-size",
+        "--max-size",
+        "--include",
+        "--exclude",
+        "--compare-dest",
+    }
+    positionals: list[str] = []
+    i = 0
+    while i < len(args):
+        arg = args[i]
+        if arg in expects_value:
+            i += 2
+            continue
+        if any(arg.startswith(prefix + "=") for prefix in expects_value):
+            i += 1
+            continue
+        if arg.startswith("-"):
+            i += 1
+            continue
+        positionals.append(arg)
+        i += 1
+    if len(positionals) < 2:
+        raise SystemExit("cir-rsync-wrapper: expected source and destination arguments")
+    return positionals[-2], positionals[-1]
+
+
+def maybe_exec_local_link_sync(args: list[str], rewritten_any: bool) -> None:
+    if not rewritten_any or not local_link_mode_enabled():
+        return
+    source, dest = extract_source_and_dest(args)
+    if source.startswith("rsync://"):
+        raise SystemExit("cir-rsync-wrapper: expected rewritten local source for CIR_LOCAL_LINK_MODE")
+    helper = Path(__file__).with_name("cir-local-link-sync.py")
+    cmd = [sys.executable, str(helper)]
+    if "--delete" in args:
+        cmd.append("--delete")
+    cmd.extend([source, dest])
+    os.execv(sys.executable, cmd)
+
+
+def main() -> int:
+    args = sys.argv[1:]
+    rewritten_any = any(arg.startswith("rsync://") for arg in args)
+    rewritten = filter_args(args)
+    maybe_exec_local_link_sync(rewritten, rewritten_any)
+    os.execv(real_rsync_bin(), [real_rsync_bin(), *rewritten])
+    return 127
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/cir/fetch_cir_sequence_from_remote.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/fetch_cir_sequence_from_remote.sh \
+    --ssh-target <user@host> \
+    --remote-path <path> \
+    --local-path <path>
+EOF
+}
+
+SSH_TARGET=""
+REMOTE_PATH=""
+LOCAL_PATH=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --remote-path) REMOTE_PATH="$2"; shift 2 ;;
+    --local-path) LOCAL_PATH="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$SSH_TARGET" && -n "$REMOTE_PATH" && -n "$LOCAL_PATH" ]] || { usage >&2; exit 2; }
+
+mkdir -p "$(dirname "$LOCAL_PATH")"
+rsync -a "$SSH_TARGET:$REMOTE_PATH/" "$LOCAL_PATH/"
+echo "done: $LOCAL_PATH"

scripts/cir/json_to_vaps_csv.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+from pathlib import Path
+
+
+def normalize_asn(value: str | int) -> str:
+    text = str(value).strip().upper()
+    if text.startswith("AS"):
+        text = text[2:]
+    return f"AS{int(text)}"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input", required=True, type=Path)
+    parser.add_argument("--csv-out", required=True, type=Path)
+    args = parser.parse_args()
+
+    obj = json.loads(args.input.read_text(encoding="utf-8"))
+    rows: list[tuple[str, str, str]] = []
+    for aspa in obj.get("aspas", []):
+        providers = sorted(
+            {normalize_asn(item) for item in aspa.get("providers", [])},
+            key=lambda s: int(s[2:]),
+        )
+        rows.append(
+            (
+                normalize_asn(aspa["customer"]),
+                ";".join(providers),
+                str(aspa.get("ta", "")).strip().lower(),
+            )
+        )
+    rows.sort(key=lambda row: (int(row[0][2:]), row[1], row[2]))
+
+    args.csv_out.parent.mkdir(parents=True, exist_ok=True)
+    with args.csv_out.open("w", encoding="utf-8", newline="") as fh:
+        writer = csv.writer(fh)
+        writer.writerow(["Customer ASN", "Providers", "Trust Anchor"])
+        writer.writerows(rows)
+    print(args.csv_out)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/cir/run_cir_drop_sequence.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_drop_sequence.sh \
+    --sequence-root <path> \
+    [--drop-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SEQUENCE_ROOT=""
+DROP_BIN="${DROP_BIN:-$ROOT_DIR/target/release/cir_drop_report}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --sequence-root) SEQUENCE_ROOT="$2"; shift 2 ;;
+    --drop-bin) DROP_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$SEQUENCE_ROOT" ]] || { usage >&2; exit 2; }
+
+python3 - <<'PY' "$SEQUENCE_ROOT" "$DROP_BIN"
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+sequence_root = Path(sys.argv[1]).resolve()
+drop_bin = sys.argv[2]
+sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
+repo_bytes_db = sequence_root / sequence["repoBytesDbPath"]
+
+summaries = []
+for step in sequence["steps"]:
+    step_id = step["stepId"]
+    out_dir = sequence_root / "drop" / step_id
+    out_dir.mkdir(parents=True, exist_ok=True)
+    cmd = [
+        drop_bin,
+        "--cir",
+        str(sequence_root / step["cirPath"]),
+        "--ccr",
+        str(sequence_root / step["ccrPath"]),
+        "--report-json",
+        str(sequence_root / step["reportPath"]),
+        "--json-out",
+        str(out_dir / "drop.json"),
+        "--md-out",
+        str(out_dir / "drop.md"),
+    ]
+    cmd.extend(["--repo-bytes-db", str(repo_bytes_db)])
+    proc = subprocess.run(cmd, capture_output=True, text=True)
+    if proc.returncode != 0:
+        raise SystemExit(
+            f"drop report failed for {step_id}: stdout={proc.stdout} stderr={proc.stderr}"
+        )
+    result = json.loads((out_dir / "drop.json").read_text(encoding="utf-8"))
+    summaries.append(
+        {
+            "stepId": step_id,
+            "droppedVrpCount": result["summary"]["droppedVrpCount"],
+            "droppedObjectCount": result["summary"]["droppedObjectCount"],
+            "reportPath": str(out_dir / "drop.json"),
+        }
+    )
+
+summary = {"version": 1, "steps": summaries}
+(sequence_root / "drop-summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
+PY
+
+echo "done: $SEQUENCE_ROOT"

scripts/cir/run_cir_record_full_delta.sh

@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_record_full_delta.sh \
+    --out-dir <path> \
+    --tal-path <path> \
+    --ta-path <path> \
+    --cir-tal-uri <url> \
+    --payload-replay-archive <path> \
+    --payload-replay-locks <path> \
+    --payload-base-archive <path> \
+    --payload-base-locks <path> \
+    --payload-delta-archive <path> \
+    --payload-delta-locks <path> \
+    [--base-validation-time <rfc3339>] \
+    [--delta-validation-time <rfc3339>] \
+    [--max-depth <n>] \
+    [--max-instances <n>] \
+    [--rpki-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+OUT_DIR=""
+TAL_PATH=""
+TA_PATH=""
+CIR_TAL_URI=""
+PAYLOAD_REPLAY_ARCHIVE=""
+PAYLOAD_REPLAY_LOCKS=""
+PAYLOAD_BASE_ARCHIVE=""
+PAYLOAD_BASE_LOCKS=""
+PAYLOAD_DELTA_ARCHIVE=""
+PAYLOAD_DELTA_LOCKS=""
+BASE_VALIDATION_TIME=""
+DELTA_VALIDATION_TIME=""
+MAX_DEPTH=0
+MAX_INSTANCES=1
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --tal-path) TAL_PATH="$2"; shift 2 ;;
+    --ta-path) TA_PATH="$2"; shift 2 ;;
+    --cir-tal-uri) CIR_TAL_URI="$2"; shift 2 ;;
+    --payload-replay-archive) PAYLOAD_REPLAY_ARCHIVE="$2"; shift 2 ;;
+    --payload-replay-locks) PAYLOAD_REPLAY_LOCKS="$2"; shift 2 ;;
+    --payload-base-archive) PAYLOAD_BASE_ARCHIVE="$2"; shift 2 ;;
+    --payload-base-locks) PAYLOAD_BASE_LOCKS="$2"; shift 2 ;;
+    --payload-delta-archive) PAYLOAD_DELTA_ARCHIVE="$2"; shift 2 ;;
+    --payload-delta-locks) PAYLOAD_DELTA_LOCKS="$2"; shift 2 ;;
+    --base-validation-time) BASE_VALIDATION_TIME="$2"; shift 2 ;;
+    --delta-validation-time) DELTA_VALIDATION_TIME="$2"; shift 2 ;;
+    --max-depth) MAX_DEPTH="$2"; shift 2 ;;
+    --max-instances) MAX_INSTANCES="$2"; shift 2 ;;
+    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$OUT_DIR" && -n "$TAL_PATH" && -n "$TA_PATH" && -n "$CIR_TAL_URI" && -n "$PAYLOAD_REPLAY_ARCHIVE" && -n "$PAYLOAD_REPLAY_LOCKS" && -n "$PAYLOAD_BASE_ARCHIVE" && -n "$PAYLOAD_BASE_LOCKS" && -n "$PAYLOAD_DELTA_ARCHIVE" && -n "$PAYLOAD_DELTA_LOCKS" ]] || {
+  usage >&2
+  exit 2
+}
+
+if [[ ! -x "$RPKI_BIN" ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin rpki
+  )
+fi
+
+resolve_validation_time() {
+  local path="$1"
+  python3 - <<'PY' "$path"
+import json, sys
+print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['validationTime'])
+PY
+}
+
+if [[ -z "$BASE_VALIDATION_TIME" ]]; then
+  BASE_VALIDATION_TIME="$(resolve_validation_time "$PAYLOAD_REPLAY_LOCKS")"
+fi
+
+if [[ -z "$DELTA_VALIDATION_TIME" ]]; then
+  DELTA_VALIDATION_TIME="$(resolve_validation_time "$PAYLOAD_DELTA_LOCKS")"
+fi
+
+rm -rf "$OUT_DIR"
+mkdir -p "$OUT_DIR/full" "$OUT_DIR/delta-001"
+REPO_BYTES_DB="$OUT_DIR/repo-bytes.db"
+
+FULL_DB="$OUT_DIR/full/db"
+DELTA_DB="$OUT_DIR/delta-001/db"
+
+"$RPKI_BIN" \
+  --db "$FULL_DB" \
+  --tal-path "$TAL_PATH" \
+  --ta-path "$TA_PATH" \
+  --payload-replay-archive "$PAYLOAD_REPLAY_ARCHIVE" \
+  --payload-replay-locks "$PAYLOAD_REPLAY_LOCKS" \
+  --validation-time "$BASE_VALIDATION_TIME" \
+  --max-depth "$MAX_DEPTH" \
+  --max-instances "$MAX_INSTANCES" \
+  --ccr-out "$OUT_DIR/full/result.ccr" \
+  --report-json "$OUT_DIR/full/report.json" \
+  --cir-enable \
+  --cir-out "$OUT_DIR/full/input.cir" \
+  --repo-bytes-db "$REPO_BYTES_DB" \
+  --cir-tal-uri "$CIR_TAL_URI" \
+  >"$OUT_DIR/full/run.stdout.log" 2>"$OUT_DIR/full/run.stderr.log"
+
+"$RPKI_BIN" \
+  --db "$DELTA_DB" \
+  --tal-path "$TAL_PATH" \
+  --ta-path "$TA_PATH" \
+  --payload-base-archive "$PAYLOAD_BASE_ARCHIVE" \
+  --payload-base-locks "$PAYLOAD_BASE_LOCKS" \
+  --payload-delta-archive "$PAYLOAD_DELTA_ARCHIVE" \
+  --payload-delta-locks "$PAYLOAD_DELTA_LOCKS" \
+  --payload-base-validation-time "$BASE_VALIDATION_TIME" \
+  --validation-time "$DELTA_VALIDATION_TIME" \
+  --max-depth "$MAX_DEPTH" \
+  --max-instances "$MAX_INSTANCES" \
+  --ccr-out "$OUT_DIR/delta-001/result.ccr" \
+  --report-json "$OUT_DIR/delta-001/report.json" \
+  --cir-enable \
+  --cir-out "$OUT_DIR/delta-001/input.cir" \
+  --repo-bytes-db "$REPO_BYTES_DB" \
+  --cir-tal-uri "$CIR_TAL_URI" \
+  >"$OUT_DIR/delta-001/run.stdout.log" 2>"$OUT_DIR/delta-001/run.stderr.log"
+
+python3 - <<'PY' "$OUT_DIR" "$BASE_VALIDATION_TIME" "$DELTA_VALIDATION_TIME"
+import json
+import os
+import sys
+from pathlib import Path
+
+out = Path(sys.argv[1])
+base_validation_time = sys.argv[2]
+delta_validation_time = sys.argv[3]
+summary = {
+    "version": 1,
+    "kind": "cir_pair",
+    "baseValidationTime": base_validation_time,
+    "deltaValidationTime": delta_validation_time,
+    "repoBytesDbPath": "repo-bytes.db",
+    "steps": [
+        {
+            "kind": "full",
+            "cirPath": "full/input.cir",
+            "ccrPath": "full/result.ccr",
+            "reportPath": "full/report.json",
+        },
+        {
+            "kind": "delta",
+            "cirPath": "delta-001/input.cir",
+            "ccrPath": "delta-001/result.ccr",
+            "reportPath": "delta-001/report.json",
+            "previous": "full",
+        },
+    ],
+    "repoBytesDbExists": (out / "repo-bytes.db").exists(),
+}
+(out / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
+PY
+
+echo "done: $OUT_DIR"

scripts/cir/run_cir_record_sequence_multi_rir_offline.sh

@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_record_sequence_multi_rir_offline.sh \
+    [--bundle-root <path>] \
+    [--rir <afrinic,apnic,arin,lacnic,ripe>] \
+    [--delta-count <n>] \
+    [--full-repo] \
+    [--out-root <path>] \
+    [--rpki-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+CASE_INFO="$ROOT_DIR/scripts/payload_replay/multi_rir_case_info.py"
+SINGLE_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_record_sequence_offline.sh"
+
+BUNDLE_ROOT="/home/yuyr/dev/rust_playground/routinator/bench/multi_rir_demo/runs/20260316-112341-multi-final3"
+RIRS="afrinic,apnic,arin,lacnic,ripe"
+DELTA_COUNT=2
+FULL_REPO=0
+OUT_ROOT="$ROOT_DIR/target/replay/cir_sequence_multi_rir_offline_$(date -u +%Y%m%dT%H%M%SZ)"
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --bundle-root) BUNDLE_ROOT="$2"; shift 2 ;;
+    --rir) RIRS="$2"; shift 2 ;;
+    --delta-count) DELTA_COUNT="$2"; shift 2 ;;
+    --full-repo) FULL_REPO=1; shift 1 ;;
+    --out-root) OUT_ROOT="$2"; shift 2 ;;
+    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+mkdir -p "$OUT_ROOT"
+SUMMARY_JSON="$OUT_ROOT/summary.json"
+SUMMARY_MD="$OUT_ROOT/summary.md"
+
+IFS=',' read -r -a RIR_ITEMS <<< "$RIRS"
+
+for rir in "${RIR_ITEMS[@]}"; do
+  CASE_JSON="$(python3 "$CASE_INFO" --bundle-root "$BUNDLE_ROOT" --repo-root "$ROOT_DIR" --rir "$rir")"
+  TAL_PATH="$(python3 - <<'PY' "$CASE_JSON"
+import json,sys
+print(json.loads(sys.argv[1])['tal_path'])
+PY
+)"
+  TA_PATH="$(python3 - <<'PY' "$CASE_JSON"
+import json,sys
+print(json.loads(sys.argv[1])['ta_path'])
+PY
+)"
+  BASE_ARCHIVE="$(python3 - <<'PY' "$CASE_JSON"
+import json,sys
+print(json.loads(sys.argv[1])['base_archive'])
+PY
+)"
+  BASE_LOCKS="$(python3 - <<'PY' "$CASE_JSON"
+import json,sys
+print(json.loads(sys.argv[1])['base_locks'])
+PY
+)"
+  DELTA_ARCHIVE="$(python3 - <<'PY' "$CASE_JSON"
+import json,sys
+print(json.loads(sys.argv[1])['delta_archive'])
+PY
+)"
+  DELTA_LOCKS="$(python3 - <<'PY' "$CASE_JSON"
+import json,sys
+print(json.loads(sys.argv[1])['delta_locks'])
+PY
+)"
+  OUT_DIR="$OUT_ROOT/$rir"
+  args=(
+    "$SINGLE_SCRIPT"
+    --out-dir "$OUT_DIR" \
+    --tal-path "$TAL_PATH" \
+    --ta-path "$TA_PATH" \
+    --cir-tal-uri "https://example.test/$rir.tal" \
+    --payload-replay-archive "$BASE_ARCHIVE" \
+    --payload-replay-locks "$BASE_LOCKS" \
+    --payload-base-archive "$BASE_ARCHIVE" \
+    --payload-base-locks "$BASE_LOCKS" \
+    --payload-delta-archive "$DELTA_ARCHIVE" \
+    --payload-delta-locks "$DELTA_LOCKS" \
+    --delta-count "$DELTA_COUNT" \
+    --rpki-bin "$RPKI_BIN"
+  )
+  if [[ "$FULL_REPO" -ne 1 ]]; then
+    args+=(--max-depth 0 --max-instances 1)
+  else
+    args+=(--full-repo)
+  fi
+  "${args[@]}"
+done
+
+python3 - <<'PY' "$OUT_ROOT" "$RIRS" "$SUMMARY_JSON" "$SUMMARY_MD"
+import json, sys
+from pathlib import Path
+out_root = Path(sys.argv[1])
+rirs = [item for item in sys.argv[2].split(',') if item]
+summary_json = Path(sys.argv[3])
+summary_md = Path(sys.argv[4])
+items = []
+for rir in rirs:
+    root = out_root / rir
+    seq = json.loads((root / "sequence.json").read_text(encoding="utf-8"))
+    summ = json.loads((root / "summary.json").read_text(encoding="utf-8"))
+    items.append({
+        "rir": rir,
+        "root": str(root),
+        "stepCount": len(seq["steps"]),
+        "repoBytesDbExists": summ.get("repoBytesDbExists", False),
+    })
+summary = {"version": 1, "rirs": items}
+summary_json.write_text(json.dumps(summary, indent=2), encoding="utf-8")
+lines = ["# Multi-RIR Offline CIR Sequence Summary", ""]
+for item in items:
+    lines.append(f"- `{item['rir']}`: `stepCount={item['stepCount']}` `repoBytesDbExists={item['repoBytesDbExists']}` `root={item['root']}`")
+summary_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
+PY
+
+echo "done: $OUT_ROOT"

scripts/cir/run_cir_record_sequence_offline.sh

@@ -0,0 +1,208 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_record_sequence_offline.sh \
+    --out-dir <path> \
+    --tal-path <path> \
+    --ta-path <path> \
+    --cir-tal-uri <url> \
+    --payload-replay-archive <path> \
+    --payload-replay-locks <path> \
+    --payload-base-archive <path> \
+    --payload-base-locks <path> \
+    --payload-delta-archive <path> \
+    --payload-delta-locks <path> \
+    [--delta-count <n>] \
+    [--base-validation-time <rfc3339>] \
+    [--delta-validation-time <rfc3339>] \
+    [--full-repo] \
+    [--max-depth <n>] \
+    [--max-instances <n>] \
+    [--rpki-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+OUT_DIR=""
+TAL_PATH=""
+TA_PATH=""
+CIR_TAL_URI=""
+PAYLOAD_REPLAY_ARCHIVE=""
+PAYLOAD_REPLAY_LOCKS=""
+PAYLOAD_BASE_ARCHIVE=""
+PAYLOAD_BASE_LOCKS=""
+PAYLOAD_DELTA_ARCHIVE=""
+PAYLOAD_DELTA_LOCKS=""
+BASE_VALIDATION_TIME=""
+DELTA_VALIDATION_TIME=""
+DELTA_COUNT=2
+FULL_REPO=0
+MAX_DEPTH=0
+MAX_INSTANCES=1
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --tal-path) TAL_PATH="$2"; shift 2 ;;
+    --ta-path) TA_PATH="$2"; shift 2 ;;
+    --cir-tal-uri) CIR_TAL_URI="$2"; shift 2 ;;
+    --payload-replay-archive) PAYLOAD_REPLAY_ARCHIVE="$2"; shift 2 ;;
+    --payload-replay-locks) PAYLOAD_REPLAY_LOCKS="$2"; shift 2 ;;
+    --payload-base-archive) PAYLOAD_BASE_ARCHIVE="$2"; shift 2 ;;
+    --payload-base-locks) PAYLOAD_BASE_LOCKS="$2"; shift 2 ;;
+    --payload-delta-archive) PAYLOAD_DELTA_ARCHIVE="$2"; shift 2 ;;
+    --payload-delta-locks) PAYLOAD_DELTA_LOCKS="$2"; shift 2 ;;
+    --base-validation-time) BASE_VALIDATION_TIME="$2"; shift 2 ;;
+    --delta-validation-time) DELTA_VALIDATION_TIME="$2"; shift 2 ;;
+    --delta-count) DELTA_COUNT="$2"; shift 2 ;;
+    --full-repo) FULL_REPO=1; shift 1 ;;
+    --max-depth) MAX_DEPTH="$2"; shift 2 ;;
+    --max-instances) MAX_INSTANCES="$2"; shift 2 ;;
+    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$OUT_DIR" && -n "$TAL_PATH" && -n "$TA_PATH" && -n "$CIR_TAL_URI" && -n "$PAYLOAD_REPLAY_ARCHIVE" && -n "$PAYLOAD_REPLAY_LOCKS" && -n "$PAYLOAD_BASE_ARCHIVE" && -n "$PAYLOAD_BASE_LOCKS" && -n "$PAYLOAD_DELTA_ARCHIVE" && -n "$PAYLOAD_DELTA_LOCKS" ]] || {
+  usage >&2
+  exit 2
+}
+
+if [[ ! -x "$RPKI_BIN" ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin rpki
+  )
+fi
+
+resolve_validation_time() {
+  local path="$1"
+  python3 - <<'PY' "$path"
+import json, sys
+print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['validationTime'])
+PY
+}
+
+if [[ -z "$BASE_VALIDATION_TIME" ]]; then
+  BASE_VALIDATION_TIME="$(resolve_validation_time "$PAYLOAD_REPLAY_LOCKS")"
+fi
+if [[ -z "$DELTA_VALIDATION_TIME" ]]; then
+  DELTA_VALIDATION_TIME="$(resolve_validation_time "$PAYLOAD_DELTA_LOCKS")"
+fi
+
+rm -rf "$OUT_DIR"
+mkdir -p "$OUT_DIR/full"
+REPO_BYTES_DB="$OUT_DIR/repo-bytes.db"
+
+run_step() {
+  local kind="$1"
+  local step_dir="$2"
+  local db_dir="$3"
+  shift 3
+  mkdir -p "$step_dir"
+  local -a cmd=(
+    "$RPKI_BIN"
+    --db "$db_dir" \
+    --tal-path "$TAL_PATH" \
+    --ta-path "$TA_PATH" \
+    --ccr-out "$step_dir/result.ccr" \
+    --report-json "$step_dir/report.json" \
+    --cir-enable \
+    --cir-out "$step_dir/input.cir" \
+    --repo-bytes-db "$REPO_BYTES_DB" \
+    --cir-tal-uri "$CIR_TAL_URI"
+  )
+  if [[ "$FULL_REPO" -ne 1 ]]; then
+    cmd+=(--max-depth "$MAX_DEPTH" --max-instances "$MAX_INSTANCES")
+  fi
+  cmd+=("$@")
+  "${cmd[@]}" >"$step_dir/run.stdout.log" 2>"$step_dir/run.stderr.log"
+}
+
+run_step \
+  full \
+  "$OUT_DIR/full" \
+  "$OUT_DIR/full/db" \
+  --payload-replay-archive "$PAYLOAD_REPLAY_ARCHIVE" \
+  --payload-replay-locks "$PAYLOAD_REPLAY_LOCKS" \
+  --validation-time "$BASE_VALIDATION_TIME"
+
+for idx in $(seq 1 "$DELTA_COUNT"); do
+  step_id="$(printf 'delta-%03d' "$idx")"
+  run_step \
+    delta \
+    "$OUT_DIR/$step_id" \
+    "$OUT_DIR/$step_id/db" \
+    --payload-base-archive "$PAYLOAD_BASE_ARCHIVE" \
+    --payload-base-locks "$PAYLOAD_BASE_LOCKS" \
+    --payload-delta-archive "$PAYLOAD_DELTA_ARCHIVE" \
+    --payload-delta-locks "$PAYLOAD_DELTA_LOCKS" \
+    --payload-base-validation-time "$BASE_VALIDATION_TIME" \
+    --validation-time "$DELTA_VALIDATION_TIME"
+done
+
+python3 - <<'PY' "$OUT_DIR" "$BASE_VALIDATION_TIME" "$DELTA_VALIDATION_TIME" "$DELTA_COUNT"
+import json
+import sys
+from pathlib import Path
+
+out = Path(sys.argv[1])
+base_validation_time = sys.argv[2]
+delta_validation_time = sys.argv[3]
+delta_count = int(sys.argv[4])
+
+steps = [
+    {
+        "stepId": "full",
+        "kind": "full",
+        "validationTime": base_validation_time,
+        "cirPath": "full/input.cir",
+        "ccrPath": "full/result.ccr",
+        "reportPath": "full/report.json",
+        "previousStepId": None,
+    }
+]
+previous = "full"
+for idx in range(1, delta_count + 1):
+    step_id = f"delta-{idx:03d}"
+    steps.append(
+        {
+            "stepId": step_id,
+            "kind": "delta",
+            "validationTime": delta_validation_time,
+            "cirPath": f"{step_id}/input.cir",
+            "ccrPath": f"{step_id}/result.ccr",
+            "reportPath": f"{step_id}/report.json",
+            "previousStepId": previous,
+        }
+    )
+    previous = step_id
+
+summary = {
+    "version": 1,
+    "kind": "cir_sequence_offline",
+    "repoBytesDbPath": "repo-bytes.db",
+    "steps": steps,
+}
+(out / "sequence.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
+(out / "summary.json").write_text(
+    json.dumps(
+        {
+            "version": 1,
+            "stepCount": len(steps),
+            "repoBytesDbPath": "repo-bytes.db",
+            "repoBytesDbExists": (out / "repo-bytes.db").exists(),
+        },
+        indent=2,
+    ),
+    encoding="utf-8",
+)
+PY
+
+echo "done: $OUT_DIR"

scripts/cir/run_cir_record_sequence_remote.sh

@@ -0,0 +1,246 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_record_sequence_remote.sh \
+    --rir <name> \
+    --remote-root <path> \
+    [--ssh-target <user@host>] \
+    [--out-subdir <path>] \
+    [--delta-count <n>] \
+    [--sleep-secs <n>] \
+    [--full-repo] \
+    [--max-depth <n>] \
+    [--max-instances <n>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
+RIR=""
+REMOTE_ROOT=""
+OUT_SUBDIR=""
+DELTA_COUNT=2
+SLEEP_SECS=30
+FULL_REPO=0
+MAX_DEPTH=0
+MAX_INSTANCES=1
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rir) RIR="$2"; shift 2 ;;
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --out-subdir) OUT_SUBDIR="$2"; shift 2 ;;
+    --delta-count) DELTA_COUNT="$2"; shift 2 ;;
+    --sleep-secs) SLEEP_SECS="$2"; shift 2 ;;
+    --full-repo) FULL_REPO=1; shift 1 ;;
+    --max-depth) MAX_DEPTH="$2"; shift 2 ;;
+    --max-instances) MAX_INSTANCES="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$RIR" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
+
+case "$RIR" in
+  afrinic) TAL_REL="tests/fixtures/tal/afrinic.tal"; TA_REL="tests/fixtures/ta/afrinic-ta.cer" ;;
+  apnic) TAL_REL="tests/fixtures/tal/apnic-rfc7730-https.tal"; TA_REL="tests/fixtures/ta/apnic-ta.cer" ;;
+  arin) TAL_REL="tests/fixtures/tal/arin.tal"; TA_REL="tests/fixtures/ta/arin-ta.cer" ;;
+  lacnic) TAL_REL="tests/fixtures/tal/lacnic.tal"; TA_REL="tests/fixtures/ta/lacnic-ta.cer" ;;
+  ripe) TAL_REL="tests/fixtures/tal/ripe-ncc.tal"; TA_REL="tests/fixtures/ta/ripe-ncc-ta.cer" ;;
+  *) echo "unsupported rir: $RIR" >&2; exit 2 ;;
+esac
+
+rsync -a --delete \
+  --exclude target \
+  --exclude .git \
+  "$ROOT_DIR/" "$SSH_TARGET:$REMOTE_ROOT/"
+ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/target/release'"
+rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_ROOT/target/release/"
+
+ssh "$SSH_TARGET" \
+  RIR="$RIR" \
+  REMOTE_ROOT="$REMOTE_ROOT" \
+  OUT_SUBDIR="$OUT_SUBDIR" \
+  DELTA_COUNT="$DELTA_COUNT" \
+  SLEEP_SECS="$SLEEP_SECS" \
+  FULL_REPO="$FULL_REPO" \
+  MAX_DEPTH="$MAX_DEPTH" \
+  MAX_INSTANCES="$MAX_INSTANCES" \
+  TAL_REL="$TAL_REL" \
+  TA_REL="$TA_REL" \
+  'bash -s' <<'EOS'
+set -euo pipefail
+
+cd "$REMOTE_ROOT"
+
+if [[ -n "${OUT_SUBDIR}" ]]; then
+  OUT="${OUT_SUBDIR}"
+else
+  OUT="target/replay/cir_sequence_remote_${RIR}_$(date -u +%Y%m%dT%H%M%SZ)"
+fi
+
+mkdir -p "$OUT"
+DB="$OUT/work-db"
+RAW_STORE_DB="$OUT/raw-store.db"
+REPO_BYTES_DB="$OUT/repo-bytes.db"
+ROWS="$OUT/.sequence_rows.tsv"
+: > "$ROWS"
+
+write_step_timing() {
+  local path="$1"
+  local start_ms="$2"
+  local end_ms="$3"
+  local started_at="$4"
+  local finished_at="$5"
+  python3 - <<'PY' "$path" "$start_ms" "$end_ms" "$started_at" "$finished_at"
+import json, sys
+path, start_ms, end_ms, started_at, finished_at = sys.argv[1:]
+start_ms = int(start_ms)
+end_ms = int(end_ms)
+with open(path, "w", encoding="utf-8") as fh:
+    json.dump(
+        {
+            "durationMs": end_ms - start_ms,
+            "startedAt": started_at,
+            "finishedAt": finished_at,
+        },
+        fh,
+        indent=2,
+    )
+PY
+}
+
+run_step() {
+  local step_id="$1"
+  local kind="$2"
+  local previous_step_id="$3"
+  shift 3
+
+  local started_at_iso started_at_ms finished_at_iso finished_at_ms prefix
+  started_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+  started_at_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  prefix="${started_at_iso}-test"
+
+  local cir_out="$OUT/${prefix}.cir"
+  local ccr_out="$OUT/${prefix}.ccr"
+  local report_out="$OUT/${prefix}.report.json"
+  local timing_out="$OUT/${prefix}.timing.json"
+  local stdout_out="$OUT/${prefix}.stdout.log"
+  local stderr_out="$OUT/${prefix}.stderr.log"
+
+  local -a cmd=(
+    target/release/rpki
+    --db "$DB"
+    --raw-store-db "$RAW_STORE_DB"
+    --repo-bytes-db "$REPO_BYTES_DB"
+    --tal-path "$TAL_REL"
+    --ta-path "$TA_REL"
+    --ccr-out "$ccr_out"
+    --report-json "$report_out"
+    --cir-enable
+    --cir-out "$cir_out"
+    --cir-tal-uri "https://example.test/${RIR}.tal"
+  )
+  if [[ "$FULL_REPO" -ne 1 ]]; then
+    cmd+=(--max-depth "$MAX_DEPTH" --max-instances "$MAX_INSTANCES")
+  fi
+  cmd+=("$@")
+  env RPKI_PROGRESS_LOG=1 "${cmd[@]}" >"$stdout_out" 2>"$stderr_out"
+
+  finished_at_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  finished_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+  write_step_timing "$timing_out" "$started_at_ms" "$finished_at_ms" "$started_at_iso" "$finished_at_iso"
+
+  local validation_time
+  validation_time="$(python3 - <<'PY' "$report_out"
+import json, sys
+print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['meta']['validation_time_rfc3339_utc'])
+PY
+)"
+
+  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
+    "$step_id" \
+    "$kind" \
+    "$validation_time" \
+    "$(basename "$cir_out")" \
+    "$(basename "$ccr_out")" \
+    "$(basename "$report_out")" \
+    "$(basename "$timing_out")" \
+    "$(basename "$stdout_out")" \
+    "$(basename "$stderr_out")" >> "$ROWS"
+}
+
+run_step "full" "full" "" 
+
+prev="full"
+for idx in $(seq 1 "$DELTA_COUNT"); do
+  sleep "$SLEEP_SECS"
+  step="$(printf 'delta-%03d' "$idx")"
+  run_step "$step" "delta" "$prev"
+  prev="$step"
+done
+
+python3 - <<'PY' "$OUT" "$ROWS" "$RIR"
+import json, sys
+from pathlib import Path
+
+out = Path(sys.argv[1])
+rows = Path(sys.argv[2]).read_text(encoding='utf-8').splitlines()
+rir = sys.argv[3]
+steps = []
+for idx, row in enumerate(rows):
+    step_id, kind, validation_time, cir_name, ccr_name, report_name, timing_name, stdout_name, stderr_name = row.split('\t')
+    steps.append({
+        "stepId": step_id,
+        "kind": kind,
+        "validationTime": validation_time,
+        "cirPath": cir_name,
+        "ccrPath": ccr_name,
+        "reportPath": report_name,
+        "timingPath": timing_name,
+        "stdoutLogPath": stdout_name,
+        "stderrLogPath": stderr_name,
+        "artifactPrefix": cir_name[:-4],  # strip .cir
+        "previousStepId": None if idx == 0 else steps[idx - 1]["stepId"],
+    })
+
+(out / "sequence.json").write_text(
+    json.dumps({"version": 1, "repoBytesDbPath": "repo-bytes.db", "steps": steps}, indent=2),
+    encoding="utf-8",
+)
+
+summary = {
+    "version": 1,
+    "rir": rir,
+    "stepCount": len(steps),
+    "steps": [],
+}
+for step in steps:
+    timing = json.loads((out / step["timingPath"]).read_text(encoding="utf-8"))
+    summary["steps"].append({
+        "stepId": step["stepId"],
+        "kind": step["kind"],
+        "validationTime": step["validationTime"],
+        "artifactPrefix": step["artifactPrefix"],
+        **timing,
+    })
+
+(out / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
+PY
+
+rm -f "$ROWS"
+echo "$OUT"
+EOS

scripts/cir/run_cir_record_sequence_remote_multi_rir.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_record_sequence_remote_multi_rir.sh \
+    --remote-root <path> \
+    [--rir <afrinic,apnic,arin,lacnic,ripe>] \
+    [--ssh-target <user@host>] \
+    [--out-subdir-root <path>] \
+    [--delta-count <n>] \
+    [--sleep-secs <n>] \
+    [--full-repo] \
+    [--max-depth <n>] \
+    [--max-instances <n>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
+REMOTE_ROOT=""
+RIRS="afrinic,apnic,arin,lacnic,ripe"
+OUT_SUBDIR_ROOT=""
+DELTA_COUNT=2
+SLEEP_SECS=30
+FULL_REPO=0
+MAX_DEPTH=0
+MAX_INSTANCES=1
+SINGLE="$ROOT_DIR/scripts/cir/run_cir_record_sequence_remote.sh"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --rir) RIRS="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --out-subdir-root) OUT_SUBDIR_ROOT="$2"; shift 2 ;;
+    --delta-count) DELTA_COUNT="$2"; shift 2 ;;
+    --sleep-secs) SLEEP_SECS="$2"; shift 2 ;;
+    --full-repo) FULL_REPO=1; shift 1 ;;
+    --max-depth) MAX_DEPTH="$2"; shift 2 ;;
+    --max-instances) MAX_INSTANCES="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
+if [[ -z "$OUT_SUBDIR_ROOT" ]]; then
+  OUT_SUBDIR_ROOT="target/replay/cir_sequence_remote_multi_rir_$(date -u +%Y%m%dT%H%M%SZ)"
+fi
+
+IFS=',' read -r -a ITEMS <<< "$RIRS"
+for rir in "${ITEMS[@]}"; do
+  args=(
+    "$SINGLE"
+    --rir "$rir" \
+    --remote-root "$REMOTE_ROOT" \
+    --ssh-target "$SSH_TARGET" \
+    --out-subdir "$OUT_SUBDIR_ROOT/$rir" \
+    --delta-count "$DELTA_COUNT" \
+    --sleep-secs "$SLEEP_SECS" \
+  )
+  if [[ "$FULL_REPO" -eq 1 ]]; then
+    args+=(--full-repo)
+  else
+    args+=(--max-depth "$MAX_DEPTH" --max-instances "$MAX_INSTANCES")
+  fi
+  "${args[@]}"
+done
+
+echo "$OUT_SUBDIR_ROOT"

scripts/cir/run_cir_record_sequence_ta_only_multi_rir.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_record_sequence_ta_only_multi_rir.sh \
+    [--rir <afrinic,apnic,arin,lacnic,ripe>] \
+    [--delta-count <n>] \
+    [--out-root <path>] \
+    [--rpki-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+HELPER_BIN="${HELPER_BIN:-$ROOT_DIR/target/release/cir_ta_only_fixture}"
+MATERIALIZE_BIN="${MATERIALIZE_BIN:-$ROOT_DIR/target/release/cir_materialize}"
+EXTRACT_BIN="${EXTRACT_BIN:-$ROOT_DIR/target/release/cir_extract_inputs}"
+WRAPPER="$ROOT_DIR/scripts/cir/cir-rsync-wrapper"
+RIRS="afrinic,apnic,arin,lacnic,ripe"
+DELTA_COUNT=2
+OUT_ROOT="$ROOT_DIR/target/replay/cir_sequence_multi_rir_ta_only_$(date -u +%Y%m%dT%H%M%SZ)"
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rir) RIRS="$2"; shift 2 ;;
+    --delta-count) DELTA_COUNT="$2"; shift 2 ;;
+    --out-root) OUT_ROOT="$2"; shift 2 ;;
+    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+if [[ ! -x "$HELPER_BIN" ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin cir_ta_only_fixture --bin rpki --bin cir_materialize --bin cir_extract_inputs
+  )
+fi
+
+case_paths() {
+  case "$1" in
+    afrinic) echo "tests/fixtures/tal/afrinic.tal tests/fixtures/ta/afrinic-ta.cer" ;;
+    apnic) echo "tests/fixtures/tal/apnic-rfc7730-https.tal tests/fixtures/ta/apnic-ta.cer" ;;
+    arin) echo "tests/fixtures/tal/arin.tal tests/fixtures/ta/arin-ta.cer" ;;
+    lacnic) echo "tests/fixtures/tal/lacnic.tal tests/fixtures/ta/lacnic-ta.cer" ;;
+    ripe) echo "tests/fixtures/tal/ripe-ncc.tal tests/fixtures/ta/ripe-ncc-ta.cer" ;;
+    *) return 1 ;;
+  esac
+}
+
+mkdir -p "$OUT_ROOT"
+IFS=',' read -r -a ITEMS <<< "$RIRS"
+for rir in "${ITEMS[@]}"; do
+  read -r tal_rel ta_rel < <(case_paths "$rir")
+  rir_root="$OUT_ROOT/$rir"
+  mkdir -p "$rir_root/full"
+  repo_bytes_db="$rir_root/repo-bytes.db"
+  "$HELPER_BIN" \
+    --tal-path "$ROOT_DIR/$tal_rel" \
+    --ta-path "$ROOT_DIR/$ta_rel" \
+    --tal-uri "https://example.test/$rir.tal" \
+    --validation-time "2026-04-09T00:00:00Z" \
+    --cir-out "$rir_root/full/input.cir" \
+    --repo-bytes-db "$repo_bytes_db"
+  "$EXTRACT_BIN" --cir "$rir_root/full/input.cir" --tals-dir "$rir_root/.tmp/tals" --meta-json "$rir_root/.tmp/meta.json"
+  "$MATERIALIZE_BIN" --cir "$rir_root/full/input.cir" --repo-bytes-db "$repo_bytes_db" --mirror-root "$rir_root/.tmp/mirror"
+  FIRST_TAL="$(python3 - <<'PY' "$rir_root/.tmp/meta.json"
+import json,sys
+print(json.load(open(sys.argv[1]))["talFiles"][0]["path"])
+PY
+)"
+  export CIR_MIRROR_ROOT="$rir_root/.tmp/mirror"
+  export REAL_RSYNC_BIN=/usr/bin/rsync
+  export CIR_LOCAL_LINK_MODE=1
+  "$RPKI_BIN" \
+    --db "$rir_root/full/db" \
+    --tal-path "$FIRST_TAL" \
+    --disable-rrdp \
+    --rsync-command "$WRAPPER" \
+    --validation-time "2026-04-09T00:00:00Z" \
+    --ccr-out "$rir_root/full/result.ccr" \
+    --report-json "$rir_root/full/report.json" >/dev/null 2>&1
+  for idx in $(seq 1 "$DELTA_COUNT"); do
+    step="$(printf 'delta-%03d' "$idx")"
+    mkdir -p "$rir_root/$step"
+    cp "$rir_root/full/input.cir" "$rir_root/$step/input.cir"
+    cp "$rir_root/full/result.ccr" "$rir_root/$step/result.ccr"
+    cp "$rir_root/full/report.json" "$rir_root/$step/report.json"
+  done
+  python3 - <<'PY' "$rir_root" "$DELTA_COUNT"
+import json, sys
+from pathlib import Path
+root = Path(sys.argv[1]); delta_count = int(sys.argv[2])
+steps = [{"stepId":"full","kind":"full","validationTime":"2026-04-09T00:00:00Z","cirPath":"full/input.cir","ccrPath":"full/result.ccr","reportPath":"full/report.json","previousStepId":None}]
+prev = "full"
+for i in range(1, delta_count + 1):
+    step = f"delta-{i:03d}"
+    steps.append({"stepId":step,"kind":"delta","validationTime":"2026-04-09T00:00:00Z","cirPath":f"{step}/input.cir","ccrPath":f"{step}/result.ccr","reportPath":f"{step}/report.json","previousStepId":prev})
+    prev = step
+(root/"sequence.json").write_text(json.dumps({"version":1,"repoBytesDbPath":"repo-bytes.db","steps":steps}, indent=2), encoding="utf-8")
+(root/"summary.json").write_text(json.dumps({"version":1,"stepCount":len(steps)}, indent=2), encoding="utf-8")
+PY
+done
+
+python3 - <<'PY' "$OUT_ROOT" "$RIRS"
+import json, sys
+from pathlib import Path
+root = Path(sys.argv[1]); rirs = [x for x in sys.argv[2].split(',') if x]
+items=[]
+for rir in rirs:
+    seq=json.loads((root/rir/'sequence.json').read_text())
+    items.append({"rir":rir,"stepCount":len(seq['steps'])})
+(root/'summary.json').write_text(json.dumps({"version":1,"rirs":items}, indent=2), encoding='utf-8')
+PY
+
+echo "done: $OUT_ROOT"

scripts/cir/run_cir_record_sequence_ta_only_remote_multi_rir.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_record_sequence_ta_only_remote_multi_rir.sh \
+    --remote-root <path> \
+    [--ssh-target <user@host>] \
+    [--rir <afrinic,apnic,arin,lacnic,ripe>] \
+    [--delta-count <n>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
+REMOTE_ROOT=""
+RIRS="afrinic,apnic,arin,lacnic,ripe"
+DELTA_COUNT=2
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --rir) RIRS="$2"; shift 2 ;;
+    --delta-count) DELTA_COUNT="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
+
+rsync -a --delete \
+  --exclude target \
+  --exclude .git \
+  "$ROOT_DIR/" "$SSH_TARGET:$REMOTE_ROOT/"
+ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/target/release'"
+for bin in rpki cir_ta_only_fixture cir_materialize cir_extract_inputs; do
+  rsync -a "$ROOT_DIR/target/release/$bin" "$SSH_TARGET:$REMOTE_ROOT/target/release/"
+done
+
+ssh "$SSH_TARGET" "bash -lc '
+set -euo pipefail
+cd $REMOTE_ROOT
+OUT=target/replay/cir_sequence_remote_ta_only_\$(date -u +%Y%m%dT%H%M%SZ)
+./scripts/cir/run_cir_record_sequence_ta_only_multi_rir.sh --rir $RIRS --delta-count $DELTA_COUNT --out-root \"\$OUT\"
+echo \"\$OUT\"
+'"

scripts/cir/run_cir_replay_matrix.sh

@@ -0,0 +1,293 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_replay_matrix.sh \
+    --cir <path> \
+    --repo-bytes-db <path> \
+    --out-dir <path> \
+    --reference-ccr <path> \
+    --rpki-client-build-dir <path> \
+    [--keep-db] \
+    [--rpki-bin <path>] \
+    [--routinator-root <path>] \
+    [--routinator-bin <path>] \
+    [--real-rsync-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+CIR=""
+REPO_BYTES_DB=""
+OUT_DIR=""
+REFERENCE_CCR=""
+RPKI_CLIENT_BUILD_DIR=""
+KEEP_DB=0
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+ROUTINATOR_ROOT="${ROUTINATOR_ROOT:-/home/yuyr/dev/rust_playground/routinator}"
+ROUTINATOR_BIN="${ROUTINATOR_BIN:-$ROUTINATOR_ROOT/target/debug/routinator}"
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+OURS_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_ours.sh"
+ROUTINATOR_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_routinator.sh"
+RPKI_CLIENT_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_rpki_client.sh"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --cir) CIR="$2"; shift 2 ;;
+    --repo-bytes-db) REPO_BYTES_DB="$2"; shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
+    --rpki-client-build-dir) RPKI_CLIENT_BUILD_DIR="$2"; shift 2 ;;
+    --keep-db) KEEP_DB=1; shift ;;
+    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
+    --routinator-root) ROUTINATOR_ROOT="$2"; shift 2 ;;
+    --routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$CIR" && -n "$REPO_BYTES_DB" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" && -n "$RPKI_CLIENT_BUILD_DIR" ]] || {
+  usage >&2
+  exit 2
+}
+
+mkdir -p "$OUT_DIR"
+
+run_with_timing() {
+  local summary_path="$1"
+  local timing_path="$2"
+  shift 2
+  local start end status
+  start="$(python3 - <<'PY'
+import time
+print(time.perf_counter_ns())
+PY
+)"
+  if "$@"; then
+    status=0
+  else
+    status=$?
+  fi
+  end="$(python3 - <<'PY'
+import time
+print(time.perf_counter_ns())
+PY
+)"
+  python3 - <<'PY' "$summary_path" "$timing_path" "$status" "$start" "$end"
+import json, sys
+summary_path, timing_path, status, start, end = sys.argv[1:]
+duration_ms = max(0, (int(end) - int(start)) // 1_000_000)
+data = {"exitCode": int(status), "durationMs": duration_ms}
+try:
+    with open(summary_path, "r", encoding="utf-8") as f:
+        data["compare"] = json.load(f)
+except FileNotFoundError:
+    data["compare"] = None
+with open(timing_path, "w", encoding="utf-8") as f:
+    json.dump(data, f, indent=2)
+PY
+  return "$status"
+}
+
+OURS_OUT="$OUT_DIR/ours"
+ROUTINATOR_OUT="$OUT_DIR/routinator"
+RPKI_CLIENT_OUT="$OUT_DIR/rpki-client"
+mkdir -p "$OURS_OUT" "$ROUTINATOR_OUT" "$RPKI_CLIENT_OUT"
+
+ours_cmd=(
+  "$OURS_SCRIPT"
+  --cir "$CIR"
+  --repo-bytes-db "$REPO_BYTES_DB"
+  --out-dir "$OURS_OUT"
+  --reference-ccr "$REFERENCE_CCR"
+  --rpki-bin "$RPKI_BIN"
+  --real-rsync-bin "$REAL_RSYNC_BIN"
+)
+routinator_cmd=(
+  "$ROUTINATOR_SCRIPT"
+  --cir "$CIR"
+  --repo-bytes-db "$REPO_BYTES_DB"
+  --out-dir "$ROUTINATOR_OUT"
+  --reference-ccr "$REFERENCE_CCR"
+  --routinator-root "$ROUTINATOR_ROOT"
+  --routinator-bin "$ROUTINATOR_BIN"
+  --real-rsync-bin "$REAL_RSYNC_BIN"
+)
+rpki_client_cmd=(
+  "$RPKI_CLIENT_SCRIPT"
+  --cir "$CIR"
+  --repo-bytes-db "$REPO_BYTES_DB"
+  --out-dir "$RPKI_CLIENT_OUT"
+  --reference-ccr "$REFERENCE_CCR"
+  --build-dir "$RPKI_CLIENT_BUILD_DIR"
+  --real-rsync-bin "$REAL_RSYNC_BIN"
+)
+
+if [[ "$KEEP_DB" -eq 1 ]]; then
+  ours_cmd+=(--keep-db)
+  routinator_cmd+=(--keep-db)
+  rpki_client_cmd+=(--keep-db)
+fi
+
+ours_status=0
+routinator_status=0
+rpki_client_status=0
+if run_with_timing "$OURS_OUT/compare-summary.json" "$OURS_OUT/timing.json" "${ours_cmd[@]}"; then
+  :
+else
+  ours_status=$?
+fi
+if run_with_timing "$ROUTINATOR_OUT/compare-summary.json" "$ROUTINATOR_OUT/timing.json" "${routinator_cmd[@]}"; then
+  :
+else
+  routinator_status=$?
+fi
+if run_with_timing "$RPKI_CLIENT_OUT/compare-summary.json" "$RPKI_CLIENT_OUT/timing.json" "${rpki_client_cmd[@]}"; then
+  :
+else
+  rpki_client_status=$?
+fi
+
+SUMMARY_JSON="$OUT_DIR/summary.json"
+SUMMARY_MD="$OUT_DIR/summary.md"
+DETAIL_MD="$OUT_DIR/detail.md"
+
+python3 - <<'PY' \
+  "$CIR" \
+  "$REPO_BYTES_DB" \
+  "$REFERENCE_CCR" \
+  "$OURS_OUT" \
+  "$ROUTINATOR_OUT" \
+  "$RPKI_CLIENT_OUT" \
+  "$SUMMARY_JSON" \
+  "$SUMMARY_MD" \
+  "$DETAIL_MD"
+import json
+import sys
+from pathlib import Path
+
+cir_path, repo_bytes_db, reference_ccr, ours_out, routinator_out, rpki_client_out, summary_json, summary_md, detail_md = sys.argv[1:]
+
+participants = []
+all_match = True
+for name, out_dir in [
+    ("ours", ours_out),
+    ("routinator", routinator_out),
+    ("rpki-client", rpki_client_out),
+]:
+    out = Path(out_dir)
+    timing = json.loads((out / "timing.json").read_text(encoding="utf-8"))
+    compare = timing.get("compare") or {}
+    vrps = compare.get("vrps") or {}
+    vaps = compare.get("vaps") or {}
+    participant = {
+        "name": name,
+        "outDir": str(out),
+        "tmpRoot": str(out / ".tmp"),
+        "mirrorPath": str(out / ".tmp" / "mirror"),
+        "timingPath": str(out / "timing.json"),
+        "summaryPath": str(out / "compare-summary.json"),
+        "exitCode": timing["exitCode"],
+        "durationMs": timing["durationMs"],
+        "compareMode": compare.get("compareMode"),
+        "talCount": compare.get("talCount"),
+        "talPaths": compare.get("talPaths", []),
+        "vrps": vrps,
+        "vaps": vaps,
+        "match": bool(vrps.get("match")) and bool(vaps.get("match")) and timing["exitCode"] == 0,
+        "logPaths": [str(path) for path in sorted(out.glob("*.log"))],
+    }
+    participants.append(participant)
+    all_match = all_match and participant["match"]
+
+summary = {
+    "cirPath": cir_path,
+    "repoBytesDb": repo_bytes_db,
+    "referenceCcr": reference_ccr,
+    "participants": participants,
+    "allMatch": all_match,
+}
+Path(summary_json).write_text(json.dumps(summary, indent=2), encoding="utf-8")
+
+lines = [
+    "# CIR Replay Matrix Summary",
+    "",
+    f"- `cir`: `{cir_path}`",
+    f"- `repo_bytes_db`: `{repo_bytes_db}`",
+    f"- `reference_ccr`: `{reference_ccr}`",
+    f"- `all_match`: `{all_match}`",
+    "",
+    "| Participant | Exit | Duration (ms) | TALs | Compare mode | VRP actual/ref | VRP match | VAP actual/ref | VAP match | Log |",
+    "| --- | ---: | ---: | ---: | --- | --- | --- | --- | --- | --- |",
+]
+for participant in participants:
+    vrps = participant["vrps"] or {}
+    vaps = participant["vaps"] or {}
+    log_path = participant["logPaths"][0] if participant["logPaths"] else ""
+    lines.append(
+        "| {name} | {exit_code} | {duration_ms} | {tal_count} | {compare_mode} | {vrp_actual}/{vrp_ref} | {vrp_match} | {vap_actual}/{vap_ref} | {vap_match} | `{log_path}` |".format(
+            name=participant["name"],
+            exit_code=participant["exitCode"],
+            duration_ms=participant["durationMs"],
+            tal_count=participant.get("talCount") if participant.get("talCount") is not None else "-",
+            compare_mode=participant.get("compareMode") or "-",
+            vrp_actual=vrps.get("actual", "-"),
+            vrp_ref=vrps.get("reference", "-"),
+            vrp_match=vrps.get("match", False),
+            vap_actual=vaps.get("actual", "-"),
+            vap_ref=vaps.get("reference", "-"),
+            vap_match=vaps.get("match", False),
+            log_path=log_path,
+        )
+    )
+Path(summary_md).write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+detail_lines = [
+    "# CIR Replay Matrix Detail",
+    "",
+]
+for participant in participants:
+    vrps = participant["vrps"] or {}
+    vaps = participant["vaps"] or {}
+    detail_lines.extend([
+        f"## {participant['name']}",
+        f"- `exit_code`: `{participant['exitCode']}`",
+        f"- `duration_ms`: `{participant['durationMs']}`",
+        f"- `out_dir`: `{participant['outDir']}`",
+        f"- `tmp_root`: `{participant['tmpRoot']}`",
+        f"- `mirror_path`: `{participant['mirrorPath']}`",
+        f"- `summary_path`: `{participant['summaryPath']}`",
+        f"- `timing_path`: `{participant['timingPath']}`",
+        f"- `compare_mode`: `{participant.get('compareMode')}`",
+        f"- `tal_count`: `{participant.get('talCount')}`",
+        f"- `log_paths`: `{', '.join(participant['logPaths'])}`",
+        f"- `vrps`: `actual={vrps.get('actual', '-')}` `reference={vrps.get('reference', '-')}` `match={vrps.get('match', False)}`",
+        f"- `vaps`: `actual={vaps.get('actual', '-')}` `reference={vaps.get('reference', '-')}` `match={vaps.get('match', False)}`",
+        f"- `vrps.only_in_actual`: `{vrps.get('only_in_actual', [])}`",
+        f"- `vrps.only_in_reference`: `{vrps.get('only_in_reference', [])}`",
+        f"- `vaps.only_in_actual`: `{vaps.get('only_in_actual', [])}`",
+        f"- `vaps.only_in_reference`: `{vaps.get('only_in_reference', [])}`",
+        "",
+    ])
+Path(detail_md).write_text("\n".join(detail_lines), encoding="utf-8")
+PY
+
+if [[ "$ours_status" -ne 0 || "$routinator_status" -ne 0 || "$rpki_client_status" -ne 0 ]]; then
+  exit 1
+fi
+
+all_match="$(python3 - <<'PY' "$SUMMARY_JSON"
+import json,sys
+print("true" if json.load(open(sys.argv[1]))["allMatch"] else "false")
+PY
+)"
+if [[ "$all_match" != "true" ]]; then
+  exit 1
+fi
+
+echo "done: $OUT_DIR"

scripts/cir/run_cir_replay_ours.sh

@@ -0,0 +1,252 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_replay_ours.sh \
+    --cir <path> \
+    --repo-bytes-db <path> \
+    --out-dir <path> \
+    --reference-ccr <path> \
+    [--keep-db] \
+    [--write-actual-ccr] \
+    [--write-report-json] \
+    [--report-json-compact] \
+    [--phase2-object-workers <n>] \
+    [--phase2-worker-queue-capacity <n>] \
+    [--rpki-bin <path>] \
+    [--real-rsync-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+CIR=""
+REPO_BYTES_DB=""
+OUT_DIR=""
+REFERENCE_CCR=""
+KEEP_DB=0
+WRITE_ACTUAL_CCR=0
+WRITE_REPORT_JSON=0
+REPORT_JSON_COMPACT=0
+PHASE2_OBJECT_WORKERS="${CIR_REPLAY_PHASE2_OBJECT_WORKERS:-4}"
+PHASE2_WORKER_QUEUE_CAPACITY="${CIR_REPLAY_PHASE2_WORKER_QUEUE_CAPACITY:-64}"
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+CIR_MATERIALIZE_BIN="${CIR_MATERIALIZE_BIN:-$ROOT_DIR/target/release/cir_materialize}"
+CIR_EXTRACT_INPUTS_BIN="${CIR_EXTRACT_INPUTS_BIN:-$ROOT_DIR/target/release/cir_extract_inputs}"
+CCR_TO_COMPARE_VIEWS_BIN="${CCR_TO_COMPARE_VIEWS_BIN:-$ROOT_DIR/target/release/ccr_to_compare_views}"
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+WRAPPER="$ROOT_DIR/scripts/cir/cir-rsync-wrapper"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --cir) CIR="$2"; shift 2 ;;
+    --repo-bytes-db) REPO_BYTES_DB="$2"; shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
+    --keep-db) KEEP_DB=1; shift ;;
+    --write-actual-ccr) WRITE_ACTUAL_CCR=1; shift ;;
+    --write-report-json) WRITE_REPORT_JSON=1; shift ;;
+    --report-json-compact) WRITE_REPORT_JSON=1; REPORT_JSON_COMPACT=1; shift ;;
+    --phase2-object-workers) PHASE2_OBJECT_WORKERS="$2"; shift 2 ;;
+    --phase2-worker-queue-capacity) PHASE2_WORKER_QUEUE_CAPACITY="$2"; shift 2 ;;
+    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$CIR" && -n "$REPO_BYTES_DB" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
+  usage >&2
+  exit 2
+}
+
+mkdir -p "$OUT_DIR"
+
+needs_build=0
+if [[ ! -x "$RPKI_BIN" || ! -x "$CIR_MATERIALIZE_BIN" || ! -x "$CIR_EXTRACT_INPUTS_BIN" || ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
+  needs_build=1
+elif [[ "$RPKI_BIN" == "$ROOT_DIR/target/release/rpki" ]] && find "$ROOT_DIR/src" "$ROOT_DIR/Cargo.toml" -newer "$RPKI_BIN" -print -quit | grep -q .; then
+  needs_build=1
+fi
+if [[ "$needs_build" -eq 1 ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin rpki --bin cir_materialize --bin cir_extract_inputs --bin ccr_to_compare_views
+  )
+fi
+
+TMP_ROOT="$OUT_DIR/.tmp"
+TALS_DIR="$TMP_ROOT/tals"
+META_JSON="$TMP_ROOT/meta.json"
+MIRROR_ROOT="$TMP_ROOT/mirror"
+DB_DIR="$TMP_ROOT/work-db"
+REPLAY_RAW_STORE_DB="$TMP_ROOT/replay-raw-store.db"
+REPLAY_REPO_BYTES_DB="$TMP_ROOT/replay-repo-bytes.db"
+ACTUAL_CCR="$OUT_DIR/actual.ccr"
+ACTUAL_REPORT="$OUT_DIR/report.json"
+ACTUAL_VRPS="$OUT_DIR/actual-vrps.csv"
+ACTUAL_VAPS="$OUT_DIR/actual-vaps.csv"
+REF_VRPS="$OUT_DIR/reference-vrps.csv"
+REF_VAPS="$OUT_DIR/reference-vaps.csv"
+COMPARE_JSON="$OUT_DIR/compare-summary.json"
+RUN_LOG="$OUT_DIR/run.log"
+
+rm -rf "$TMP_ROOT"
+mkdir -p "$TMP_ROOT"
+
+"$CIR_EXTRACT_INPUTS_BIN" --cir "$CIR" --tals-dir "$TALS_DIR" --meta-json "$META_JSON"
+materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --repo-bytes-db "$REPO_BYTES_DB" --mirror-root "$MIRROR_ROOT")
+if [[ "$KEEP_DB" -eq 1 ]]; then
+  materialize_cmd+=(--keep-db)
+fi
+"${materialize_cmd[@]}"
+
+VALIDATION_TIME="$(python3 - <<'PY' "$META_JSON"
+import json,sys
+print(json.load(open(sys.argv[1]))["validationTime"])
+PY
+)"
+mapfile -t TAL_PATHS < <(python3 - <<'PY' "$META_JSON"
+import json, sys
+for item in json.load(open(sys.argv[1], encoding="utf-8"))["talFiles"]:
+    print(item["path"])
+PY
+)
+TAL_ARGS=()
+for tal_path in "${TAL_PATHS[@]}"; do
+  TAL_ARGS+=(--tal-path "$tal_path")
+done
+
+export CIR_MIRROR_ROOT="$(python3 - <<'PY' "$MIRROR_ROOT"
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
+export CIR_LOCAL_LINK_MODE=1
+
+REPORT_JSON_ARGS=(--skip-report-build)
+VCIR_ARGS=(--skip-vcir-persist)
+if [[ "$WRITE_REPORT_JSON" -eq 1 ]]; then
+  REPORT_JSON_ARGS=(--report-json "$ACTUAL_REPORT")
+  if [[ "$REPORT_JSON_COMPACT" -eq 1 ]]; then
+    REPORT_JSON_ARGS+=(--report-json-compact)
+  fi
+fi
+CCR_ARGS=()
+if [[ "$WRITE_ACTUAL_CCR" -eq 1 ]]; then
+  CCR_ARGS=(--ccr-out "$ACTUAL_CCR")
+fi
+
+"$RPKI_BIN" \
+  --db "$DB_DIR" \
+  --raw-store-db "$REPLAY_RAW_STORE_DB" \
+  --repo-bytes-db "$REPLAY_REPO_BYTES_DB" \
+  "${TAL_ARGS[@]}" \
+  --parallel-phase2-object-workers "$PHASE2_OBJECT_WORKERS" \
+  --parallel-phase2-worker-queue-capacity "$PHASE2_WORKER_QUEUE_CAPACITY" \
+  --disable-rrdp \
+  --rsync-command "$WRAPPER" \
+  --validation-time "$VALIDATION_TIME" \
+  "${CCR_ARGS[@]}" \
+  --vrps-csv-out "$ACTUAL_VRPS" \
+  --vaps-csv-out "$ACTUAL_VAPS" \
+  --compare-view-trust-anchor unknown \
+  "${VCIR_ARGS[@]}" \
+  "${REPORT_JSON_ARGS[@]}" \
+  >"$RUN_LOG" 2>&1
+
+sort_compare_csv() {
+  local path="$1"
+  local tmp="${path}.sorted.tmp"
+  {
+    head -n 1 "$path"
+    tail -n +2 "$path" | LC_ALL=C sort -u
+  } >"$tmp"
+  mv "$tmp" "$path"
+}
+
+sort_compare_csv "$ACTUAL_VRPS"
+sort_compare_csv "$ACTUAL_VAPS"
+"$CCR_TO_COMPARE_VIEWS_BIN" --ccr "$REFERENCE_CCR" --vrps-out "$REF_VRPS" --vaps-out "$REF_VAPS" --trust-anchor unknown
+
+python3 - <<'PY' "$ACTUAL_VRPS" "$REF_VRPS" "$ACTUAL_VAPS" "$REF_VAPS" "$COMPARE_JSON" "$META_JSON" "$WRITE_REPORT_JSON" "$WRITE_ACTUAL_CCR" "$PHASE2_OBJECT_WORKERS" "$PHASE2_WORKER_QUEUE_CAPACITY"
+import csv, json, sys
+
+def next_row(reader):
+    try:
+        return tuple(next(reader))
+    except StopIteration:
+        return None
+
+def compare_sorted_csv(actual_path, ref_path):
+    actual_count = 0
+    ref_count = 0
+    only_actual_count = 0
+    only_ref_count = 0
+    only_actual_sample = []
+    only_ref_sample = []
+    with open(actual_path, newline="") as actual_file, open(ref_path, newline="") as ref_file:
+        actual_reader = csv.reader(actual_file)
+        ref_reader = csv.reader(ref_file)
+        next(actual_reader, None)
+        next(ref_reader, None)
+        actual = next_row(actual_reader)
+        ref = next_row(ref_reader)
+        while actual is not None or ref is not None:
+            if ref is None or (actual is not None and actual < ref):
+                actual_count += 1
+                only_actual_count += 1
+                if len(only_actual_sample) < 20:
+                    only_actual_sample.append(list(actual))
+                actual = next_row(actual_reader)
+            elif actual is None or ref < actual:
+                ref_count += 1
+                only_ref_count += 1
+                if len(only_ref_sample) < 20:
+                    only_ref_sample.append(list(ref))
+                ref = next_row(ref_reader)
+            else:
+                actual_count += 1
+                ref_count += 1
+                actual = next_row(actual_reader)
+                ref = next_row(ref_reader)
+    return {
+        "actual": actual_count,
+        "reference": ref_count,
+        "only_in_actual": only_actual_sample,
+        "only_in_reference": only_ref_sample,
+        "only_in_actual_count": only_actual_count,
+        "only_in_reference_count": only_ref_count,
+        "match": only_actual_count == 0 and only_ref_count == 0,
+    }
+
+vrps = compare_sorted_csv(sys.argv[1], sys.argv[2])
+vaps = compare_sorted_csv(sys.argv[3], sys.argv[4])
+meta = json.load(open(sys.argv[6], encoding="utf-8"))
+summary = {
+  "compareMode": "trust-anchor-agnostic",
+  "talCount": len(meta["talFiles"]),
+  "talPaths": [item["path"] for item in meta["talFiles"]],
+  "actualCcrWritten": sys.argv[8] == "1",
+  "reportJsonWritten": sys.argv[7] == "1",
+  "replayParallelism": {
+    "phase2ObjectWorkers": int(sys.argv[9]),
+    "phase2WorkerQueueCapacity": int(sys.argv[10]),
+  },
+  "vrps": vrps,
+  "vaps": vaps,
+}
+with open(sys.argv[5], "w") as f:
+    json.dump(summary, f, indent=2)
+PY
+
+if [[ "$KEEP_DB" -ne 1 ]]; then
+  rm -rf "$TMP_ROOT"
+fi
+
+echo "done: $OUT_DIR"

scripts/cir/run_cir_replay_routinator.sh

@@ -0,0 +1,239 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_replay_routinator.sh \
+    --cir <path> \
+    --repo-bytes-db <path> \
+    --out-dir <path> \
+    --reference-ccr <path> \
+    [--keep-db] \
+    [--routinator-root <path>] \
+    [--routinator-bin <path>] \
+    [--real-rsync-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RPKI_DEV_ROOT="${RPKI_DEV_ROOT:-$ROOT_DIR}"
+
+CIR=""
+REPO_BYTES_DB=""
+OUT_DIR=""
+REFERENCE_CCR=""
+KEEP_DB=0
+ROUTINATOR_ROOT="${ROUTINATOR_ROOT:-/home/yuyr/dev/rust_playground/routinator}"
+ROUTINATOR_BIN="${ROUTINATOR_BIN:-$ROUTINATOR_ROOT/target/debug/routinator}"
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+CIR_MATERIALIZE_BIN="${CIR_MATERIALIZE_BIN:-$ROOT_DIR/target/release/cir_materialize}"
+CIR_EXTRACT_INPUTS_BIN="${CIR_EXTRACT_INPUTS_BIN:-$ROOT_DIR/target/release/cir_extract_inputs}"
+CCR_TO_COMPARE_VIEWS_BIN="${CCR_TO_COMPARE_VIEWS_BIN:-$ROOT_DIR/target/release/ccr_to_compare_views}"
+WRAPPER="$ROOT_DIR/scripts/cir/cir-rsync-wrapper"
+JSON_TO_VAPS="$ROOT_DIR/scripts/cir/json_to_vaps_csv.py"
+FAKETIME_LIB="${FAKETIME_LIB:-$ROOT_DIR/target/tools/faketime_pkg/extracted/libfaketime/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --cir) CIR="$2"; shift 2 ;;
+    --repo-bytes-db) REPO_BYTES_DB="$2"; shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
+    --keep-db) KEEP_DB=1; shift ;;
+    --routinator-root) ROUTINATOR_ROOT="$2"; shift 2 ;;
+    --routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$CIR" && -n "$REPO_BYTES_DB" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
+  usage >&2
+  exit 2
+}
+if [[ ! -x "$ROUTINATOR_BIN" ]]; then
+  echo "routinator binary not executable: $ROUTINATOR_BIN" >&2
+  exit 2
+fi
+
+mkdir -p "$OUT_DIR"
+if [[ ! -x "$CIR_MATERIALIZE_BIN" || ! -x "$CIR_EXTRACT_INPUTS_BIN" || ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin cir_materialize --bin cir_extract_inputs --bin ccr_to_compare_views
+  )
+fi
+
+TMP_ROOT="$OUT_DIR/.tmp"
+TALS_DIR="$TMP_ROOT/tals"
+META_JSON="$TMP_ROOT/meta.json"
+MIRROR_ROOT="$TMP_ROOT/mirror"
+WORK_REPO="$TMP_ROOT/repository"
+RUN_LOG="$OUT_DIR/routinator.log"
+ACTUAL_VRPS="$OUT_DIR/actual-vrps.csv"
+ACTUAL_VAPS_JSON="$OUT_DIR/actual-vaps.json"
+ACTUAL_VAPS="$OUT_DIR/actual-vaps.csv"
+REF_VRPS="$OUT_DIR/reference-vrps.csv"
+REF_VAPS="$OUT_DIR/reference-vaps.csv"
+SUMMARY_JSON="$OUT_DIR/compare-summary.json"
+
+rm -rf "$TMP_ROOT"
+mkdir -p "$TMP_ROOT"
+
+"$CIR_EXTRACT_INPUTS_BIN" --cir "$CIR" --tals-dir "$TALS_DIR" --meta-json "$META_JSON"
+python3 - <<'PY' "$TALS_DIR"
+from pathlib import Path
+import sys
+for tal in Path(sys.argv[1]).glob("*.tal"):
+    lines = tal.read_text(encoding="utf-8").splitlines()
+    rsync_uris = [line for line in lines if line.startswith("rsync://")]
+    base64_lines = []
+    seen_sep = False
+    for line in lines:
+        if seen_sep:
+            if line.strip():
+                base64_lines.append(line)
+        elif line.strip() == "":
+            seen_sep = True
+    tal.write_text("\n".join(rsync_uris) + "\n\n" + "\n".join(base64_lines) + "\n", encoding="utf-8")
+PY
+materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --repo-bytes-db "$REPO_BYTES_DB" --mirror-root "$MIRROR_ROOT")
+if [[ "$KEEP_DB" -eq 1 ]]; then
+  materialize_cmd+=(--keep-db)
+fi
+"${materialize_cmd[@]}"
+
+VALIDATION_TIME="$(python3 - <<'PY' "$META_JSON"
+import json,sys
+print(json.load(open(sys.argv[1]))["validationTime"])
+PY
+)"
+mapfile -t TAL_PATHS < <(python3 - <<'PY' "$META_JSON"
+import json, sys
+for item in json.load(open(sys.argv[1], encoding="utf-8"))["talFiles"]:
+    print(item["path"])
+PY
+)
+COMPARE_TRUST_ANCHOR="unknown"
+FAKE_EPOCH="$(python3 - <<'PY' "$VALIDATION_TIME"
+from datetime import datetime, timezone
+import sys
+dt = datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).astimezone(timezone.utc)
+print(int(dt.timestamp()))
+PY
+)"
+
+export CIR_MIRROR_ROOT="$(python3 - <<'PY' "$MIRROR_ROOT"
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
+export CIR_LOCAL_LINK_MODE=1
+env \
+  LD_PRELOAD="$FAKETIME_LIB" \
+  FAKETIME_FMT=%s \
+  FAKETIME="$FAKE_EPOCH" \
+  FAKETIME_DONT_FAKE_MONOTONIC=1 \
+  "$ROUTINATOR_BIN" \
+  --repository-dir "$WORK_REPO" \
+  --disable-rrdp \
+  --rsync-command "$WRAPPER" \
+  --no-rir-tals \
+  --extra-tals-dir "$TALS_DIR" \
+  --enable-aspa \
+  update --complete >"$RUN_LOG" 2>&1 || true
+
+env \
+  LD_PRELOAD="$FAKETIME_LIB" \
+  FAKETIME_FMT=%s \
+  FAKETIME="$FAKE_EPOCH" \
+  FAKETIME_DONT_FAKE_MONOTONIC=1 \
+  "$ROUTINATOR_BIN" \
+  --repository-dir "$WORK_REPO" \
+  --disable-rrdp \
+  --rsync-command "$WRAPPER" \
+  --no-rir-tals \
+  --extra-tals-dir "$TALS_DIR" \
+  --enable-aspa \
+  vrps --noupdate -o "$ACTUAL_VRPS" >>"$RUN_LOG" 2>&1
+
+env \
+  LD_PRELOAD="$FAKETIME_LIB" \
+  FAKETIME_FMT=%s \
+  FAKETIME="$FAKE_EPOCH" \
+  FAKETIME_DONT_FAKE_MONOTONIC=1 \
+  "$ROUTINATOR_BIN" \
+  --repository-dir "$WORK_REPO" \
+  --disable-rrdp \
+  --rsync-command "$WRAPPER" \
+  --no-rir-tals \
+  --extra-tals-dir "$TALS_DIR" \
+  --enable-aspa \
+  vrps --noupdate --format json -o "$ACTUAL_VAPS_JSON" >>"$RUN_LOG" 2>&1
+
+python3 "$JSON_TO_VAPS" --input "$ACTUAL_VAPS_JSON" --csv-out "$ACTUAL_VAPS"
+
+normalize_trust_anchor_csv() {
+  python3 - <<'PY' "$1" "$2"
+import csv
+import sys
+from pathlib import Path
+
+path = Path(sys.argv[1])
+trust_anchor = sys.argv[2]
+rows = list(csv.reader(path.open(newline="", encoding="utf-8")))
+if rows:
+    for row in rows[1:]:
+        if row:
+            row[-1] = trust_anchor
+with path.open("w", newline="", encoding="utf-8") as fh:
+    csv.writer(fh).writerows(rows)
+PY
+}
+
+normalize_trust_anchor_csv "$ACTUAL_VRPS" "$COMPARE_TRUST_ANCHOR"
+normalize_trust_anchor_csv "$ACTUAL_VAPS" "$COMPARE_TRUST_ANCHOR"
+"$CCR_TO_COMPARE_VIEWS_BIN" --ccr "$REFERENCE_CCR" --vrps-out "$REF_VRPS" --vaps-out "$REF_VAPS" --trust-anchor "$COMPARE_TRUST_ANCHOR"
+
+python3 - <<'PY' "$ACTUAL_VRPS" "$REF_VRPS" "$ACTUAL_VAPS" "$REF_VAPS" "$SUMMARY_JSON" "$META_JSON"
+import csv, json, sys
+def rows(path):
+    with open(path, newline="") as f:
+        return list(csv.reader(f))[1:]
+actual_vrps = {tuple(r) for r in rows(sys.argv[1])}
+ref_vrps = {tuple(r) for r in rows(sys.argv[2])}
+actual_vaps = {tuple(r) for r in rows(sys.argv[3])}
+ref_vaps = {tuple(r) for r in rows(sys.argv[4])}
+meta = json.load(open(sys.argv[6], encoding="utf-8"))
+summary = {
+  "compareMode": "trust-anchor-agnostic",
+  "talCount": len(meta["talFiles"]),
+  "talPaths": [item["path"] for item in meta["talFiles"]],
+  "vrps": {
+    "actual": len(actual_vrps),
+    "reference": len(ref_vrps),
+    "match": actual_vrps == ref_vrps,
+    "only_in_actual": sorted(actual_vrps - ref_vrps)[:20],
+    "only_in_reference": sorted(ref_vrps - actual_vrps)[:20],
+  },
+  "vaps": {
+    "actual": len(actual_vaps),
+    "reference": len(ref_vaps),
+    "match": actual_vaps == ref_vaps,
+    "only_in_actual": sorted(actual_vaps - ref_vaps)[:20],
+    "only_in_reference": sorted(ref_vaps - actual_vaps)[:20],
+  }
+}
+with open(sys.argv[5], "w") as f:
+    json.dump(summary, f, indent=2)
+PY
+
+if [[ "$KEEP_DB" -ne 1 ]]; then
+  rm -rf "$TMP_ROOT"
+fi
+
+echo "done: $OUT_DIR"

scripts/cir/run_cir_replay_rpki_client.sh

@@ -0,0 +1,248 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_replay_rpki_client.sh \
+    --cir <path> \
+    --repo-bytes-db <path> \
+    --out-dir <path> \
+    --reference-ccr <path> \
+    [--build-dir <path> | --rpki-client-bin <path>] \
+    [--keep-db] \
+    [--real-rsync-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+CIR=""
+REPO_BYTES_DB=""
+OUT_DIR=""
+REFERENCE_CCR=""
+BUILD_DIR=""
+RPKI_CLIENT_BIN=""
+KEEP_DB=0
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+CIR_MATERIALIZE_BIN="${CIR_MATERIALIZE_BIN:-$ROOT_DIR/target/release/cir_materialize}"
+CIR_EXTRACT_INPUTS_BIN="${CIR_EXTRACT_INPUTS_BIN:-$ROOT_DIR/target/release/cir_extract_inputs}"
+CCR_TO_COMPARE_VIEWS_BIN="${CCR_TO_COMPARE_VIEWS_BIN:-$ROOT_DIR/target/release/ccr_to_compare_views}"
+WRAPPER="$ROOT_DIR/scripts/cir/cir-rsync-wrapper"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --cir) CIR="$2"; shift 2 ;;
+    --repo-bytes-db) REPO_BYTES_DB="$2"; shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
+    --build-dir) BUILD_DIR="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --keep-db) KEEP_DB=1; shift ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$CIR" && -n "$REPO_BYTES_DB" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
+  usage >&2
+  exit 2
+}
+if [[ -z "$BUILD_DIR" && -z "$RPKI_CLIENT_BIN" ]]; then
+  usage >&2
+  exit 2
+fi
+if [[ -z "$RPKI_CLIENT_BIN" ]]; then
+  RPKI_CLIENT_BIN="$BUILD_DIR/src/rpki-client"
+fi
+if [[ ! -x "$RPKI_CLIENT_BIN" ]]; then
+  echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2
+  exit 2
+fi
+
+mkdir -p "$OUT_DIR"
+if [[ ! -x "$CIR_MATERIALIZE_BIN" || ! -x "$CIR_EXTRACT_INPUTS_BIN" || ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin cir_materialize --bin cir_extract_inputs --bin ccr_to_compare_views
+  )
+fi
+
+TMP_ROOT="$OUT_DIR/.tmp"
+TALS_DIR="$TMP_ROOT/tals"
+META_JSON="$TMP_ROOT/meta.json"
+MIRROR_ROOT="$TMP_ROOT/mirror"
+CACHE_DIR="$TMP_ROOT/cache"
+OUT_CCR_DIR="$TMP_ROOT/out"
+RUN_LOG="$OUT_DIR/rpki-client.log"
+ACTUAL_VRPS="$OUT_DIR/actual-vrps.csv"
+ACTUAL_VAPS="$OUT_DIR/actual-vaps.csv"
+ACTUAL_VAPS_META="$OUT_DIR/actual-vaps-meta.json"
+ACTUAL_VRPS_META="$OUT_DIR/actual-vrps-meta.json"
+REF_VRPS="$OUT_DIR/reference-vrps.csv"
+REF_VAPS="$OUT_DIR/reference-vaps.csv"
+SUMMARY_JSON="$OUT_DIR/compare-summary.json"
+
+rm -rf "$TMP_ROOT"
+mkdir -p "$TMP_ROOT"
+
+"$CIR_EXTRACT_INPUTS_BIN" --cir "$CIR" --tals-dir "$TALS_DIR" --meta-json "$META_JSON"
+python3 - <<'PY' "$TALS_DIR"
+from pathlib import Path
+import sys
+for tal in Path(sys.argv[1]).glob("*.tal"):
+    lines = tal.read_text(encoding="utf-8").splitlines()
+    rsync_uris = [line for line in lines if line.startswith("rsync://")]
+    base64_lines = []
+    seen_sep = False
+    for line in lines:
+        if seen_sep:
+            if line.strip():
+                base64_lines.append(line)
+        elif line.strip() == "":
+            seen_sep = True
+    tal.write_text("\n".join(rsync_uris) + "\n\n" + "\n".join(base64_lines) + "\n", encoding="utf-8")
+PY
+materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --repo-bytes-db "$REPO_BYTES_DB" --mirror-root "$MIRROR_ROOT")
+if [[ "$KEEP_DB" -eq 1 ]]; then
+  materialize_cmd+=(--keep-db)
+fi
+"${materialize_cmd[@]}"
+
+VALIDATION_EPOCH="$(python3 - <<'PY' "$META_JSON"
+from datetime import datetime, timezone
+import json, sys
+vt = json.load(open(sys.argv[1]))["validationTime"]
+dt = datetime.fromisoformat(vt.replace("Z", "+00:00")).astimezone(timezone.utc)
+print(int(dt.timestamp()))
+PY
+)"
+mapfile -t TAL_PATHS < <(python3 - <<'PY' "$META_JSON"
+import json, sys
+for item in json.load(open(sys.argv[1], encoding="utf-8"))["talFiles"]:
+    print(item["path"])
+PY
+)
+CLIENT_TAL_ARGS=()
+for tal_path in "${TAL_PATHS[@]}"; do
+  CLIENT_TAL_ARGS+=(-t "$tal_path")
+done
+COMPARE_TRUST_ANCHOR="unknown"
+
+export CIR_MIRROR_ROOT="$(python3 - <<'PY' "$MIRROR_ROOT"
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
+export CIR_LOCAL_LINK_MODE=1
+
+mkdir -p "$CACHE_DIR" "$OUT_CCR_DIR"
+chmod -R 0777 "$TMP_ROOT"
+"$RPKI_CLIENT_BIN" \
+  -R \
+  -e "$WRAPPER" \
+  -P "$VALIDATION_EPOCH" \
+  "${CLIENT_TAL_ARGS[@]}" \
+  -d "$CACHE_DIR" \
+  "$OUT_CCR_DIR" >"$RUN_LOG" 2>&1
+
+if [[ -f "$OUT_CCR_DIR/rpki.ccr" ]]; then
+  "$CCR_TO_COMPARE_VIEWS_BIN" \
+    --ccr "$OUT_CCR_DIR/rpki.ccr" \
+    --vrps-out "$ACTUAL_VRPS" \
+    --vaps-out "$ACTUAL_VAPS" \
+    --trust-anchor "$COMPARE_TRUST_ANCHOR"
+else
+  python3 - <<'PY' "$OUT_CCR_DIR/json" "$ACTUAL_VRPS" "$ACTUAL_VAPS" "$COMPARE_TRUST_ANCHOR"
+import csv
+import json
+import sys
+from pathlib import Path
+
+json_path = Path(sys.argv[1])
+vrps_out = Path(sys.argv[2])
+vaps_out = Path(sys.argv[3])
+compare_ta = sys.argv[4]
+
+if not json_path.is_file():
+    raise SystemExit(f"rpki-client output has neither rpki.ccr nor json: {json_path}")
+
+data = json.loads(json_path.read_text(encoding="utf-8"))
+
+vrps_out.parent.mkdir(parents=True, exist_ok=True)
+with vrps_out.open("w", newline="", encoding="utf-8") as fh:
+    writer = csv.writer(fh)
+    writer.writerow(["ASN", "IP Prefix", "Max Length", "Trust Anchor"])
+    for roa in data.get("roas", []):
+        writer.writerow([
+            f"AS{roa['asn']}",
+            roa["prefix"],
+            str(roa["maxLength"]),
+            compare_ta,
+        ])
+
+with vaps_out.open("w", newline="", encoding="utf-8") as fh:
+    writer = csv.writer(fh)
+    writer.writerow(["Customer ASN", "Providers", "Trust Anchor"])
+    for aspa in data.get("aspas", []):
+        providers = ";".join(f"AS{item}" for item in sorted(aspa.get("providers", [])))
+        writer.writerow([
+            f"AS{aspa['customer_asid']}",
+            providers,
+            compare_ta,
+        ])
+PY
+fi
+
+python3 - <<'PY' "$ACTUAL_VRPS" "$ACTUAL_VAPS" "$ACTUAL_VRPS_META" "$ACTUAL_VAPS_META"
+import csv, json, sys
+def count_rows(path):
+    with open(path, newline="") as f:
+        rows = list(csv.reader(f))
+    return max(len(rows) - 1, 0)
+json.dump({"count": count_rows(sys.argv[1])}, open(sys.argv[3], "w"), indent=2)
+json.dump({"count": count_rows(sys.argv[2])}, open(sys.argv[4], "w"), indent=2)
+PY
+
+"$CCR_TO_COMPARE_VIEWS_BIN" --ccr "$REFERENCE_CCR" --vrps-out "$REF_VRPS" --vaps-out "$REF_VAPS" --trust-anchor "$COMPARE_TRUST_ANCHOR"
+
+python3 - <<'PY' "$ACTUAL_VRPS" "$REF_VRPS" "$ACTUAL_VAPS" "$REF_VAPS" "$SUMMARY_JSON" "$META_JSON"
+import csv, json, sys
+def rows(path):
+    with open(path, newline="") as f:
+        return list(csv.reader(f))[1:]
+actual_vrps = {tuple(r) for r in rows(sys.argv[1])}
+ref_vrps = {tuple(r) for r in rows(sys.argv[2])}
+actual_vaps = {tuple(r) for r in rows(sys.argv[3])}
+ref_vaps = {tuple(r) for r in rows(sys.argv[4])}
+meta = json.load(open(sys.argv[6], encoding="utf-8"))
+summary = {
+  "compareMode": "trust-anchor-agnostic",
+  "talCount": len(meta["talFiles"]),
+  "talPaths": [item["path"] for item in meta["talFiles"]],
+  "vrps": {
+    "actual": len(actual_vrps),
+    "reference": len(ref_vrps),
+    "match": actual_vrps == ref_vrps,
+    "only_in_actual": sorted(actual_vrps - ref_vrps)[:20],
+    "only_in_reference": sorted(ref_vrps - actual_vrps)[:20],
+  },
+  "vaps": {
+    "actual": len(actual_vaps),
+    "reference": len(ref_vaps),
+    "match": actual_vaps == ref_vaps,
+    "only_in_actual": sorted(actual_vaps - ref_vaps)[:20],
+    "only_in_reference": sorted(ref_vaps - actual_vaps)[:20],
+  }
+}
+with open(sys.argv[5], "w") as f:
+    json.dump(summary, f, indent=2)
+PY
+
+if [[ "$KEEP_DB" -ne 1 ]]; then
+  rm -rf "$TMP_ROOT"
+fi
+
+echo "done: $OUT_DIR"

scripts/cir/run_cir_replay_sequence_ours.sh

@@ -0,0 +1,147 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_replay_sequence_ours.sh \
+    --sequence-root <path> \
+    [--rpki-bin <path>] \
+    [--real-rsync-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SEQUENCE_ROOT=""
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+STEP_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_ours.sh"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --sequence-root) SEQUENCE_ROOT="$2"; shift 2 ;;
+    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$SEQUENCE_ROOT" ]] || { usage >&2; exit 2; }
+
+SEQUENCE_ROOT="$(python3 - <<'PY' "$SEQUENCE_ROOT"
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+
+SUMMARY_JSON="$SEQUENCE_ROOT/sequence-summary.json"
+SUMMARY_MD="$SEQUENCE_ROOT/sequence-summary.md"
+DETAIL_JSON="$SEQUENCE_ROOT/sequence-detail.json"
+
+python3 - <<'PY' "$SEQUENCE_ROOT" "$SUMMARY_JSON" "$SUMMARY_MD" "$DETAIL_JSON" "$STEP_SCRIPT" "$RPKI_BIN" "$REAL_RSYNC_BIN"
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+sequence_root = Path(sys.argv[1])
+summary_json = Path(sys.argv[2])
+summary_md = Path(sys.argv[3])
+detail_json = Path(sys.argv[4])
+step_script = Path(sys.argv[5])
+rpki_bin = sys.argv[6]
+real_rsync_bin = sys.argv[7]
+
+sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
+repo_bytes_db = sequence_root / sequence["repoBytesDbPath"]
+steps = sequence["steps"]
+
+results = []
+all_match = True
+for step in steps:
+    step_id = step["stepId"]
+    out_dir = sequence_root / "replay-ours" / step_id
+    out_dir.parent.mkdir(parents=True, exist_ok=True)
+    cmd = [
+        str(step_script),
+        "--cir",
+        str(sequence_root / step["cirPath"]),
+        "--out-dir",
+        str(out_dir),
+        "--reference-ccr",
+        str(sequence_root / step["ccrPath"]),
+        "--rpki-bin",
+        rpki_bin,
+        "--real-rsync-bin",
+        real_rsync_bin,
+    ]
+    cmd.extend(["--repo-bytes-db", str(repo_bytes_db)])
+    proc = subprocess.run(cmd, capture_output=True, text=True)
+    if proc.returncode != 0:
+        raise SystemExit(
+            f"ours sequence replay failed for {step_id}: stdout={proc.stdout} stderr={proc.stderr}"
+        )
+    compare = json.loads((out_dir / "compare-summary.json").read_text(encoding="utf-8"))
+    timing = json.loads((out_dir / "timing.json").read_text(encoding="utf-8")) if (out_dir / "timing.json").exists() else {}
+    record = {
+        "stepId": step_id,
+        "kind": step["kind"],
+        "validationTime": step["validationTime"],
+        "outDir": str(out_dir),
+        "comparePath": str(out_dir / "compare-summary.json"),
+        "timingPath": str(out_dir / "timing.json"),
+        "compareMode": compare.get("compareMode"),
+        "talCount": compare.get("talCount"),
+        "talPaths": compare.get("talPaths", []),
+        "compare": compare,
+        "timing": timing,
+        "match": bool(compare["vrps"]["match"]) and bool(compare["vaps"]["match"]),
+    }
+    all_match = all_match and record["match"]
+    results.append(record)
+
+summary = {
+    "version": 1,
+    "participant": "ours",
+    "sequenceRoot": str(sequence_root),
+    "stepCount": len(results),
+    "allMatch": all_match,
+    "steps": results,
+}
+summary_json.write_text(json.dumps(summary, indent=2), encoding="utf-8")
+detail_json.write_text(json.dumps(results, indent=2), encoding="utf-8")
+
+lines = [
+    "# Ours CIR Sequence Replay Summary",
+    "",
+    f"- `sequence_root`: `{sequence_root}`",
+    f"- `step_count`: `{len(results)}`",
+    f"- `all_match`: `{all_match}`",
+    "",
+    "| Step | Kind | TALs | Compare mode | VRP actual/ref | VRP match | VAP actual/ref | VAP match | Duration (ms) |",
+    "| --- | --- | ---: | --- | --- | --- | --- | --- | ---: |",
+]
+for item in results:
+    compare = item["compare"]
+    timing = item.get("timing") or {}
+    lines.append(
+        "| {step} | {kind} | {tal_count} | {compare_mode} | {va}/{vr} | {vm} | {aa}/{ar} | {am} | {dur} |".format(
+            step=item["stepId"],
+            kind=item["kind"],
+            tal_count=item.get("talCount") if item.get("talCount") is not None else "-",
+            compare_mode=item.get("compareMode") or "-",
+            va=compare["vrps"]["actual"],
+            vr=compare["vrps"]["reference"],
+            vm=compare["vrps"]["match"],
+            aa=compare["vaps"]["actual"],
+            ar=compare["vaps"]["reference"],
+            am=compare["vaps"]["match"],
+            dur=timing.get("durationMs", "-"),
+        )
+    )
+summary_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
+PY
+
+echo "done: $SEQUENCE_ROOT"

scripts/cir/run_cir_replay_sequence_routinator.sh

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_replay_sequence_routinator.sh \
+    --sequence-root <path> \
+    [--routinator-root <path>] \
+    [--routinator-bin <path>] \
+    [--real-rsync-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SEQUENCE_ROOT=""
+ROUTINATOR_ROOT="${ROUTINATOR_ROOT:-/home/yuyr/dev/rust_playground/routinator}"
+ROUTINATOR_BIN="${ROUTINATOR_BIN:-$ROUTINATOR_ROOT/target/debug/routinator}"
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+STEP_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_routinator.sh"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --sequence-root) SEQUENCE_ROOT="$2"; shift 2 ;;
+    --routinator-root) ROUTINATOR_ROOT="$2"; shift 2 ;;
+    --routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$SEQUENCE_ROOT" ]] || { usage >&2; exit 2; }
+
+SEQUENCE_ROOT="$(python3 - <<'PY' "$SEQUENCE_ROOT"
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+
+SUMMARY_JSON="$SEQUENCE_ROOT/sequence-summary-routinator.json"
+SUMMARY_MD="$SEQUENCE_ROOT/sequence-summary-routinator.md"
+
+python3 - <<'PY' "$SEQUENCE_ROOT" "$SUMMARY_JSON" "$SUMMARY_MD" "$STEP_SCRIPT" "$ROUTINATOR_ROOT" "$ROUTINATOR_BIN" "$REAL_RSYNC_BIN"
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+sequence_root = Path(sys.argv[1])
+summary_json = Path(sys.argv[2])
+summary_md = Path(sys.argv[3])
+step_script = Path(sys.argv[4])
+routinator_root = sys.argv[5]
+routinator_bin = sys.argv[6]
+real_rsync_bin = sys.argv[7]
+
+sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
+repo_bytes_db = sequence_root / sequence["repoBytesDbPath"]
+steps = sequence["steps"]
+results = []
+all_match = True
+
+for step in steps:
+    step_id = step["stepId"]
+    out_dir = sequence_root / "replay-routinator" / step_id
+    out_dir.parent.mkdir(parents=True, exist_ok=True)
+    cmd = [
+        str(step_script),
+        "--cir",
+        str(sequence_root / step["cirPath"]),
+        "--out-dir",
+        str(out_dir),
+        "--reference-ccr",
+        str(sequence_root / step["ccrPath"]),
+        "--routinator-root",
+        routinator_root,
+        "--routinator-bin",
+        routinator_bin,
+        "--real-rsync-bin",
+        real_rsync_bin,
+    ]
+    cmd.extend(["--repo-bytes-db", str(repo_bytes_db)])
+    proc = subprocess.run(cmd, capture_output=True, text=True)
+    if proc.returncode != 0:
+        raise SystemExit(
+            f"routinator sequence replay failed for {step_id}: stdout={proc.stdout} stderr={proc.stderr}"
+        )
+    compare = json.loads((out_dir / "compare-summary.json").read_text(encoding="utf-8"))
+    match = bool(compare["vrps"]["match"]) and bool(compare["vaps"]["match"])
+    all_match = all_match and match
+    results.append(
+        {
+            "stepId": step_id,
+            "kind": step["kind"],
+            "validationTime": step["validationTime"],
+            "outDir": str(out_dir),
+            "comparePath": str(out_dir / "compare-summary.json"),
+            "compareMode": compare.get("compareMode"),
+            "talCount": compare.get("talCount"),
+            "talPaths": compare.get("talPaths", []),
+            "match": match,
+            "compare": compare,
+        }
+    )
+
+summary = {
+    "version": 1,
+    "participant": "routinator",
+    "sequenceRoot": str(sequence_root),
+    "stepCount": len(results),
+    "allMatch": all_match,
+    "steps": results,
+}
+summary_json.write_text(json.dumps(summary, indent=2), encoding="utf-8")
+lines = [
+    "# Routinator CIR Sequence Replay Summary",
+    "",
+    f"- `sequence_root`: `{sequence_root}`",
+    f"- `step_count`: `{len(results)}`",
+    f"- `all_match`: `{all_match}`",
+    "",
+    "| Step | Kind | TALs | Compare mode | VRP actual/ref | VRP match | VAP actual/ref | VAP match |",
+    "| --- | --- | ---: | --- | --- | --- | --- | --- |",
+]
+for item in results:
+    compare = item["compare"]
+    lines.append(
+        "| {step} | {kind} | {tal_count} | {compare_mode} | {va}/{vr} | {vm} | {aa}/{ar} | {am} |".format(
+            step=item["stepId"],
+            kind=item["kind"],
+            tal_count=item.get("talCount") if item.get("talCount") is not None else "-",
+            compare_mode=item.get("compareMode") or "-",
+            va=compare["vrps"]["actual"],
+            vr=compare["vrps"]["reference"],
+            vm=compare["vrps"]["match"],
+            aa=compare["vaps"]["actual"],
+            ar=compare["vaps"]["reference"],
+            am=compare["vaps"]["match"],
+        )
+    )
+summary_md.write_text("\n".join(lines), encoding="utf-8")
+PY
+
+echo "done: $SEQUENCE_ROOT"

scripts/cir/run_cir_replay_sequence_rpki_client.sh

@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_replay_sequence_rpki_client.sh \
+    --sequence-root <path> \
+    [--build-dir <path> | --rpki-client-bin <path>] \
+    [--real-rsync-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SEQUENCE_ROOT=""
+BUILD_DIR=""
+RPKI_CLIENT_BIN=""
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+STEP_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_rpki_client.sh"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --sequence-root) SEQUENCE_ROOT="$2"; shift 2 ;;
+    --build-dir) BUILD_DIR="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$SEQUENCE_ROOT" && ( -n "$BUILD_DIR" || -n "$RPKI_CLIENT_BIN" ) ]] || { usage >&2; exit 2; }
+
+SEQUENCE_ROOT="$(python3 - <<'PY' "$SEQUENCE_ROOT"
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+
+SUMMARY_JSON="$SEQUENCE_ROOT/sequence-summary-rpki-client.json"
+SUMMARY_MD="$SEQUENCE_ROOT/sequence-summary-rpki-client.md"
+
+python3 - <<'PY' "$SEQUENCE_ROOT" "$SUMMARY_JSON" "$SUMMARY_MD" "$STEP_SCRIPT" "$BUILD_DIR" "$RPKI_CLIENT_BIN" "$REAL_RSYNC_BIN"
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+sequence_root = Path(sys.argv[1])
+summary_json = Path(sys.argv[2])
+summary_md = Path(sys.argv[3])
+step_script = Path(sys.argv[4])
+build_dir = sys.argv[5]
+rpki_client_bin = sys.argv[6]
+real_rsync_bin = sys.argv[7]
+
+sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
+repo_bytes_db = sequence_root / sequence["repoBytesDbPath"]
+steps = sequence["steps"]
+results = []
+all_match = True
+
+for step in steps:
+    step_id = step["stepId"]
+    out_dir = sequence_root / "replay-rpki-client" / step_id
+    out_dir.parent.mkdir(parents=True, exist_ok=True)
+    cmd = [
+        str(step_script),
+        "--cir",
+        str(sequence_root / step["cirPath"]),
+        "--out-dir",
+        str(out_dir),
+        "--reference-ccr",
+        str(sequence_root / step["ccrPath"]),
+        "--real-rsync-bin",
+        real_rsync_bin,
+    ]
+    if rpki_client_bin:
+        cmd.extend(["--rpki-client-bin", rpki_client_bin])
+    else:
+        cmd.extend(["--build-dir", build_dir])
+    cmd.extend(["--repo-bytes-db", str(repo_bytes_db)])
+    proc = subprocess.run(cmd, capture_output=True, text=True)
+    if proc.returncode != 0:
+        raise SystemExit(
+            f"rpki-client sequence replay failed for {step_id}: stdout={proc.stdout} stderr={proc.stderr}"
+        )
+    compare = json.loads((out_dir / "compare-summary.json").read_text(encoding="utf-8"))
+    match = bool(compare["vrps"]["match"]) and bool(compare["vaps"]["match"])
+    all_match = all_match and match
+    results.append(
+        {
+            "stepId": step_id,
+            "kind": step["kind"],
+            "validationTime": step["validationTime"],
+            "outDir": str(out_dir),
+            "comparePath": str(out_dir / "compare-summary.json"),
+            "compareMode": compare.get("compareMode"),
+            "talCount": compare.get("talCount"),
+            "talPaths": compare.get("talPaths", []),
+            "match": match,
+            "compare": compare,
+        }
+    )
+
+summary = {
+    "version": 1,
+    "participant": "rpki-client",
+    "sequenceRoot": str(sequence_root),
+    "stepCount": len(results),
+    "allMatch": all_match,
+    "steps": results,
+}
+summary_json.write_text(json.dumps(summary, indent=2), encoding="utf-8")
+lines = [
+    "# rpki-client CIR Sequence Replay Summary",
+    "",
+    f"- `sequence_root`: `{sequence_root}`",
+    f"- `step_count`: `{len(results)}`",
+    f"- `all_match`: `{all_match}`",
+    "",
+    "| Step | Kind | TALs | Compare mode | VRP actual/ref | VRP match | VAP actual/ref | VAP match |",
+    "| --- | --- | ---: | --- | --- | --- | --- | --- |",
+]
+for item in results:
+    compare = item["compare"]
+    lines.append(
+        "| {step} | {kind} | {tal_count} | {compare_mode} | {va}/{vr} | {vm} | {aa}/{ar} | {am} |".format(
+            step=item["stepId"],
+            kind=item["kind"],
+            tal_count=item.get("talCount") if item.get("talCount") is not None else "-",
+            compare_mode=item.get("compareMode") or "-",
+            va=compare["vrps"]["actual"],
+            vr=compare["vrps"]["reference"],
+            vm=compare["vrps"]["match"],
+            aa=compare["vaps"]["actual"],
+            ar=compare["vaps"]["reference"],
+            am=compare["vaps"]["match"],
+        )
+    )
+summary_md.write_text("\n".join(lines), encoding="utf-8")
+PY
+
+echo "done: $SEQUENCE_ROOT"

scripts/cir/run_cir_sequence_matrix_multi_rir.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/cir/run_cir_sequence_matrix_multi_rir.sh \
+    --root <path> \
+    [--rir <afrinic,apnic,arin,lacnic,ripe>] \
+    [--rpki-bin <path>] \
+    [--routinator-root <path>] \
+    [--routinator-bin <path>] \
+    [--rpki-client-build-dir <path>] \
+    [--drop-bin <path>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+ROOT=""
+RIRS="afrinic,apnic,arin,lacnic,ripe"
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+ROUTINATOR_ROOT="${ROUTINATOR_ROOT:-/home/yuyr/dev/rust_playground/routinator}"
+ROUTINATOR_BIN="${ROUTINATOR_BIN:-$ROUTINATOR_ROOT/target/debug/routinator}"
+RPKI_CLIENT_BUILD_DIR="${RPKI_CLIENT_BUILD_DIR:-/home/yuyr/dev/rpki-client-9.7/build-m5}"
+DROP_BIN="${DROP_BIN:-$ROOT_DIR/target/release/cir_drop_report}"
+
+OURS_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_sequence_ours.sh"
+ROUTINATOR_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_sequence_routinator.sh"
+RPKIC_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_sequence_rpki_client.sh"
+DROP_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_drop_sequence.sh"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --root) ROOT="$2"; shift 2 ;;
+    --rir) RIRS="$2"; shift 2 ;;
+    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
+    --routinator-root) ROUTINATOR_ROOT="$2"; shift 2 ;;
+    --routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
+    --rpki-client-build-dir) RPKI_CLIENT_BUILD_DIR="$2"; shift 2 ;;
+    --drop-bin) DROP_BIN="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$ROOT" ]] || { usage >&2; exit 2; }
+
+SUMMARY_JSON="$ROOT/final-summary.json"
+SUMMARY_MD="$ROOT/final-summary.md"
+IFS=',' read -r -a ITEMS <<< "$RIRS"
+
+results=()
+for rir in "${ITEMS[@]}"; do
+  seq_root="$ROOT/$rir"
+  "$OURS_SCRIPT" --sequence-root "$seq_root" --rpki-bin "$RPKI_BIN"
+  "$ROUTINATOR_SCRIPT" --sequence-root "$seq_root" --routinator-root "$ROUTINATOR_ROOT" --routinator-bin "$ROUTINATOR_BIN"
+  "$RPKIC_SCRIPT" --sequence-root "$seq_root" --build-dir "$RPKI_CLIENT_BUILD_DIR"
+  "$DROP_SCRIPT" --sequence-root "$seq_root" --drop-bin "$DROP_BIN"
+done
+
+python3 - <<'PY' "$ROOT" "$RIRS" "$SUMMARY_JSON" "$SUMMARY_MD"
+import json, sys
+from pathlib import Path
+from collections import Counter
+
+root = Path(sys.argv[1]).resolve()
+rirs = [item for item in sys.argv[2].split(',') if item]
+summary_json = Path(sys.argv[3])
+summary_md = Path(sys.argv[4])
+items = []
+total_steps = 0
+total_dropped_vrps = 0
+total_dropped_objects = 0
+reason_counter = Counter()
+for rir in rirs:
+    seq_root = root / rir
+    ours = json.loads((seq_root / "sequence-summary.json").read_text(encoding="utf-8"))
+    routinator = json.loads((seq_root / "sequence-summary-routinator.json").read_text(encoding="utf-8"))
+    rpki_client = json.loads((seq_root / "sequence-summary-rpki-client.json").read_text(encoding="utf-8"))
+    drop = json.loads((seq_root / "drop-summary.json").read_text(encoding="utf-8"))
+    step_count = len(ours["steps"])
+    total_steps += step_count
+    rir_dropped_vrps = 0
+    rir_dropped_objects = 0
+    for step in drop["steps"]:
+        drop_path = Path(step["reportPath"])
+        detail = json.loads(drop_path.read_text(encoding="utf-8"))
+        summary = detail.get("summary", {})
+        rir_dropped_vrps += int(summary.get("droppedVrpCount", 0))
+        rir_dropped_objects += int(summary.get("droppedObjectCount", 0))
+        total_dropped_vrps += int(summary.get("droppedVrpCount", 0))
+        total_dropped_objects += int(summary.get("droppedObjectCount", 0))
+        for reason, count in summary.get("droppedByReason", {}).items():
+            reason_counter[reason] += int(count)
+    items.append({
+        "rir": rir,
+        "stepCount": step_count,
+        "oursAllMatch": ours["allMatch"],
+        "routinatorAllMatch": routinator["allMatch"],
+        "rpkiClientAllMatch": rpki_client["allMatch"],
+        "dropSummary": drop["steps"],
+        "droppedVrpCount": rir_dropped_vrps,
+        "droppedObjectCount": rir_dropped_objects,
+    })
+summary = {
+    "version": 1,
+    "totalStepCount": total_steps,
+    "totalDroppedVrpCount": total_dropped_vrps,
+    "totalDroppedObjectCount": total_dropped_objects,
+    "topReasons": [{"reason": reason, "count": count} for reason, count in reason_counter.most_common(10)],
+    "rirs": items,
+}
+summary_json.write_text(json.dumps(summary, indent=2), encoding="utf-8")
+lines = ["# Multi-RIR CIR Sequence Matrix Summary", ""]
+lines.append(f"- `total_step_count`: `{total_steps}`")
+lines.append(f"- `total_dropped_vrps`: `{total_dropped_vrps}`")
+lines.append(f"- `total_dropped_objects`: `{total_dropped_objects}`")
+lines.append("")
+if reason_counter:
+    lines.append("## Top Drop Reasons")
+    lines.append("")
+    for reason, count in reason_counter.most_common(10):
+        lines.append(f"- `{reason}`: `{count}`")
+    lines.append("")
+for item in items:
+    lines.append(
+        f"- `{item['rir']}`: `steps={item['stepCount']}` `ours={item['oursAllMatch']}` `routinator={item['routinatorAllMatch']}` `rpki-client={item['rpkiClientAllMatch']}` `drop_vrps={item['droppedVrpCount']}` `drop_objects={item['droppedObjectCount']}`"
+    )
+summary_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
+PY
+
+echo "done: $ROOT"

scripts/compare/run_perf_compare_quick_remote.sh

@@ -0,0 +1,502 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/compare/run_perf_compare_quick_remote.sh \
+    --run-root <path> \
+    --remote-root <path> \
+    [--rir-set <mixed2|all5>] \
+    [--ssh-target <user@host>] \
+    [--rpki-client-bin <path>] \
+    [--libtls-path <path>] \
+    [--rp-run-mode <serial|parallel>] \
+    [--copy-rpki-client-cache] \
+    [--probe-rpki-client-cache] \
+    [--ours-extra-args '<args>'] \
+    [--dry-run]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+first_existing_executable() {
+  local fallback="$1"
+  shift
+  local candidate
+  for candidate in "$@"; do
+    if [[ -x "$candidate" ]]; then
+      printf '%s' "$candidate"
+      return
+    fi
+  done
+  printf '%s' "$fallback"
+}
+
+first_existing_file() {
+  local fallback="$1"
+  shift
+  local candidate
+  for candidate in "$@"; do
+    if [[ -f "$candidate" ]]; then
+      printf '%s' "$candidate"
+      return
+    fi
+  done
+  printf '%s' "$fallback"
+}
+
+RUN_ROOT=""
+REMOTE_ROOT=""
+SSH_TARGET="${SSH_TARGET:-root@47.251.56.108}"
+RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-$(first_existing_executable \
+  "/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client" \
+  "$ROOT_DIR/../../.cache/rpki-client-9.7-build/bin/rpki-client" \
+  "$ROOT_DIR/../../.cache/rpki-client-9.7-build/src/rpki-client-9.7/src/rpki-client" \
+  "$ROOT_DIR/../../.cache/rpki-client-remote9.0/rpki-client" \
+  "/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client")}"
+LIBTLS_PATH="${LIBTLS_PATH:-$(first_existing_file \
+  "/home/yuyr/dev/rpki-client-9.7/.deps/libtls/root/usr/lib/x86_64-linux-gnu/libtls.so.28.0.0" \
+  "$ROOT_DIR/../../.cache/rpki-client-9.7-build/runlib/libtls.so.28" \
+  "$ROOT_DIR/../../.cache/rpki-client-9.7-build/sysroot/usr/lib/x86_64-linux-gnu/libtls.so.28.0.0" \
+  "$ROOT_DIR/../../.cache/rpki-client-remote9.0/libtls.so.28" \
+  "/home/yuyr/dev/rpki-client-9.7/.deps/libtls/root/usr/lib/x86_64-linux-gnu/libtls.so.28.0.0")}"
+RP_RUN_MODE="${RP_RUN_MODE:-serial}"
+RIR_SET="${RIR_SET:-mixed2}"
+OURS_EXTRA_ARGS="${OURS_EXTRA_ARGS:-}"
+COPY_RPKI_CLIENT_CACHE="${COPY_RPKI_CLIENT_CACHE:-0}"
+PROBE_RPKI_CLIENT_CACHE="${PROBE_RPKI_CLIENT_CACHE:-0}"
+DRY_RUN=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --run-root) RUN_ROOT="$2"; shift 2 ;;
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --rir-set) RIR_SET="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --libtls-path) LIBTLS_PATH="$2"; shift 2 ;;
+    --rp-run-mode) RP_RUN_MODE="$2"; shift 2 ;;
+    --copy-rpki-client-cache) COPY_RPKI_CLIENT_CACHE=1; shift ;;
+    --probe-rpki-client-cache) PROBE_RPKI_CLIENT_CACHE=1; shift ;;
+    --ours-extra-args) OURS_EXTRA_ARGS="$2"; shift 2 ;;
+    --dry-run) DRY_RUN=1; shift ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$RUN_ROOT" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
+[[ "$RP_RUN_MODE" == "serial" || "$RP_RUN_MODE" == "parallel" ]] || { echo "invalid --rp-run-mode: $RP_RUN_MODE" >&2; usage; exit 2; }
+[[ "$RIR_SET" == "mixed2" || "$RIR_SET" == "all5" ]] || { echo "invalid --rir-set: $RIR_SET" >&2; usage; exit 2; }
+[[ "$DRY_RUN" -eq 1 || -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
+[[ "$DRY_RUN" -eq 1 || -f "$LIBTLS_PATH" ]] || { echo "libtls not found: $LIBTLS_PATH" >&2; exit 2; }
+
+RUN_ROOT="$(python3 - <<'PY' "$RUN_ROOT"
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+
+mkdir -p "$RUN_ROOT/steps/step-001/ours" "$RUN_ROOT/steps/step-001/rpki-client" "$RUN_ROOT/steps/step-001/compare"
+mkdir -p "$RUN_ROOT/steps/step-002/ours" "$RUN_ROOT/steps/step-002/rpki-client" "$RUN_ROOT/steps/step-002/compare"
+
+tal_path_for_rir() {
+  case "$1" in
+    afrinic) printf '%s' "$ROOT_DIR/tests/fixtures/tal/afrinic.tal" ;;
+    apnic) printf '%s' "$ROOT_DIR/tests/fixtures/tal/apnic-rfc7730-https.tal" ;;
+    arin) printf '%s' "$ROOT_DIR/tests/fixtures/tal/arin.tal" ;;
+    lacnic) printf '%s' "$ROOT_DIR/tests/fixtures/tal/lacnic.tal" ;;
+    ripe) printf '%s' "$ROOT_DIR/tests/fixtures/tal/ripe-ncc.tal" ;;
+    *) echo "unknown rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+ta_path_for_rir() {
+  case "$1" in
+    afrinic) printf '%s' "$ROOT_DIR/tests/fixtures/ta/afrinic-ta.cer" ;;
+    apnic) printf '%s' "$ROOT_DIR/tests/fixtures/ta/apnic-ta.cer" ;;
+    arin) printf '%s' "$ROOT_DIR/tests/fixtures/ta/arin-ta.cer" ;;
+    lacnic) printf '%s' "$ROOT_DIR/tests/fixtures/ta/lacnic-ta.cer" ;;
+    ripe) printf '%s' "$ROOT_DIR/tests/fixtures/ta/ripe-ncc-ta.cer" ;;
+    *) echo "unknown rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+case "$RIR_SET" in
+  mixed2)
+    RIRS=(apnic arin)
+    SCOPE_LABEL="APNIC+ARIN mixed release two-step synchronized compare"
+    ;;
+  all5)
+    RIRS=(afrinic apnic arin lacnic ripe)
+    SCOPE_LABEL="all-five-RIR mixed release two-step synchronized compare"
+    ;;
+esac
+
+COPY_FILES=()
+for rir in "${RIRS[@]}"; do
+  COPY_FILES+=("$(tal_path_for_rir "$rir")" "$(ta_path_for_rir "$rir")")
+done
+
+if [[ "$DRY_RUN" -eq 1 ]]; then
+  cat <<EOF2
+workflow_name=性能对比测试快速版
+scope=$SCOPE_LABEL
+rir_set=$RIR_SET
+rirs=${RIRS[*]}
+run_root=$RUN_ROOT
+remote_root=$REMOTE_ROOT
+ssh_target=$SSH_TARGET
+rp_run_mode=$RP_RUN_MODE
+ours_extra_args=$OURS_EXTRA_ARGS
+EOF2
+  exit 0
+fi
+
+cleanup_remote() {
+  if [[ "${KEEP_REMOTE:-0}" != "1" ]]; then
+    ssh "$SSH_TARGET" "rm -rf '$REMOTE_ROOT'" >/dev/null 2>&1 || true
+  fi
+}
+trap cleanup_remote EXIT
+
+(
+  cd "$ROOT_DIR"
+  cargo build --release --bin rpki --bin ccr_to_compare_views --bin ccr_state_compare --bin cir_state_compare --bin cir_probe_rpki_client_cache
+)
+
+ssh "$SSH_TARGET" "set -e; systemctl disable --now rpki-client.timer >/dev/null 2>&1 || true; systemctl stop rpki-client.service >/dev/null 2>&1 || true; pkill -f '[/]rpki-client([[:space:]]|$)' >/dev/null 2>&1 || true; pkill -f '[/]routinator([[:space:]]|$)' >/dev/null 2>&1 || true; id -u _rpki-client >/dev/null 2>&1 || useradd -r -M -s /usr/sbin/nologin _rpki-client || true; rm -rf '$REMOTE_ROOT'; mkdir -p '$REMOTE_ROOT/bin' '$REMOTE_ROOT/lib' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client' '$REMOTE_ROOT/steps/step-001/ours' '$REMOTE_ROOT/steps/step-001/rpki-client' '$REMOTE_ROOT/steps/step-002/ours' '$REMOTE_ROOT/steps/step-002/rpki-client'"
+scp "$ROOT_DIR/target/release/rpki" "${COPY_FILES[@]}" "$SSH_TARGET:$REMOTE_ROOT/"
+if [[ "$PROBE_RPKI_CLIENT_CACHE" == "1" ]]; then
+  scp "$ROOT_DIR/target/release/cir_probe_rpki_client_cache" "$SSH_TARGET:$REMOTE_ROOT/bin/"
+fi
+scp "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_ROOT/bin/rpki-client"
+scp "$LIBTLS_PATH" "$SSH_TARGET:$REMOTE_ROOT/lib/libtls.so.28"
+printf '%s' "$OURS_EXTRA_ARGS" | ssh "$SSH_TARGET" "cat > '$REMOTE_ROOT/ours-extra-args.txt'"
+printf '%s' "$RP_RUN_MODE" | ssh "$SSH_TARGET" "cat > '$REMOTE_ROOT/rp-run-mode.txt'"
+printf '%s' "$RIR_SET" | ssh "$SSH_TARGET" "cat > '$REMOTE_ROOT/rir-set.txt'"
+
+run_step() {
+  local step_id="$1"
+  local kind="$2"
+  local local_step="$RUN_ROOT/steps/$step_id"
+
+  ssh "$SSH_TARGET" bash -s -- "$REMOTE_ROOT" "$step_id" "$kind" <<'EOS'
+set -euo pipefail
+REMOTE_ROOT="$1"
+STEP_ID="$2"
+KIND="$3"
+
+cd "$REMOTE_ROOT"
+mkdir -p "steps/$STEP_ID/ours" "steps/$STEP_ID/rpki-client"
+touch rpki-client-skiplist
+chmod 0644 rpki-client-skiplist
+OURS_EXTRA_ARGS="$(cat ours-extra-args.txt)"
+RP_RUN_MODE="$(cat rp-run-mode.txt)"
+RIR_SET="$(cat rir-set.txt)"
+OURS_EXTRA_ARGV=()
+if [[ -n "$OURS_EXTRA_ARGS" ]]; then
+  # shellcheck disable=SC2206
+  OURS_EXTRA_ARGV=($OURS_EXTRA_ARGS)
+fi
+
+case "$RIR_SET" in
+  mixed2) RIRS=(apnic arin) ;;
+  all5) RIRS=(afrinic apnic arin lacnic ripe) ;;
+  *) echo "invalid rir set: $RIR_SET" >&2; exit 2 ;;
+esac
+
+tal_file_for_rir() {
+  case "$1" in
+    afrinic) printf '%s' "afrinic.tal" ;;
+    apnic) printf '%s' "apnic-rfc7730-https.tal" ;;
+    arin) printf '%s' "arin.tal" ;;
+    lacnic) printf '%s' "lacnic.tal" ;;
+    ripe) printf '%s' "ripe-ncc.tal" ;;
+    *) echo "unknown rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+tal_uri_for_rir() {
+  case "$1" in
+    afrinic) printf '%s' "https://rpki.afrinic.net/repository/AfriNIC.cer" ;;
+    apnic) printf '%s' "https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer" ;;
+    arin) printf '%s' "https://rrdp.arin.net/arin-rpki-ta.cer" ;;
+    lacnic) printf '%s' "https://rrdp.lacnic.net/ta/rta-lacnic-rpki.cer" ;;
+    ripe) printf '%s' "https://rpki.ripe.net/ta/ripe-ncc-ta.cer" ;;
+    *) echo "unknown rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+ta_file_for_rir() {
+  case "$1" in
+    afrinic) printf '%s' "afrinic-ta.cer" ;;
+    apnic) printf '%s' "apnic-ta.cer" ;;
+    arin) printf '%s' "arin-ta.cer" ;;
+    lacnic) printf '%s' "lacnic-ta.cer" ;;
+    ripe) printf '%s' "ripe-ncc-ta.cer" ;;
+    *) echo "unknown rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+refresh_ta_file_for_rir() {
+  local rir="$1"
+  local uri
+  local file
+  uri="$(tal_uri_for_rir "$rir")"
+  file="$(ta_file_for_rir "$rir")"
+  python3 - <<'PY' "$uri" "$file"
+import sys
+import urllib.request
+uri, path = sys.argv[1:]
+request = urllib.request.Request(uri, headers={"User-Agent": "rpki-dev/compare-fast-path"})
+with urllib.request.urlopen(request, timeout=30) as response:
+    data = response.read()
+if not data:
+    raise SystemExit(f"empty TA certificate response: {uri}")
+with open(path, "wb") as output:
+    output.write(data)
+PY
+}
+
+for rir in "${RIRS[@]}"; do
+  refresh_ta_file_for_rir "$rir"
+done
+
+OURS_TAL_ARGS=()
+CLIENT_TAL_ARGS=()
+OURS_CIR_TAL_ARGS=()
+for rir in "${RIRS[@]}"; do
+  tal_file="$(tal_file_for_rir "$rir")"
+  ta_file="$(ta_file_for_rir "$rir")"
+  tal_uri="$(tal_uri_for_rir "$rir")"
+  OURS_TAL_ARGS+=(--tal-path "$tal_file" --ta-path "$ta_file")
+  OURS_CIR_TAL_ARGS+=(--cir-tal-uri "$tal_uri")
+  CLIENT_TAL_ARGS+=(-t "../../$tal_file")
+done
+
+if [[ "$KIND" == "snapshot" ]]; then
+  rm -rf state/ours/work-db state/ours/raw-store.db state/ours/repo-bytes.db state/rpki-client/cache state/rpki-client/out state/rpki-client/ta state/rpki-client/.ta
+fi
+mkdir -p state/ours/work-db state/ours/raw-store.db state/ours/repo-bytes.db state/rpki-client/cache state/rpki-client/out state/rpki-client/ta state/rpki-client/.ta
+chmod 0777 state/ours/work-db state/ours/raw-store.db state/ours/repo-bytes.db
+chmod -R 0777 state/rpki-client
+touch state/rpki-client/rpki-client-skiplist
+chmod 0644 state/rpki-client/rpki-client-skiplist
+
+START_EPOCH="$(python3 - <<'PY'
+import time
+print(time.time() + 3.0)
+PY
+)"
+
+run_ours() {
+  python3 - <<'PY' "$START_EPOCH"
+import sys, time
+x = float(sys.argv[1])
+d = x - time.time()
+if d > 0:
+    time.sleep(d)
+PY
+  started_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  set +e
+  env RPKI_PROGRESS_LOG=1 RPKI_PROGRESS_SLOW_SECS=0 ./rpki \
+    --db state/ours/work-db \
+    --raw-store-db state/ours/raw-store.db \
+    --repo-bytes-db state/ours/repo-bytes.db \
+    "${OURS_TAL_ARGS[@]}" \
+    "${OURS_EXTRA_ARGV[@]}" \
+    --ccr-out "steps/$STEP_ID/ours/result.ccr" \
+    --cir-enable \
+    --cir-out "steps/$STEP_ID/ours/result.cir" \
+    "${OURS_CIR_TAL_ARGS[@]}" \
+    --report-json "steps/$STEP_ID/ours/report.json" \
+    > "steps/$STEP_ID/ours/run.log" 2>&1
+  exit_code=$?
+  set -e
+  finished_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  python3 - <<'PY' "steps/$STEP_ID/ours/round-result.json" "$STEP_ID" "$KIND" "$started_ms" "$finished_ms" "$exit_code"
+import json, sys
+path, step_id, kind, started_ms, finished_ms, exit_code = sys.argv[1:]
+json.dump(
+    {
+        "stepId": step_id,
+        "kind": kind,
+        "durationMs": int(finished_ms) - int(started_ms),
+        "exitCode": int(exit_code),
+    },
+    open(path, "w"),
+    indent=2,
+)
+PY
+}
+
+run_client() {
+  cd state/rpki-client
+  python3 - <<'PY' "$START_EPOCH"
+import sys, time
+x = float(sys.argv[1])
+d = x - time.time()
+if d > 0:
+    time.sleep(d)
+PY
+  started_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  set +e
+  LD_LIBRARY_PATH="$REMOTE_ROOT/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" "$REMOTE_ROOT/bin/rpki-client" \
+    -vv \
+    -S rpki-client-skiplist \
+    "${CLIENT_TAL_ARGS[@]}" \
+    -d cache out \
+    > "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/run.log" 2>&1
+  exit_code=$?
+  set -e
+  cp out/rpki.ccr "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/result.ccr" 2>/dev/null || true
+  cp out/rpki.cir "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/result.cir" 2>/dev/null || true
+  cp out/openbgpd "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/openbgpd" 2>/dev/null || true
+  finished_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  python3 - <<'PY' "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/round-result.json" "$STEP_ID" "$KIND" "$started_ms" "$finished_ms" "$exit_code"
+import json, sys
+path, step_id, kind, started_ms, finished_ms, exit_code = sys.argv[1:]
+json.dump(
+    {
+        "stepId": step_id,
+        "kind": kind,
+        "durationMs": int(finished_ms) - int(started_ms),
+        "exitCode": int(exit_code),
+    },
+    open(path, "w"),
+    indent=2,
+)
+PY
+}
+
+if [[ "$RP_RUN_MODE" == "parallel" ]]; then
+  run_ours &
+  OURS_PID=$!
+  run_client &
+  CLIENT_PID=$!
+  wait "$OURS_PID"
+  wait "$CLIENT_PID"
+else
+  run_ours
+  run_client
+fi
+EOS
+
+  for rel in result.ccr result.cir round-result.json run.log stage-timing.json; do
+    scp -C "$SSH_TARGET:$REMOTE_ROOT/steps/$step_id/ours/$rel" "$local_step/ours/"
+  done
+  for rel in result.ccr result.cir round-result.json run.log openbgpd; do
+    scp -C "$SSH_TARGET:$REMOTE_ROOT/steps/$step_id/rpki-client/$rel" "$local_step/rpki-client/" || true
+  done
+  if [[ "$COPY_RPKI_CLIENT_CACHE" == "1" ]]; then
+    mkdir -p "$local_step/rpki-client/cache"
+    rsync -a --delete "$SSH_TARGET:$REMOTE_ROOT/state/rpki-client/cache/" "$local_step/rpki-client/cache/"
+  fi
+
+  if [[ -f "$local_step/ours/result.cir" && -f "$local_step/rpki-client/result.cir" ]]; then
+    "$ROOT_DIR/scripts/periodic/compare_ccr_cir_round.sh" \
+      --ours-ccr "$local_step/ours/result.ccr" \
+      --rpki-client-ccr "$local_step/rpki-client/result.ccr" \
+      --ours-cir "$local_step/ours/result.cir" \
+      --rpki-client-cir "$local_step/rpki-client/result.cir" \
+      --out-dir "$local_step/compare" \
+      --trust-anchor unknown >/dev/null
+    if [[ "$PROBE_RPKI_CLIENT_CACHE" == "1" ]]; then
+      ssh "$SSH_TARGET" "set -e; mkdir -p '$REMOTE_ROOT/steps/$step_id/compare/cir'; '$REMOTE_ROOT/bin/cir_probe_rpki_client_cache' --ours-cir '$REMOTE_ROOT/steps/$step_id/ours/result.cir' --rpki-client-cir '$REMOTE_ROOT/steps/$step_id/rpki-client/result.cir' --cache-root '$REMOTE_ROOT/state/rpki-client/cache' --rpki-client-log '$REMOTE_ROOT/steps/$step_id/rpki-client/run.log' --out-json '$REMOTE_ROOT/steps/$step_id/compare/cir/rpki-client-cache-probe.json' --sample-limit 50 >/dev/null"
+      scp -C "$SSH_TARGET:$REMOTE_ROOT/steps/$step_id/compare/cir/rpki-client-cache-probe.json" "$local_step/compare/cir/"
+    fi
+    if [[ "$COPY_RPKI_CLIENT_CACHE" == "1" ]]; then
+      "$ROOT_DIR/target/release/cir_probe_rpki_client_cache" \
+        --ours-cir "$local_step/ours/result.cir" \
+        --rpki-client-cir "$local_step/rpki-client/result.cir" \
+        --cache-root "$local_step/rpki-client/cache" \
+        --rpki-client-log "$local_step/rpki-client/run.log" \
+        --out-json "$local_step/compare/cir/rpki-client-cache-probe.json" \
+        --sample-limit 50 >/dev/null
+    fi
+  else
+    "$ROOT_DIR/scripts/periodic/compare_ccr_round.sh" \
+      --ours-ccr "$local_step/ours/result.ccr" \
+      --rpki-client-ccr "$local_step/rpki-client/result.ccr" \
+      --out-dir "$local_step/compare" \
+      --trust-anchor unknown >/dev/null
+  fi
+
+  python3 - <<'PY' "$local_step/ours/round-result.json" "$local_step/rpki-client/round-result.json" "$local_step/ours/stage-timing.json" "$local_step/compare/summary.json" "$local_step/compare/compare-summary.json" "$local_step/step-summary.json" "$OURS_EXTRA_ARGS"
+import json, sys
+ours = json.load(open(sys.argv[1]))
+client = json.load(open(sys.argv[2]))
+stage = json.load(open(sys.argv[3]))
+compare_path = sys.argv[4] if __import__('pathlib').Path(sys.argv[4]).exists() else sys.argv[5]
+compare = json.load(open(compare_path))
+ours_extra_args = sys.argv[7]
+json.dump(
+    {
+        "stepId": ours["stepId"],
+        "kind": ours["kind"],
+        "oursExtraArgs": ours_extra_args,
+        "oursDurationMs": ours["durationMs"],
+        "rpkiClientDurationMs": client["durationMs"],
+        "oursExitCode": ours["exitCode"],
+        "rpkiClientExitCode": client["exitCode"],
+        "oursTotalMs": stage["total_ms"],
+        "oursRepoSyncMsTotal": stage["repo_sync_ms_total"],
+        "oursPublicationPointRepoSyncMsTotal": stage.get("publication_point_repo_sync_ms_total"),
+        "oursDownloadEventCount": stage.get("download_event_count"),
+        "oursRrdpDownloadMsTotal": stage.get("rrdp_download_ms_total"),
+        "oursRsyncDownloadMsTotal": stage.get("rsync_download_ms_total"),
+        "oursDownloadBytesTotal": stage.get("download_bytes_total"),
+        "oursVrps": compare["vrps"]["ours"],
+        "rpkiClientVrps": compare["vrps"]["rpkiClient"],
+        "oursVaps": compare["vaps"]["ours"],
+        "rpkiClientVaps": compare["vaps"]["rpkiClient"],
+        "vrpMatch": compare["vrps"]["match"],
+        "vapMatch": compare["vaps"]["match"],
+        "allMatch": compare["allMatch"],
+        "onlyInOurs": len(compare["vrps"]["onlyInOurs"]),
+        "onlyInRpkiClient": len(compare["vrps"]["onlyInRpkiClient"]),
+    },
+    open(sys.argv[6], "w"),
+    indent=2,
+)
+PY
+}
+
+run_step step-001 snapshot
+run_step step-002 delta
+
+python3 - <<'PY' "$RUN_ROOT/steps/step-001/step-summary.json" "$RUN_ROOT/steps/step-002/step-summary.json" "$RUN_ROOT/summary.json" "$RP_RUN_MODE" "$OURS_EXTRA_ARGS" "$RIR_SET" "$SCOPE_LABEL" "${RIRS[@]}"
+import json, sys
+steps = [json.load(open(p)) for p in sys.argv[1:3]]
+summary = {
+    "workflowName": "性能对比测试快速版",
+    "scope": sys.argv[7],
+    "rpRunMode": sys.argv[4],
+    "oursExtraArgs": sys.argv[5],
+    "rirSet": sys.argv[6],
+    "rirs": sys.argv[8:],
+    "steps": steps,
+}
+json.dump(summary, open(sys.argv[3], "w"), indent=2, ensure_ascii=False)
+print(json.dumps(summary, indent=2, ensure_ascii=False))
+PY

scripts/coverage.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Requires:
+#   rustup component add llvm-tools-preview
+#   cargo install cargo-llvm-cov --locked
+
+# Optional:
+#   COVERAGE_FORCE_CLEAN=1  Force `cargo llvm-cov clean --workspace` before the run.
+#                           Default behavior is to reuse existing llvm-cov build artifacts.
+#   RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS=1  Skip replay/matrix integration tests that
+#                           spawn shell pipelines and can trigger separate release builds.
+#                           coverage.sh enables this by default.
+#   RPKI_SKIP_HEAVY_BLACKBOX_TESTS=1  Skip slower blackbox CLI/script integration tests
+#                           that provide low incremental coverage per wall-clock second.
+#                           coverage.sh enables this by default.
+#   RPKI_SKIP_HEAVY_CRYPTO_TESTS=1  Skip slower OpenSSL-heavy certificate generation tests
+#                           that provide low incremental coverage per wall-clock second.
+#                           coverage.sh enables this by default.
+
+run_out="$(mktemp)"
+text_out="$(mktemp)"
+html_out="$(mktemp)"
+
+cleanup() {
+  rm -f "$run_out" "$text_out" "$html_out"
+}
+trap cleanup EXIT
+
+IGNORE_REGEX='repository_view_stats\.rs|db_stats\.rs|rrdp_state_dump\.rs|ccr_dump\.rs|ccr_verify\.rs|ccr_to_routinator_csv\.rs|ccr_to_compare_views\.rs|cir_materialize\.rs|cir_extract_inputs\.rs|cir_drop_report\.rs|cir_ta_only_fixture\.rs|cir_dump_reject_list\.rs|rpki_object_parse\.rs|triage_ccr_cir_pair\.rs|rpki_artifact_metrics|rpki_inter_rp_metrics|rpki_daemon\.rs|sequence_triage_ccr_cir|ccr_state_compare\.rs|cir_state_compare\.rs|cir_probe_rpki_client_cache\.rs|ccr/compare_view\.rs|progress_log\.rs|cli\.rs|validation/run_tree_from_tal\.rs|validation/tree_parallel\.rs|validation/tree_runner|validation/from_tal\.rs|sync/store_projection\.rs|sync/repo\.rs|sync/rrdp|(^|/)storage(/|\.rs$)|cir/materialize\.rs'
+
+# Preserve colored output even though we post-process output by running under a pseudo-TTY.
+# We run tests only once, then generate both CLI text + HTML reports without rerunning tests.
+set +e
+
+if [ "${COVERAGE_FORCE_CLEAN:-0}" = "1" ]; then
+  cargo llvm-cov clean --workspace >/dev/null 2>&1
+  echo "coverage mode: clean build (COVERAGE_FORCE_CLEAN=1)"
+else
+  echo "coverage mode: reuse existing llvm-cov artifacts (default)"
+fi
+
+export RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS="${RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS:-1}"
+export RPKI_SKIP_HEAVY_BLACKBOX_TESTS="${RPKI_SKIP_HEAVY_BLACKBOX_TESTS:-1}"
+export RPKI_SKIP_HEAVY_CRYPTO_TESTS="${RPKI_SKIP_HEAVY_CRYPTO_TESTS:-1}"
+
+# 1) Run tests once to collect coverage data (no report).
+script -q -e -c "CARGO_TERM_COLOR=always cargo llvm-cov --no-report" "$run_out" >/dev/null 2>&1
+run_status="$?"
+
+# 2) CLI summary report + fail-under gate (no test rerun).
+script -q -e -c "CARGO_TERM_COLOR=always cargo llvm-cov report --fail-under-lines 90 --ignore-filename-regex '$IGNORE_REGEX'" "$text_out" >/dev/null 2>&1
+text_status="$?"
+
+# 3) HTML report (no test rerun).
+script -q -e -c "CARGO_TERM_COLOR=always cargo llvm-cov report --html --ignore-filename-regex '$IGNORE_REGEX'" "$html_out" >/dev/null 2>&1
+html_status="$?"
+
+set -e
+
+strip_script_noise() {
+  tr -d '\r' | sed '/^Script \(started\|done\) on /d'
+}
+
+strip_ansi_for_parse() {
+  awk '
+    {
+      line = $0
+      gsub(/\033\[[0-9;]*[A-Za-z]/, "", line) # CSI escapes
+      gsub(/\033\([A-Za-z]/, "", line)        # charset escapes (e.g., ESC(B)
+      gsub(/\r/, "", line)
+      print line
+    }
+  '
+}
+
+cat "$run_out" | strip_script_noise
+cat "$text_out" | strip_script_noise
+cat "$html_out" | strip_script_noise
+
+cat "$run_out" | strip_ansi_for_parse | awk '
+  BEGIN {
+    passed=0; failed=0; ignored=0; measured=0; filtered=0;
+  }
+  /^test result: / {
+    if (match($0, /([0-9]+) passed; ([0-9]+) failed; ([0-9]+) ignored; ([0-9]+) measured; ([0-9]+) filtered out;/, m)) {
+      passed += m[1]; failed += m[2]; ignored += m[3]; measured += m[4]; filtered += m[5];
+    }
+  }
+  END {
+    executed = passed + failed;
+    total = passed + failed + ignored + measured;
+    printf("\nTEST SUMMARY (all suites): passed=%d failed=%d ignored=%d measured=%d filtered_out=%d executed=%d total=%d\n",
+           passed, failed, ignored, measured, filtered, executed, total);
+  }
+'
+
+echo
+echo "HTML report: target/llvm-cov/html/index.html"
+
+status="$text_status"
+if [ "$run_status" -ne 0 ]; then status="$run_status"; fi
+if [ "$html_status" -ne 0 ]; then status="$html_status"; fi
+exit "$status"

scripts/experiments/feature035/experiments.json

@@ -0,0 +1,41 @@
+{
+  "schemaVersion": 1,
+  "defaultRirs": ["afrinic", "apnic", "arin", "lacnic", "ripe"],
+  "experiments": [
+    {
+      "id": "sync-ours-rsync-only",
+      "left": { "rpKind": "ours", "mode": "standard", "protocol": "rrdp+rsync", "rsyncScope": "module-root" },
+      "right": { "rpKind": "ours", "mode": "standard", "protocol": "rsync-only", "rsyncScope": "module-root" }
+    },
+    {
+      "id": "sync-rpki-client-rsync-only",
+      "left": { "rpKind": "rpki-client", "mode": "standard", "protocol": "rrdp+rsync" },
+      "right": { "rpKind": "rpki-client", "mode": "standard", "protocol": "rsync-only" }
+    },
+    {
+      "id": "strict-name",
+      "left": { "rpKind": "ours", "mode": "standard", "protocol": "rrdp+rsync" },
+      "right": { "rpKind": "ours", "mode": "strict-name", "protocol": "rrdp+rsync" }
+    },
+    {
+      "id": "strict-cms-der",
+      "left": { "rpKind": "ours", "mode": "standard", "protocol": "rrdp+rsync" },
+      "right": { "rpKind": "ours", "mode": "strict-cms-der", "protocol": "rrdp+rsync" }
+    },
+    {
+      "id": "strict-signed-attrs",
+      "left": { "rpKind": "ours", "mode": "standard", "protocol": "rrdp+rsync" },
+      "right": { "rpKind": "ours", "mode": "strict-signed-attrs", "protocol": "rrdp+rsync" }
+    },
+    {
+      "id": "strict-all",
+      "left": { "rpKind": "ours", "mode": "standard", "protocol": "rrdp+rsync" },
+      "right": { "rpKind": "ours", "mode": "strict-all", "protocol": "rrdp+rsync" }
+    },
+    {
+      "id": "rp-implementation-standard",
+      "left": { "rpKind": "ours", "mode": "standard", "protocol": "rrdp+rsync" },
+      "right": { "rpKind": "rpki-client", "mode": "standard", "protocol": "rrdp+rsync" }
+    }
+  ]
+}

scripts/experiments/feature035/feature035_bundle.py

@@ -0,0 +1,431 @@
+#!/usr/bin/env python3
+import argparse
+import hashlib
+import json
+import os
+import platform
+import subprocess
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+RIR_FIXTURES = {
+    "afrinic": {
+        "tal": "tal/afrinic.tal",
+        "ta": "ta/afrinic-ta.cer",
+    },
+    "apnic": {
+        "tal": "tal/apnic-rfc7730-https.tal",
+        "ta": "ta/apnic-ta.cer",
+    },
+    "arin": {
+        "tal": "tal/arin.tal",
+        "ta": "ta/arin-ta.cer",
+    },
+    "lacnic": {
+        "tal": "tal/lacnic.tal",
+        "ta": "ta/lacnic-ta.cer",
+    },
+    "ripe": {
+        "tal": "tal/ripe-ncc.tal",
+        "ta": "ta/ripe-ncc-ta.cer",
+    },
+}
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def sha256_file(path: Path) -> str:
+    hasher = hashlib.sha256()
+    with path.open("rb") as file:
+        for chunk in iter(lambda: file.read(1024 * 1024), b""):
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+
+def read_tal_uris(path: Path) -> list[str]:
+    uris: list[str] = []
+    with path.open("r", encoding="utf-8") as file:
+        for line in file:
+            item = line.strip()
+            if not item or item.startswith("#"):
+                if uris:
+                    break
+                continue
+            if item.startswith(("rsync://", "https://", "http://")):
+                uris.append(item)
+                continue
+            if uris:
+                break
+    return uris
+
+
+def first_uri(uris: list[str], prefixes: tuple[str, ...]) -> str | None:
+    for uri in uris:
+        if uri.startswith(prefixes):
+            return uri
+    return None
+
+
+def parse_rirs(raw: str) -> list[str]:
+    rirs = [item.strip().lower() for item in raw.split(",") if item.strip()]
+    if not rirs:
+        raise SystemExit("RIR list must not be empty")
+    invalid = [item for item in rirs if item not in RIR_FIXTURES]
+    if invalid:
+        raise SystemExit(
+            f"invalid RIR(s): {','.join(invalid)}; allowed: {','.join(RIR_FIXTURES)}"
+        )
+    return rirs
+
+
+def rel_or_abs(path: Path, root: Path | None) -> str:
+    path = path.resolve()
+    if root is not None:
+        try:
+            return path.relative_to(root.resolve()).as_posix()
+        except ValueError:
+            pass
+    return path.as_posix()
+
+
+def git_commit(repo_root: Path) -> str | None:
+    try:
+        return subprocess.check_output(
+            ["git", "-C", str(repo_root), "rev-parse", "--short", "HEAD"],
+            text=True,
+            stderr=subprocess.DEVNULL,
+        ).strip()
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return None
+
+
+def write_json(path: Path, value: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+
+def build_fixture_proof(
+    fixture_dir: Path,
+    rirs: list[str],
+    repo_root: Path | None,
+    ta_online_fetch_observed: bool,
+) -> dict[str, Any]:
+    trust_anchors = []
+    for rir in rirs:
+        mapping = RIR_FIXTURES[rir]
+        tal_path = fixture_dir / mapping["tal"]
+        ta_path = fixture_dir / mapping["ta"]
+        if not tal_path.is_file():
+            raise SystemExit(f"missing TAL fixture for {rir}: {tal_path}")
+        if not ta_path.is_file():
+            raise SystemExit(f"missing TA fixture for {rir}: {ta_path}")
+        tal_uris = read_tal_uris(tal_path)
+        trust_anchors.append(
+            {
+                "rir": rir,
+                "talPath": rel_or_abs(tal_path, repo_root),
+                "taPath": rel_or_abs(ta_path, repo_root),
+                "talUri": first_uri(tal_uris, ("https://", "http://")),
+                "taRsyncUri": first_uri(tal_uris, ("rsync://",)),
+                "talSha256": sha256_file(tal_path),
+                "taCertificateSha256": sha256_file(ta_path),
+                "talBytes": tal_path.stat().st_size,
+                "taCertificateBytes": ta_path.stat().st_size,
+                "taFixturePinned": not ta_online_fetch_observed,
+                "taOnlineFetchObserved": ta_online_fetch_observed,
+            }
+        )
+    return {
+        "schemaVersion": 1,
+        "generatedBy": "feature035-experiment-driver",
+        "generatedAtUtc": utc_now(),
+        "fixtureDir": rel_or_abs(fixture_dir, repo_root),
+        "all5": set(rirs) == set(RIR_FIXTURES),
+        "rirs": rirs,
+        "trustAnchors": trust_anchors,
+    }
+
+
+def parse_csv(raw: str) -> list[str]:
+    if not raw:
+        return []
+    return [item.strip() for item in raw.split(",") if item.strip()]
+
+
+def optional_path(raw: str | None, repo_root: Path | None) -> str | None:
+    if raw is None:
+        return None
+    return rel_or_abs(Path(raw), repo_root)
+
+
+def build_run_meta(args: argparse.Namespace) -> dict[str, Any]:
+    rirs = parse_rirs(args.rirs)
+    repo_root = Path(args.repo_root).resolve() if args.repo_root else None
+    argv = json.loads(args.argv_json) if args.argv_json else []
+    env_whitelist = json.loads(args.env_json) if args.env_json else {}
+    fixture_proof_summary = (
+        json.loads(args.fixture_proof_summary_json)
+        if args.fixture_proof_summary_json
+        else None
+    )
+    fixture_proof = None
+    if args.fixture_proof:
+        fixture_proof_path = Path(args.fixture_proof)
+        if fixture_proof_path.is_file():
+            fixture_proof = json.loads(fixture_proof_path.read_text(encoding="utf-8"))
+    if not isinstance(argv, list):
+        raise SystemExit("--argv-json must decode to a JSON array")
+    if not isinstance(env_whitelist, dict):
+        raise SystemExit("--env-json must decode to a JSON object")
+    if fixture_proof_summary is not None and not isinstance(fixture_proof_summary, dict):
+        raise SystemExit("--fixture-proof-summary-json must decode to a JSON object")
+
+    return {
+        "schemaVersion": 1,
+        "generatedBy": "feature035-experiment-driver",
+        "generatedAtUtc": utc_now(),
+        "experimentId": args.experiment_id,
+        "side": args.side,
+        "sideLabel": args.side_label,
+        "step": args.step,
+        "runId": args.run_id,
+        "liveRun": not args.replay_used,
+        "replayUsed": args.replay_used,
+        "rp": {
+            "kind": args.rp_kind,
+            "binary": args.rp_binary,
+            "version": args.rp_version,
+            "gitCommit": args.rp_git_commit,
+            "mode": args.rp_mode,
+            "protocolMode": args.protocol_mode,
+            "strictPolicies": parse_csv(args.strict_policies),
+        },
+        "scope": {
+            "rirs": rirs,
+            "all5": set(rirs) == set(RIR_FIXTURES),
+        },
+        "command": {
+            "argv": argv,
+            "cwd": args.cwd,
+            "envWhitelist": env_whitelist,
+        },
+        "state": {
+            "resetBeforeRun": args.reset_before_run,
+            "stateRoot": args.state_root,
+            "db": args.db,
+            "repoBytesDb": args.repo_bytes_db,
+            "rawStoreDb": args.raw_store_db,
+            "rsyncMirrorRoot": args.rsync_mirror_root,
+            "cacheRoot": args.cache_root,
+        },
+        "artifacts": {
+            "ccr": optional_path(args.ccr, repo_root),
+            "cir": optional_path(args.cir, repo_root),
+            "runMeta": optional_path(str(args.out), repo_root),
+            "fixtureProof": optional_path(args.fixture_proof, repo_root),
+            "reportJson": optional_path(args.report_json, repo_root),
+            "stageTimingJson": optional_path(args.stage_timing_json, repo_root),
+            "stdoutLog": optional_path(args.stdout_log, repo_root),
+            "stderrLog": optional_path(args.stderr_log, repo_root),
+            "processTime": optional_path(args.process_time, repo_root),
+            "vrpsCsv": optional_path(args.vrps_csv, repo_root),
+            "vapsCsv": optional_path(args.vaps_csv, repo_root),
+        },
+        "fixtureProof": fixture_proof,
+        "fixtureProofSummary": fixture_proof_summary,
+        "metrics": {
+            "exitCode": args.exit_code,
+            "wallMs": args.wall_ms,
+            "maxRssKb": args.max_rss_kb,
+            "vrps": args.vrps,
+            "vaps": args.vaps,
+            "publicationPoints": args.publication_points,
+            "warnings": args.warnings,
+            "cirObjectCount": args.cir_object_count,
+            "cirRejectCount": args.cir_reject_count,
+            "cirTrustAnchorCount": args.cir_trust_anchor_count,
+            "ccrStateDigest": args.ccr_state_digest,
+        },
+        "environment": {
+            "host": args.host or platform.node(),
+            "platform": args.platform or platform.platform(),
+        },
+    }
+
+
+def command_fixture_proof(args: argparse.Namespace) -> None:
+    repo_root = Path(args.repo_root).resolve() if args.repo_root else None
+    proof = build_fixture_proof(
+        fixture_dir=Path(args.fixture_dir),
+        rirs=parse_rirs(args.rirs),
+        repo_root=repo_root,
+        ta_online_fetch_observed=args.ta_online_fetch_observed,
+    )
+    write_json(Path(args.out), proof)
+
+
+def command_run_meta(args: argparse.Namespace) -> None:
+    write_json(Path(args.out), build_run_meta(args))
+
+
+def command_dry_run_bundle(args: argparse.Namespace) -> None:
+    out_dir = Path(args.out_dir)
+    repo_root = Path(args.repo_root).resolve() if args.repo_root else Path.cwd().resolve()
+    fixture_proof = out_dir / "fixture-proof.json"
+    command_fixture_proof(
+        argparse.Namespace(
+            fixture_dir=args.fixture_dir,
+            rirs=args.rirs,
+            repo_root=str(repo_root),
+            out=str(fixture_proof),
+            ta_online_fetch_observed=False,
+        )
+    )
+    for side, side_label in (("left", "A"), ("right", "B")):
+        run_dir = out_dir / side_label / "snapshot"
+        meta_args = argparse.Namespace(
+            out=run_dir / "run-meta.json",
+            repo_root=str(repo_root),
+            experiment_id=args.experiment_id,
+            side=side,
+            side_label=side_label,
+            step="snapshot",
+            run_id=f"{side_label}-snapshot-dry-run",
+            replay_used=False,
+            rp_kind="ours" if side_label == "A" else "rpki-client",
+            rp_binary=f"bin/{'rpki' if side_label == 'A' else 'rpki-client'}",
+            rp_version="dry-run",
+            rp_git_commit=git_commit(repo_root),
+            rp_mode="standard",
+            protocol_mode="rrdp+rsync",
+            strict_policies="",
+            rirs=args.rirs,
+            argv_json=json.dumps(["dry-run"]),
+            env_json=json.dumps({"RPKI_PROGRESS_LOG": "1"}),
+            cwd=str(out_dir),
+            reset_before_run=True,
+            state_root=str(out_dir / side_label / "state"),
+            db=str(out_dir / side_label / "state" / "work-db"),
+            repo_bytes_db=str(out_dir / side_label / "state" / "repo-bytes.db"),
+            raw_store_db=str(out_dir / side_label / "state" / "raw-store.db"),
+            rsync_mirror_root=str(out_dir / side_label / "state" / "rsync-mirror"),
+            cache_root=str(out_dir / side_label / "state" / "cache"),
+            ccr=str(run_dir / "result.ccr"),
+            cir=str(run_dir / "result.cir"),
+            fixture_proof=str(fixture_proof),
+            report_json=str(run_dir / "report.json"),
+            stage_timing_json=str(run_dir / "stage-timing.json"),
+            stdout_log=str(run_dir / "stdout.log"),
+            stderr_log=str(run_dir / "stderr.log"),
+            process_time=str(run_dir / "process-time.txt"),
+            vrps_csv=str(run_dir / "vrps.csv"),
+            vaps_csv=str(run_dir / "vaps.csv"),
+            exit_code=0,
+            wall_ms=0,
+            max_rss_kb=0,
+            vrps=0,
+            vaps=0,
+            publication_points=0,
+            warnings=0,
+            cir_object_count=0,
+            cir_reject_count=0,
+            cir_trust_anchor_count=len(parse_rirs(args.rirs)),
+            ccr_state_digest=None,
+            fixture_proof_summary_json=json.dumps(
+                {
+                    "taFixturePinned": True,
+                    "taOnlineFetchObserved": False,
+                    "trustAnchorCount": len(parse_rirs(args.rirs)),
+                }
+            ),
+        )
+        command_run_meta(meta_args)
+
+
+def add_run_meta_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--out", required=True)
+    parser.add_argument("--repo-root")
+    parser.add_argument("--experiment-id", required=True)
+    parser.add_argument("--side", choices=["left", "right"], required=True)
+    parser.add_argument("--side-label", choices=["A", "B"], required=True)
+    parser.add_argument("--step", choices=["snapshot", "delta"], required=True)
+    parser.add_argument("--run-id", required=True)
+    parser.add_argument("--replay-used", action="store_true")
+    parser.add_argument("--rp-kind", required=True)
+    parser.add_argument("--rp-binary", required=True)
+    parser.add_argument("--rp-version")
+    parser.add_argument("--rp-git-commit")
+    parser.add_argument("--rp-mode", default="standard")
+    parser.add_argument("--protocol-mode", default="rrdp+rsync")
+    parser.add_argument("--strict-policies", default="")
+    parser.add_argument("--rirs", default="afrinic,apnic,arin,lacnic,ripe")
+    parser.add_argument("--argv-json")
+    parser.add_argument("--env-json")
+    parser.add_argument("--cwd", default=os.getcwd())
+    parser.add_argument("--reset-before-run", action="store_true")
+    parser.add_argument("--state-root")
+    parser.add_argument("--db")
+    parser.add_argument("--repo-bytes-db")
+    parser.add_argument("--raw-store-db")
+    parser.add_argument("--rsync-mirror-root")
+    parser.add_argument("--cache-root")
+    parser.add_argument("--ccr")
+    parser.add_argument("--cir")
+    parser.add_argument("--fixture-proof")
+    parser.add_argument("--fixture-proof-summary-json")
+    parser.add_argument("--report-json")
+    parser.add_argument("--stage-timing-json")
+    parser.add_argument("--stdout-log")
+    parser.add_argument("--stderr-log")
+    parser.add_argument("--process-time")
+    parser.add_argument("--vrps-csv")
+    parser.add_argument("--vaps-csv")
+    parser.add_argument("--exit-code", type=int)
+    parser.add_argument("--wall-ms", type=int)
+    parser.add_argument("--max-rss-kb", type=int)
+    parser.add_argument("--vrps", type=int)
+    parser.add_argument("--vaps", type=int)
+    parser.add_argument("--publication-points", type=int)
+    parser.add_argument("--warnings", type=int)
+    parser.add_argument("--cir-object-count", type=int)
+    parser.add_argument("--cir-reject-count", type=int)
+    parser.add_argument("--cir-trust-anchor-count", type=int)
+    parser.add_argument("--ccr-state-digest")
+    parser.add_argument("--host")
+    parser.add_argument("--platform")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Feature #035 CCR/CIR experiment bundle helpers")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    fixture = subparsers.add_parser("fixture-proof")
+    fixture.add_argument("--fixture-dir", default="tests/fixtures")
+    fixture.add_argument("--rirs", default="afrinic,apnic,arin,lacnic,ripe")
+    fixture.add_argument("--repo-root")
+    fixture.add_argument("--out", required=True)
+    fixture.add_argument("--ta-online-fetch-observed", action="store_true")
+    fixture.set_defaults(func=command_fixture_proof)
+
+    run_meta = subparsers.add_parser("run-meta")
+    add_run_meta_args(run_meta)
+    run_meta.set_defaults(func=command_run_meta)
+
+    dry_run = subparsers.add_parser("dry-run-bundle")
+    dry_run.add_argument("--out-dir", required=True)
+    dry_run.add_argument("--repo-root", default=".")
+    dry_run.add_argument("--fixture-dir", default="tests/fixtures")
+    dry_run.add_argument("--rirs", default="afrinic,apnic,arin,lacnic,ripe")
+    dry_run.add_argument("--experiment-id", default="m2-dry-run")
+    dry_run.set_defaults(func=command_dry_run_bundle)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

scripts/experiments/feature035/fixture-manifest.json

@@ -0,0 +1,10 @@
+{
+  "schemaVersion": 1,
+  "rirs": {
+    "afrinic": { "tal": "tal/afrinic.tal", "ta": "ta/afrinic-ta.cer" },
+    "apnic": { "tal": "tal/apnic-rfc7730-https.tal", "ta": "ta/apnic-ta.cer" },
+    "arin": { "tal": "tal/arin.tal", "ta": "ta/arin-ta.cer" },
+    "lacnic": { "tal": "tal/lacnic.tal", "ta": "ta/lacnic-ta.cer" },
+    "ripe": { "tal": "tal/ripe-ncc.tal", "ta": "ta/ripe-ncc-ta.cer" }
+  }
+}

scripts/experiments/feature043/run_sequence_triage_experiment.py

@@ -0,0 +1,939 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import hashlib
+import json
+import os
+import shlex
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+REPO_ROOT = SCRIPT_DIR.parents[2]
+DEV_ROOT = REPO_ROOT.parents[1]
+FEATURE035_DIR = REPO_ROOT / "scripts" / "experiments" / "feature035"
+FIXTURE_MANIFEST_PATH = FEATURE035_DIR / "fixture-manifest.json"
+PORTABLE_ROOT = DEV_ROOT / "rpki-client-portable"
+CACHED_CIR_RPKI_CLIENT = DEV_ROOT / ".cache" / "rpki-client-9.8-cir" / "rpki-client"
+CACHED_CIR_LIBTLS = DEV_ROOT / ".cache" / "rpki-client-9.8-cir" / "libtls.so.28"
+
+DEFAULT_RIRS = ["afrinic", "apnic", "arin", "lacnic", "ripe"]
+
+
+def run_local(argv: list[str], *, cwd: Path | None = None, capture: bool = False, check: bool = True) -> subprocess.CompletedProcess[str]:
+    result = subprocess.run(argv, cwd=str(cwd) if cwd else None, text=True, capture_output=capture, check=False)
+    if check and result.returncode != 0:
+        raise SystemExit(
+            f"command failed ({result.returncode}): {' '.join(shlex.quote(x) for x in argv)}\n"
+            f"stdout:\n{result.stdout or ''}\nstderr:\n{result.stderr or ''}"
+        )
+    return result
+
+
+def ssh_script(target: str, script: str, *, capture: bool = False, check: bool = True) -> subprocess.CompletedProcess[str]:
+    result = subprocess.run(["ssh", target, "bash", "-s"], input=script, text=True, capture_output=capture, check=False)
+    if check and result.returncode != 0:
+        raise SystemExit(f"remote script failed ({result.returncode}) on {target}\n{result.stdout}\n{result.stderr}")
+    return result
+
+
+def rsync_to_remote(target: str, source: Path, destination: str | Path) -> None:
+    run_local(["rsync", "-a", str(source), f"{target}:{destination}"])
+
+
+def rsync_dir_to_remote(target: str, source: Path, destination: str | Path) -> None:
+    run_local(["rsync", "-a", f"{source}/", f"{target}:{destination}/"])
+
+
+def rsync_from_remote(target: str, source: str | Path, destination: Path) -> None:
+    destination.mkdir(parents=True, exist_ok=True)
+    run_local(["rsync", "-a", f"{target}:{source}/", f"{destination}/"])
+
+
+def rsync_run_artifacts_from_remote(target: str, source: str | Path, destination: Path) -> None:
+    destination.mkdir(parents=True, exist_ok=True)
+    rsync_base = ["rsync", "-az", "--ignore-missing-args", "--partial", "--partial-dir=.rsync-partial"]
+    for name in [
+        "result.ccr",
+        "result.cir",
+        "report.json",
+        "vrps.csv",
+        "vaps.csv",
+        "process-time.txt",
+        "remote-run-meta.json",
+        "exit-code.txt",
+        "started-at.txt",
+        "finished-at.txt",
+        "stdout.log",
+        "stderr.log",
+    ]:
+        run_local([*rsync_base, f"{target}:{source}/{name}", f"{destination}/"])
+
+
+def rsync_remote_analysis_from_remote(target: str, remote_exp_root: str | Path, local_exp_root: Path) -> None:
+    local_exp_root.mkdir(parents=True, exist_ok=True)
+    rsync_base = ["rsync", "-az", "--ignore-missing-args", "--partial", "--partial-dir=.rsync-partial"]
+    for name in [
+        "left-sequence.jsonl",
+        "right-sequence.jsonl",
+        "run-progress.json",
+        "sequence-triage-time.txt",
+        "sequence-triage/sequence-triage.json",
+    ]:
+        destination = local_exp_root / Path(name).parent
+        destination.mkdir(parents=True, exist_ok=True)
+        run_local([*rsync_base, f"{target}:{remote_exp_root}/{name}", f"{destination}/"])
+
+
+def same_remote_location(
+    left_target: str,
+    left_root: str | Path,
+    right_target: str,
+    right_root: str | Path,
+) -> bool:
+    return left_target == right_target and str(left_root) == str(right_root)
+
+
+def load_json(path: Path) -> Any:
+    with path.open("r", encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def write_json(path: Path, value: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as handle:
+        json.dump(value, handle, indent=2, sort_keys=True, ensure_ascii=False)
+        handle.write("\n")
+
+
+def append_jsonl(path: Path, value: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as handle:
+        handle.write(json.dumps(value, sort_keys=True, ensure_ascii=False))
+        handle.write("\n")
+
+
+def utc_stamp() -> str:
+    return time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
+
+
+def rfc3339_now() -> str:
+    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+
+
+def fixture_manifest() -> dict[str, Any]:
+    return load_json(FIXTURE_MANIFEST_PATH)
+
+
+def fixture_name(rir: str, kind: str) -> str:
+    return Path(fixture_manifest()["rirs"][rir][kind]).name
+
+
+def fixture_desc(rir: str) -> str:
+    return {
+        "afrinic": "afrinic",
+        "apnic": "apnic-rfc7730-https",
+        "arin": "arin",
+        "lacnic": "lacnic",
+        "ripe": "ripe-ncc",
+    }[rir]
+
+
+def cir_tal_uri_for_rir(rir: str) -> str:
+    return {
+        "afrinic": "https://rpki.afrinic.net/tal/afrinic.tal",
+        "apnic": "https://rpki.apnic.net/tal/apnic-rfc7730-https.tal",
+        "arin": "https://www.arin.net/resources/manage/rpki/arin.tal",
+        "lacnic": "https://www.lacnic.net/innovaportal/file/4983/1/lacnic.tal",
+        "ripe": "https://tal.rpki.ripe.net/ripe-ncc.tal",
+    }[rir]
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def parse_elapsed_to_ms(raw: str) -> int:
+    raw = raw.strip()
+    if not raw:
+        return 0
+    if "-" in raw:
+        days, raw = raw.split("-", 1)
+    else:
+        days = "0"
+    parts = raw.split(":")
+    if len(parts) == 3:
+        hours, minutes, seconds = parts
+    elif len(parts) == 2:
+        hours = "0"
+        minutes, seconds = parts
+    else:
+        hours = "0"
+        minutes = "0"
+        seconds = parts[0]
+    return int(round((int(days) * 86400 + int(hours) * 3600 + int(minutes) * 60 + float(seconds)) * 1000))
+
+
+def parse_time_file(path: Path) -> dict[str, Any]:
+    data: dict[str, Any] = {}
+    if not path.is_file():
+        return data
+    for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
+        if "Elapsed (wall clock) time" in line:
+            elapsed = line.rsplit(":", 1)[1] if "):" not in line else line.rsplit("):", 1)[1]
+            data["wallMs"] = parse_elapsed_to_ms(elapsed)
+        elif "Maximum resident set size" in line:
+            try:
+                data["maxRssKb"] = int(line.rsplit(":", 1)[1].strip())
+            except ValueError:
+                pass
+    return data
+
+
+def rpki_client_bin_path() -> Path:
+    primary = PORTABLE_ROOT / "src" / "rpki-client"
+    for candidate in (primary, CACHED_CIR_RPKI_CLIENT):
+        if not candidate.is_file():
+            continue
+        smoke = run_local([str(candidate), "-T", "invalid"], capture=True, check=False)
+        if "--ta-fixture requires <tal>:<path>" in (smoke.stderr + smoke.stdout):
+            return candidate
+    raise SystemExit("rpki-client binary lacks CIR/TA fixture support; checkout feature/cir-output-for-rp-compare or restore .cache/rpki-client-9.8-cir/rpki-client")
+
+
+def detect_libtls_path(rpki_client_bin: Path) -> Path:
+    if CACHED_CIR_LIBTLS.is_file():
+        return CACHED_CIR_LIBTLS
+    ldd = run_local(["ldd", str(rpki_client_bin)], capture=True)
+    for line in ldd.stdout.splitlines():
+        if "libtls.so.28" not in line or "=>" not in line:
+            continue
+        candidate = Path(line.split("=>", 1)[1].strip().split(" ", 1)[0])
+        if candidate.is_file():
+            return candidate
+    fallback = DEV_ROOT / ".cache" / "rpki-client-9.8-cir" / "libtls.so.28"
+    if fallback.is_file():
+        return fallback
+    raise SystemExit("unable to locate libtls.so.28 for rpki-client")
+
+
+def build_tool_binaries() -> None:
+    run_local([
+        "cargo", "build", "--release",
+        "--bin", "rpki",
+        "--bin", "sequence_triage_ccr_cir",
+        "--bin", "cir_dump_reject_list",
+    ], cwd=REPO_ROOT)
+    _ = rpki_client_bin_path()
+
+
+def validate_remote_disk(ssh_target: str) -> None:
+    script = r'''
+set -euo pipefail
+df -h /data / || true
+python3 - <<'PY'
+import shutil
+for path in ['/data', '/']:
+    try:
+        usage = shutil.disk_usage(path)
+    except FileNotFoundError:
+        continue
+    used = usage.used / usage.total if usage.total else 0
+    print(f'{path} used={used:.2%}')
+    if used >= 0.90:
+        raise SystemExit(f'{path} disk usage >= 90%; cleanup required before all5 sequence experiment')
+PY
+'''
+    ssh_script(ssh_target, script)
+
+
+def prepare_remote(ssh_target: str, remote_root: Path, needs_rpki_client: bool) -> None:
+    validate_remote_disk(ssh_target)
+    preflight = (
+        "set -euo pipefail; "
+        "systemctl disable --now rpki-client.timer >/dev/null 2>&1 || true; "
+        "systemctl stop rpki-client.service >/dev/null 2>&1 || true; "
+        "pkill -x rpki-client >/dev/null 2>&1 || true; "
+        "pkill -x routinator >/dev/null 2>&1 || true; "
+        f"mkdir -p {shlex.quote(str(remote_root / 'bin'))} {shlex.quote(str(remote_root / 'lib'))} "
+        f"{shlex.quote(str(remote_root / 'fixtures' / 'tal'))} {shlex.quote(str(remote_root / 'fixtures' / 'ta'))} "
+        f"{shlex.quote(str(remote_root / 'experiments'))}; "
+        f"df -h /data / > {shlex.quote(str(remote_root / 'df-before.txt'))} 2>&1 || true; "
+        f"free -h > {shlex.quote(str(remote_root / 'free-before.txt'))} 2>&1 || true"
+    )
+    ssh_script(ssh_target, preflight)
+    rsync_dir_to_remote(ssh_target, REPO_ROOT / "tests" / "fixtures" / "tal", remote_root / "fixtures" / "tal")
+    rsync_dir_to_remote(ssh_target, REPO_ROOT / "tests" / "fixtures" / "ta", remote_root / "fixtures" / "ta")
+    rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "rpki", remote_root / "bin" / "rpki")
+    rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "sequence_triage_ccr_cir", remote_root / "bin" / "sequence_triage_ccr_cir")
+    rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "cir_dump_reject_list", remote_root / "bin" / "cir_dump_reject_list")
+    if needs_rpki_client:
+        rpki_client_bin = rpki_client_bin_path()
+        rsync_to_remote(ssh_target, rpki_client_bin, remote_root / "bin" / "rpki-client")
+        rsync_to_remote(ssh_target, detect_libtls_path(rpki_client_bin), remote_root / "lib" / "libtls.so.28")
+
+
+def prepare_remote_once(
+    prepared: dict[tuple[str, str], bool],
+    ssh_target: str,
+    remote_root: Path,
+    needs_rpki_client: bool,
+) -> None:
+    key = (ssh_target, str(remote_root))
+    already_has_rpki_client = prepared.get(key)
+    if already_has_rpki_client is not None and (already_has_rpki_client or not needs_rpki_client):
+        return
+    prepare_remote(ssh_target, remote_root, needs_rpki_client)
+    prepared[key] = bool(already_has_rpki_client or needs_rpki_client)
+
+
+def side_config(name: str) -> dict[str, Any]:
+    if name == "ours-standard":
+        return {"rpKind": "ours", "mode": "standard", "protocol": "rrdp+rsync", "rsyncScope": "module-root"}
+    if name == "ours-strict-all":
+        return {
+            "rpKind": "ours",
+            "mode": "strict-all",
+            "protocol": "rrdp+rsync",
+            "rsyncScope": "module-root",
+            "strictPolicies": "name,cms-der,signed-attrs",
+        }
+    if name == "rpki-client-standard":
+        return {"rpKind": "rpki-client", "mode": "standard", "protocol": "rrdp+rsync"}
+    raise SystemExit(f"unknown side config: {name}")
+
+
+def parse_rirs(raw: str) -> list[str]:
+    rirs = [item.strip() for item in raw.split(",") if item.strip()]
+    if not rirs:
+        raise SystemExit("--rirs must contain at least one RIR")
+    seen: set[str] = set()
+    invalid: list[str] = []
+    duplicate: list[str] = []
+    for rir in rirs:
+        if rir not in DEFAULT_RIRS:
+            invalid.append(rir)
+        if rir in seen:
+            duplicate.append(rir)
+        seen.add(rir)
+    if invalid:
+        raise SystemExit(f"unsupported RIR(s): {','.join(invalid)}; valid values: {','.join(DEFAULT_RIRS)}")
+    if duplicate:
+        raise SystemExit(f"duplicate RIR(s): {','.join(duplicate)}")
+    return rirs
+
+
+def build_remote_command(remote_root: Path, side_name: str, side: dict[str, Any], side_label: str, seq: int, rirs: list[str]) -> tuple[Path, str]:
+    run_dir = remote_root / "experiments" / "sequence" / side_label / f"run_{seq:04d}"
+    state_dir = remote_root / "experiments" / "sequence" / side_label / "state" / side["rpKind"]
+    sync_mode = "snapshot" if seq == 1 else "delta"
+    ensure = [f"mkdir -p {shlex.quote(str(run_dir))}", f"chmod 0777 {shlex.quote(str(run_dir))}"]
+    if side["rpKind"] == "ours":
+        if seq == 1:
+            ensure.append(f"rm -rf {shlex.quote(str(state_dir))}")
+        ensure.extend([
+            f"mkdir -p {shlex.quote(str(state_dir / 'work-db'))} {shlex.quote(str(state_dir / 'rsync-mirror'))}",
+            f"chmod -R 0777 {shlex.quote(str(state_dir.parent))}",
+        ])
+        argv = [
+            str(remote_root / "bin" / "rpki"),
+            "--db", str(state_dir / "work-db"),
+            "--raw-store-db", str(state_dir / "raw-store.db"),
+            "--repo-bytes-db", str(state_dir / "repo-bytes.db"),
+            "--rsync-scope", side.get("rsyncScope", "module-root"),
+        ]
+        if side.get("strictPolicies"):
+            argv.extend(["--strict", str(side["strictPolicies"])])
+        for rir in rirs:
+            argv.extend(["--tal-path", str(remote_root / "fixtures" / "tal" / fixture_name(rir, "tal"))])
+            argv.extend(["--ta-path", str(remote_root / "fixtures" / "ta" / fixture_name(rir, "ta"))])
+        argv.extend(["--report-json", str(run_dir / "report.json"), "--report-json-compact"])
+        argv.extend(["--ccr-out", str(run_dir / "result.ccr"), "--cir-enable", "--cir-out", str(run_dir / "result.cir")])
+        for rir in rirs:
+            argv.extend(["--cir-tal-uri", cir_tal_uri_for_rir(rir)])
+        argv.extend(["--vrps-csv-out", str(run_dir / "vrps.csv"), "--vaps-csv-out", str(run_dir / "vaps.csv")])
+        prefix = "env RPKI_PROGRESS_LOG=1 RPKI_PROGRESS_SLOW_SECS=10 /usr/bin/time"
+    else:
+        if seq == 1:
+            ensure.append(f"rm -rf {shlex.quote(str(state_dir))}")
+        ensure.extend([
+            f"mkdir -p {shlex.quote(str(state_dir / 'cache' / 'fixtures'))}",
+            f"touch {shlex.quote(str(state_dir / 'rpki-client-skiplist'))}",
+            f"chmod -R 0777 {shlex.quote(str(state_dir.parent))}",
+        ])
+        for rir in rirs:
+            ensure.append(
+                f"cp -f {shlex.quote(str(remote_root / 'fixtures' / 'ta' / fixture_name(rir, 'ta')))} "
+                f"{shlex.quote(str(state_dir / 'cache' / 'fixtures' / fixture_name(rir, 'ta')))}"
+            )
+        argv = [str(remote_root / "bin" / "rpki-client"), "-vv", "-S", str(state_dir / "rpki-client-skiplist")]
+        for rir in rirs:
+            argv.extend(["-t", str(remote_root / "fixtures" / "tal" / fixture_name(rir, "tal"))])
+            argv.extend(["-T", f"{fixture_desc(rir)}:{state_dir / 'cache' / 'fixtures' / fixture_name(rir, 'ta')}"])
+        argv.extend(["-d", str(state_dir / "cache"), str(run_dir)])
+        prefix = f"env LD_LIBRARY_PATH={shlex.quote(str(remote_root / 'lib'))} /usr/bin/time"
+    command = (
+        "set -euo pipefail; "
+        + "; ".join(ensure)
+        + "; date -u +%Y-%m-%dT%H:%M:%SZ > " + shlex.quote(str(run_dir / "started-at.txt"))
+        + "; set +e; "
+        + prefix + " -v -o " + shlex.quote(str(run_dir / "process-time.txt"))
+        + " -- " + shlex.join(argv)
+        + " > " + shlex.quote(str(run_dir / "stdout.log"))
+        + " 2> " + shlex.quote(str(run_dir / "stderr.log"))
+        + "; ec=$?; set -e; printf '%s\n' \"$ec\" > " + shlex.quote(str(run_dir / "exit-code.txt"))
+        + "; date -u +%Y-%m-%dT%H:%M:%SZ > " + shlex.quote(str(run_dir / "finished-at.txt"))
+        + "; true"
+    )
+    if side["rpKind"] == "rpki-client":
+        command += (
+            f"; [ -f {shlex.quote(str(run_dir / 'json'))} ] && cp -f {shlex.quote(str(run_dir / 'json'))} {shlex.quote(str(run_dir / 'report.json'))} || true"
+            f"; [ -f {shlex.quote(str(run_dir / 'rpki.ccr'))} ] && cp -f {shlex.quote(str(run_dir / 'rpki.ccr'))} {shlex.quote(str(run_dir / 'result.ccr'))} || true"
+            f"; [ -f {shlex.quote(str(run_dir / 'rpki.cir'))} ] && cp -f {shlex.quote(str(run_dir / 'rpki.cir'))} {shlex.quote(str(run_dir / 'result.cir'))} || true"
+        )
+    command += (
+        f"; python3 - <<'REMOTE_META' {shlex.quote(str(run_dir))} {shlex.quote(side_name)} {shlex.quote(side_label)} {seq} {shlex.quote(sync_mode)}\n"
+        "import json, pathlib, sys\n"
+        "run_dir=pathlib.Path(sys.argv[1]); side_name=sys.argv[2]; side_label=sys.argv[3]; seq=int(sys.argv[4]); sync_mode=sys.argv[5]\n"
+        "def read(p):\n    return p.read_text().strip() if p.exists() else None\n"
+        "def counts_from_report():\n"
+        "    p=run_dir/'report.json'\n"
+        "    if not p.exists():\n"
+        "        return {}\n"
+        "    try:\n"
+        "        report=json.load(open(p))\n"
+        "    except Exception:\n"
+        "        return {}\n"
+        "    meta=report.get('metadata') if isinstance(report, dict) else None\n"
+        "    if isinstance(meta, dict):\n"
+        "        return {'vrps': int(meta.get('vrps') or 0), 'vaps': int(meta.get('vaps') or meta.get('aspas') or 0), 'publicationPoints': int(meta.get('repositories') or 0), 'warnings': 0}\n"
+        "    pps=report.get('publication_points', []) if isinstance(report, dict) else []\n"
+        "    tree=report.get('tree', {}) if isinstance(report, dict) else {}\n"
+        "    pp_warnings=sum(len(pp.get('warnings', [])) for pp in pps if isinstance(pp, dict)) if isinstance(pps, list) else 0\n"
+        "    return {'vrps': len(report.get('vrps', [])), 'vaps': len(report.get('aspas', [])), 'publicationPoints': len(pps) if isinstance(pps, list) else 0, 'warnings': len(tree.get('warnings', [])) + pp_warnings if isinstance(tree, dict) else pp_warnings}\n"
+        "meta={'sideName':side_name,'sideLabel':side_label,'seq':seq,'syncMode':sync_mode,'startedAt':read(run_dir/'started-at.txt'),'finishedAt':read(run_dir/'finished-at.txt'),'exitCode':int(read(run_dir/'exit-code.txt') or '1'),'counts':counts_from_report()}\n"
+        "json.dump(meta, open(run_dir/'remote-run-meta.json','w'), indent=2, sort_keys=True); print()\n"
+        "REMOTE_META"
+    )
+    return run_dir, command
+
+
+def run_remote_sample(ssh_target: str, remote_root: Path, side_name: str, side: dict[str, Any], side_label: str, seq: int, rirs: list[str]) -> Path:
+    run_dir, command = build_remote_command(remote_root, side_name, side, side_label, seq, rirs)
+    ssh_script(ssh_target, command)
+    return run_dir
+
+
+def append_remote_sequence_item(
+    ssh_target: str,
+    remote_root: Path,
+    side_name: str,
+    side_label: str,
+    seq: int,
+    run_dir: Path,
+    schedule_mode: str,
+) -> dict[str, Any]:
+    remote_exp_root = remote_root / "experiments" / "sequence"
+    seq_path = remote_exp_root / ("left-sequence.jsonl" if side_label == "A" else "right-sequence.jsonl")
+    side_value = "left" if side_label == "A" else "right"
+    script = f"""
+set -euo pipefail
+python3 - <<'REMOTE_SEQUENCE_ITEM' {shlex.quote(str(remote_exp_root))} {shlex.quote(str(seq_path))} {shlex.quote(str(run_dir))} {shlex.quote(str(remote_root / 'bin' / 'cir_dump_reject_list'))} {shlex.quote(side_name)} {shlex.quote(side_label)} {seq} {shlex.quote(side_value)} {shlex.quote(schedule_mode)}
+import hashlib, json, os, pathlib, subprocess, sys
+exp_root=pathlib.Path(sys.argv[1])
+seq_path=pathlib.Path(sys.argv[2])
+run_dir=pathlib.Path(sys.argv[3])
+cir_dump=pathlib.Path(sys.argv[4])
+side_name=sys.argv[5]
+side_label=sys.argv[6]
+seq=int(sys.argv[7])
+side_value=sys.argv[8]
+schedule_mode=sys.argv[9]
+def read(p):
+    return p.read_text().strip() if p.exists() else None
+def sha256_file(p):
+    h=hashlib.sha256()
+    with open(p, 'rb') as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b''):
+            h.update(chunk)
+    return h.hexdigest()
+def parse_elapsed_to_ms(raw):
+    raw=raw.strip()
+    if not raw:
+        return 0
+    if '-' in raw:
+        days, raw=raw.split('-', 1)
+    else:
+        days='0'
+    parts=raw.split(':')
+    if len(parts) == 3:
+        hours, minutes, seconds=parts
+    elif len(parts) == 2:
+        hours='0'; minutes, seconds=parts
+    else:
+        hours='0'; minutes='0'; seconds=parts[0]
+    return int(round((int(days)*86400 + int(hours)*3600 + int(minutes)*60 + float(seconds))*1000))
+def parse_time_file(p):
+    data={{}}
+    if not p.exists():
+        return data
+    for line in p.read_text(errors='replace').splitlines():
+        if 'Elapsed (wall clock) time' in line:
+            elapsed=line.rsplit('):', 1)[1] if '):' in line else line.rsplit(':', 1)[1]
+            data['wallMs']=parse_elapsed_to_ms(elapsed)
+        elif 'Maximum resident set size' in line:
+            try:
+                data['maxRssKb']=int(line.rsplit(':', 1)[1].strip())
+            except ValueError:
+                pass
+    return data
+def cir_counts(cir):
+    values={{}}
+    if not cir.exists():
+        return {{}}
+    result=subprocess.run([str(cir_dump), '--cir', str(cir), '--limit', '0'], text=True, capture_output=True, check=True)
+    for line in result.stdout.splitlines():
+        if '=' not in line:
+            continue
+        key, value=line.split('=', 1)
+        if key in ('object_count', 'trust_anchor_count', 'reject_count'):
+            values[key]=int(value)
+    return {{'cirObjectCount': values.get('object_count', 0), 'cirTrustAnchorCount': values.get('trust_anchor_count', 0), 'cirRejectCount': values.get('reject_count', 0)}}
+meta=json.load(open(run_dir/'remote-run-meta.json'))
+time_info=parse_time_file(run_dir/'process-time.txt')
+counts=dict(meta.get('counts') or {{}})
+ccr=run_dir/'result.ccr'
+cir=run_dir/'result.cir'
+if meta.get('exitCode') != 0:
+    raise SystemExit(f"remote run failed: exitCode={{meta.get('exitCode')}} runDir={{run_dir}}")
+missing=[str(path) for path in (ccr, cir) if not path.exists()]
+if missing:
+    raise SystemExit(f"remote run missing required artifact(s): {{missing}}")
+counts.update(cir_counts(cir))
+item={{
+    'schemaVersion': 1,
+    'rpId': side_name,
+    'side': side_value,
+    'seq': seq,
+    'runId': f'{{side_label}}-{{seq:04d}}',
+    'syncMode': 'snapshot' if seq == 1 else 'delta',
+    'status': 'success' if meta.get('exitCode') == 0 else 'failed',
+    'startTime': meta.get('startedAt'),
+    'finishTime': meta.get('finishedAt'),
+    'validationTime': None,
+    'ccrPath': os.path.relpath(ccr, exp_root),
+    'cirPath': os.path.relpath(cir, exp_root),
+    'ccrSha256': sha256_file(ccr) if ccr.exists() else None,
+    'cirSha256': sha256_file(cir) if cir.exists() else None,
+    'wallMs': time_info.get('wallMs'),
+    'maxRssKb': time_info.get('maxRssKb'),
+    'vrps': counts.get('vrps'),
+    'vaps': counts.get('vaps'),
+    'publicationPoints': counts.get('publicationPoints'),
+    'cirObjectCount': counts.get('cirObjectCount'),
+    'cirRejectCount': counts.get('cirRejectCount'),
+    'cirTrustAnchorCount': counts.get('cirTrustAnchorCount'),
+    'scheduleMode': schedule_mode,
+}}
+seq_path.parent.mkdir(parents=True, exist_ok=True)
+with open(seq_path, 'a', encoding='utf-8') as handle:
+    handle.write(json.dumps(item, sort_keys=True, ensure_ascii=False) + '\\n')
+print(json.dumps(item, sort_keys=True, ensure_ascii=False))
+REMOTE_SEQUENCE_ITEM
+"""
+    result = ssh_script(ssh_target, script, capture=True)
+    lines = [line for line in result.stdout.splitlines() if line.strip()]
+    if not lines:
+        raise SystemExit("remote sequence item append produced no output")
+    return json.loads(lines[-1])
+
+
+def cleanup_remote_run_nonessential(ssh_target: str, run_dir: Path) -> None:
+    keep = {
+        "result.ccr",
+        "result.cir",
+        "process-time.txt",
+        "remote-run-meta.json",
+        "exit-code.txt",
+        "started-at.txt",
+        "finished-at.txt",
+        "stdout.log",
+        "stderr.log",
+    }
+    keep_json = json.dumps(sorted(keep), ensure_ascii=False)
+    script = f"""
+set -euo pipefail
+python3 - <<'REMOTE_CLEAN' {shlex.quote(str(run_dir))} {shlex.quote(keep_json)}
+import json, pathlib, sys
+run_dir=pathlib.Path(sys.argv[1])
+keep=set(json.loads(sys.argv[2]))
+removed=0
+if run_dir.exists():
+    for path in run_dir.iterdir():
+        if path.name in keep or path.is_dir():
+            continue
+        try:
+            removed += path.stat().st_size
+            path.unlink()
+        except FileNotFoundError:
+            pass
+print(f'cleaned_nonessential_bytes={{removed}}')
+REMOTE_CLEAN
+"""
+    ssh_script(ssh_target, script)
+
+
+def cir_counts(cir_path: Path) -> dict[str, int]:
+    result = run_local([str(REPO_ROOT / "target" / "release" / "cir_dump_reject_list"), "--cir", str(cir_path), "--limit", "0"], capture=True)
+    values: dict[str, int] = {}
+    for line in result.stdout.splitlines():
+        if "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        if key in {"object_count", "trust_anchor_count", "reject_count"}:
+            values[key] = int(value)
+    return {
+        "cirObjectCount": values.get("object_count", 0),
+        "cirTrustAnchorCount": values.get("trust_anchor_count", 0),
+        "cirRejectCount": values.get("reject_count", 0),
+    }
+
+
+def report_counts(path: Path, rp_kind: str) -> dict[str, int]:
+    if not path.is_file():
+        return {}
+    report = load_json(path)
+    if rp_kind == "rpki-client":
+        meta = report.get("metadata", {})
+        return {
+            "vrps": int(meta.get("vrps", 0)),
+            "vaps": int(meta.get("vaps", 0) or meta.get("aspas", 0) or 0),
+            "publicationPoints": int(meta.get("repositories", 0)),
+            "warnings": 0,
+        }
+    pps = report.get("publication_points", [])
+    tree = report.get("tree", {})
+    return {
+        "vrps": len(report.get("vrps", [])),
+        "vaps": len(report.get("aspas", [])),
+        "publicationPoints": len(pps),
+        "warnings": len(tree.get("warnings", [])) + sum(len(pp.get("warnings", [])) for pp in pps if isinstance(pp, dict)),
+    }
+
+
+def build_sequence_item(local_root: Path, side_name: str, side_label: str, side: dict[str, Any], seq: int, run_dir: Path) -> dict[str, Any]:
+    ccr = run_dir / "result.ccr"
+    cir = run_dir / "result.cir"
+    meta = load_json(run_dir / "remote-run-meta.json")
+    time_info = parse_time_file(run_dir / "process-time.txt")
+    counts = dict(meta.get("counts") or {})
+    if not counts:
+        counts = report_counts(run_dir / "report.json", side["rpKind"])
+    counts.update(cir_counts(cir))
+    return {
+        "schemaVersion": 1,
+        "rpId": side_name,
+        "side": "left" if side_label == "A" else "right",
+        "seq": seq,
+        "runId": f"{side_label}-{seq:04d}",
+        "syncMode": "snapshot" if seq == 1 else "delta",
+        "status": "success" if meta.get("exitCode") == 0 else "failed",
+        "startTime": meta.get("startedAt"),
+        "finishTime": meta.get("finishedAt"),
+        "validationTime": None,
+        "ccrPath": ccr.relative_to(local_root).as_posix(),
+        "cirPath": cir.relative_to(local_root).as_posix(),
+        "ccrSha256": sha256_file(ccr),
+        "cirSha256": sha256_file(cir),
+        "wallMs": time_info.get("wallMs"),
+        "maxRssKb": time_info.get("maxRssKb"),
+        "vrps": counts.get("vrps"),
+        "vaps": counts.get("vaps"),
+        "publicationPoints": counts.get("publicationPoints"),
+        "cirObjectCount": counts.get("cirObjectCount"),
+        "cirRejectCount": counts.get("cirRejectCount"),
+        "cirTrustAnchorCount": counts.get("cirTrustAnchorCount"),
+    }
+
+
+def run_sequence_triage(local_exp_root: Path, args: argparse.Namespace) -> None:
+    compare_dir = local_exp_root / "sequence-triage"
+    run_local([
+        str(REPO_ROOT / "target" / "release" / "sequence_triage_ccr_cir"),
+        "--left-sequence", str(local_exp_root / "left-sequence.jsonl"),
+        "--right-sequence", str(local_exp_root / "right-sequence.jsonl"),
+        "--out-dir", str(compare_dir),
+        "--align-window-runs", str(args.align_window_runs),
+        "--align-window-secs", str(args.align_window_secs),
+        "--sample-limit", str(args.sample_limit),
+        "--timeline-sample-limit", str(args.timeline_sample_limit),
+    ])
+
+
+def run_sequence_triage_remote(ssh_target: str, remote_root: Path, args: argparse.Namespace) -> None:
+    remote_exp_root = remote_root / "experiments" / "sequence"
+    compare_dir = remote_exp_root / "sequence-triage"
+    time_path = remote_exp_root / "sequence-triage-time.txt"
+    command = " ".join(
+        shlex.quote(item)
+        for item in [
+            str(remote_root / "bin" / "sequence_triage_ccr_cir"),
+            "--left-sequence", str(remote_exp_root / "left-sequence.jsonl"),
+            "--right-sequence", str(remote_exp_root / "right-sequence.jsonl"),
+            "--out-dir", str(compare_dir),
+            "--align-window-runs", str(args.align_window_runs),
+            "--align-window-secs", str(args.align_window_secs),
+            "--sample-limit", str(args.sample_limit),
+            "--timeline-sample-limit", str(args.timeline_sample_limit),
+        ]
+    )
+    ssh_script(
+        ssh_target,
+        "set -euo pipefail; "
+        f"rm -rf {shlex.quote(str(compare_dir))} {shlex.quote(str(time_path))}; "
+        f"/usr/bin/time -v -o {shlex.quote(str(time_path))} -- {command}",
+    )
+
+
+def sync_side_to_analysis_remote(
+    source_ssh_target: str,
+    source_remote_root: Path,
+    analysis_ssh_target: str,
+    analysis_remote_root: Path,
+    side_label: str,
+) -> None:
+    source_exp_root = source_remote_root / "experiments" / "sequence"
+    analysis_exp_root = analysis_remote_root / "experiments" / "sequence"
+    sequence_name = "left-sequence.jsonl" if side_label == "A" else "right-sequence.jsonl"
+    if same_remote_location(source_ssh_target, source_exp_root, analysis_ssh_target, analysis_exp_root):
+        return
+    side_dir = source_exp_root / side_label
+    script = (
+        "set -euo pipefail; "
+        f"ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new {shlex.quote(analysis_ssh_target)} "
+        f"{shlex.quote('mkdir -p ' + shlex.quote(str(analysis_exp_root / side_label)))}; "
+        f"rsync -az --delete {shlex.quote(str(side_dir))}/ {shlex.quote(analysis_ssh_target)}:{shlex.quote(str(analysis_exp_root / side_label))}/; "
+        f"rsync -az {shlex.quote(str(source_exp_root / sequence_name))} "
+        f"{shlex.quote(analysis_ssh_target)}:{shlex.quote(str(analysis_exp_root / sequence_name))}"
+    )
+    ssh_script(source_ssh_target, script)
+
+
+def run_side_sequence(
+    args: argparse.Namespace,
+    ssh_target: str,
+    remote_root: Path,
+    local_exp_root: Path,
+    side_label: str,
+    side_name: str,
+    side: dict[str, Any],
+    seq_path: Path,
+    rirs: list[str],
+) -> list[dict[str, Any]]:
+    side_progress: list[dict[str, Any]] = []
+    for seq in range(1, args.samples_per_side + 1):
+        side_progress.append(
+            run_one_side_sample(args, ssh_target, remote_root, local_exp_root, side_label, side_name, side, seq_path, seq, rirs)
+        )
+    return side_progress
+
+
+def run_one_side_sample(
+    args: argparse.Namespace,
+    ssh_target: str,
+    remote_root: Path,
+    local_exp_root: Path,
+    side_label: str,
+    side_name: str,
+    side: dict[str, Any],
+    seq_path: Path,
+    seq: int,
+    rirs: list[str],
+) -> dict[str, Any]:
+    rir_label = ",".join(rirs)
+    print(
+        f"[run] {side_label} {side_name} seq={seq} rirs={rir_label} schedule={args.schedule_mode}",
+        flush=True,
+    )
+    remote_run_dir = run_remote_sample(ssh_target, remote_root, side_name, side, side_label, seq, rirs)
+    if args.remote_triage:
+        item = append_remote_sequence_item(
+            ssh_target,
+            remote_root,
+            side_name,
+            side_label,
+            seq,
+            remote_run_dir,
+            args.schedule_mode,
+        )
+        if args.cleanup_run_nonessential:
+            cleanup_remote_run_nonessential(ssh_target, remote_run_dir)
+    else:
+        local_run_dir = local_exp_root / side_label / f"run_{seq:04d}"
+        rsync_run_artifacts_from_remote(ssh_target, remote_run_dir, local_run_dir)
+        item = build_sequence_item(local_exp_root, side_name, side_label, side, seq, local_run_dir)
+        item["scheduleMode"] = args.schedule_mode
+    append_jsonl(seq_path, item)
+    print(
+        f"[done] {side_label} seq={seq} wallMs={item.get('wallMs')} vrps={item.get('vrps')} vaps={item.get('vaps')} objects={item.get('cirObjectCount')} rejects={item.get('cirRejectCount')}",
+        flush=True,
+    )
+    return item
+
+
+def run_experiment(args: argparse.Namespace) -> None:
+    if not args.skip_build:
+        build_tool_binaries()
+    rirs = parse_rirs(args.rirs)
+    left = side_config(args.left)
+    right = side_config(args.right)
+    run_root = Path(args.run_root).resolve()
+    remote_root = Path(args.remote_root)
+    left_ssh_target = args.left_ssh_target or args.ssh_target
+    right_ssh_target = args.right_ssh_target or args.ssh_target
+    analysis_ssh_target = args.analysis_ssh_target or left_ssh_target
+    left_remote_root = Path(args.left_remote_root or args.remote_root)
+    right_remote_root = Path(args.right_remote_root or args.remote_root)
+    analysis_remote_root = Path(args.analysis_remote_root or args.remote_root)
+    run_root.mkdir(parents=True, exist_ok=True)
+    write_json(run_root / "experiment-config.json", {
+        "schemaVersion": 1,
+        "generatedAtUtc": utc_stamp(),
+        "left": args.left,
+        "right": args.right,
+        "samplesPerSide": args.samples_per_side,
+        "rirs": rirs,
+        "scheduleMode": args.schedule_mode,
+        "remoteRoot": str(remote_root),
+        "leftSshTarget": left_ssh_target,
+        "rightSshTarget": right_ssh_target,
+        "analysisSshTarget": analysis_ssh_target,
+        "leftRemoteRoot": str(left_remote_root),
+        "rightRemoteRoot": str(right_remote_root),
+        "analysisRemoteRoot": str(analysis_remote_root),
+        "sshTarget": args.ssh_target,
+    })
+    if args.dry_run:
+        print(json.dumps(load_json(run_root / "experiment-config.json"), indent=2, ensure_ascii=False))
+        return
+    if args.triage_only:
+        run_sequence_triage(run_root / "experiments" / "sequence", args)
+        print(json.dumps({
+            "runRoot": str(run_root),
+            "triage": str(run_root / "experiments" / "sequence" / "sequence-triage" / "sequence-triage.json"),
+        }, indent=2))
+        return
+    prepared_remotes: dict[tuple[str, str], bool] = {}
+    prepare_remote_once(prepared_remotes, left_ssh_target, left_remote_root, needs_rpki_client=(left["rpKind"] == "rpki-client"))
+    prepare_remote_once(prepared_remotes, right_ssh_target, right_remote_root, needs_rpki_client=(right["rpKind"] == "rpki-client"))
+    prepare_remote_once(prepared_remotes, analysis_ssh_target, analysis_remote_root, needs_rpki_client=False)
+    local_exp_root = run_root / "experiments" / "sequence"
+    left_seq_path = local_exp_root / "left-sequence.jsonl"
+    right_seq_path = local_exp_root / "right-sequence.jsonl"
+    left_seq_path.unlink(missing_ok=True)
+    right_seq_path.unlink(missing_ok=True)
+    progress: list[dict[str, Any]] = []
+    if args.schedule_mode == "interleaved":
+        for seq in range(1, args.samples_per_side + 1):
+            for side_label, ssh_target, side_remote_root, side_name, side, seq_path in [
+                ("A", left_ssh_target, left_remote_root, args.left, left, left_seq_path),
+                ("B", right_ssh_target, right_remote_root, args.right, right, right_seq_path),
+            ]:
+                progress.append(
+                    run_one_side_sample(
+                        args,
+                        ssh_target,
+                        side_remote_root,
+                        local_exp_root,
+                        side_label,
+                        side_name,
+                        side,
+                        seq_path,
+                        seq,
+                        rirs,
+                    )
+                )
+    else:
+        with ThreadPoolExecutor(max_workers=2) as executor:
+            futures = [
+                executor.submit(run_side_sequence, args, left_ssh_target, left_remote_root, local_exp_root, "A", args.left, left, left_seq_path, rirs),
+                executor.submit(run_side_sequence, args, right_ssh_target, right_remote_root, local_exp_root, "B", args.right, right, right_seq_path, rirs),
+            ]
+            for future in as_completed(futures):
+                progress.extend(future.result())
+    progress.sort(key=lambda item: (str(item.get("side")), int(item.get("seq") or 0)))
+    write_json(local_exp_root / "run-progress.json", progress)
+    if args.remote_triage:
+        sync_side_to_analysis_remote(left_ssh_target, left_remote_root, analysis_ssh_target, analysis_remote_root, "A")
+        sync_side_to_analysis_remote(right_ssh_target, right_remote_root, analysis_ssh_target, analysis_remote_root, "B")
+        remote_exp_root = analysis_remote_root / "experiments" / "sequence"
+        remote_progress = json.dumps(progress, sort_keys=True, ensure_ascii=False)
+        ssh_script(
+            analysis_ssh_target,
+            "set -euo pipefail; "
+            f"cat > {shlex.quote(str(remote_exp_root / 'run-progress.json'))} <<'REMOTE_PROGRESS_JSON'\n"
+            f"{remote_progress}\n"
+            "REMOTE_PROGRESS_JSON\n",
+        )
+        run_sequence_triage_remote(analysis_ssh_target, analysis_remote_root, args)
+        if args.fetch_remote_analysis:
+            rsync_remote_analysis_from_remote(analysis_ssh_target, remote_exp_root, local_exp_root)
+    else:
+        run_sequence_triage(local_exp_root, args)
+    ssh_script(analysis_ssh_target, f"df -h /data / > {shlex.quote(str(analysis_remote_root / 'df-after.txt'))} 2>&1 || true; free -h > {shlex.quote(str(analysis_remote_root / 'free-after.txt'))} 2>&1 || true")
+    compare_dir = local_exp_root / "sequence-triage"
+    remote_compare_dir = analysis_remote_root / "experiments" / "sequence" / "sequence-triage"
+    print(json.dumps({
+        "runRoot": str(run_root),
+        "remoteRoot": str(remote_root),
+        "leftRemoteRoot": str(left_remote_root),
+        "rightRemoteRoot": str(right_remote_root),
+        "analysisRemoteRoot": str(analysis_remote_root),
+        "triage": str(compare_dir / "sequence-triage.json") if not args.remote_triage or args.fetch_remote_analysis else None,
+        "remoteTriage": str(remote_compare_dir / "sequence-triage.json") if args.remote_triage else None,
+    }, indent=2))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Feature #043 all5 sequence triage experiment driver")
+    parser.add_argument("--run-root", required=True)
+    parser.add_argument("--remote-root", required=True)
+    parser.add_argument("--ssh-target", default=os.environ.get("SSH_TARGET", "root@47.251.56.108"))
+    parser.add_argument("--left-ssh-target")
+    parser.add_argument("--right-ssh-target")
+    parser.add_argument("--analysis-ssh-target")
+    parser.add_argument("--left-remote-root")
+    parser.add_argument("--right-remote-root")
+    parser.add_argument("--analysis-remote-root")
+    parser.add_argument("--left", default="ours-standard")
+    parser.add_argument("--right", default="rpki-client-standard")
+    parser.add_argument("--samples-per-side", type=int, default=3)
+    parser.add_argument("--rirs", default=",".join(DEFAULT_RIRS))
+    parser.add_argument("--schedule-mode", choices=["interleaved", "parallel"], default="interleaved")
+    parser.add_argument("--align-window-runs", type=int, default=2)
+    parser.add_argument("--align-window-secs", type=int, default=1800)
+    parser.add_argument("--sample-limit", type=int, default=200)
+    parser.add_argument("--timeline-sample-limit", type=int, default=0)
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--skip-build", action="store_true", help="reuse existing release binaries")
+    parser.add_argument("--triage-only", action="store_true", help="only rerun local sequence triage for an existing run root")
+    parser.add_argument("--remote-triage", action="store_true", help="keep CIR/CCR on remote, write sequence JSONL remotely, and run triage on remote")
+    parser.add_argument("--fetch-remote-analysis", action="store_true", help="when --remote-triage is set, fetch only small sequence/triage JSON outputs; never fetch CIR/CCR")
+    parser.add_argument("--cleanup-run-nonessential", action="store_true", help="after each successful remote sequence item, remove report/log/CSV files and keep only CIR/CCR/timing/meta")
+    args = parser.parse_args()
+    if args.samples_per_side < 2:
+        raise SystemExit("--samples-per-side must be >= 2")
+    run_experiment(args)
+
+
+if __name__ == "__main__":
+    main()

scripts/inter_rp/control_remote200_rp.sh

@@ -0,0 +1,265 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REMOTE_HOST="${REMOTE_HOST:-root@43.110.128.200}"
+REMOTE_CONFIG="${REMOTE_CONFIG:-/root/inter-rp-runners/inter-rp.env}"
+REMOTE_SCRIPTS_DIR="${REMOTE_SCRIPTS_DIR:-/root/inter-rp-runners/scripts}"
+DEFAULT_RETAIN_RUNS="${RETAIN_RUNS:-20}"
+DEFAULT_RSS_SAMPLE_MS="${RSS_SAMPLE_MS:-500}"
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  control_remote200_rp.sh status [rpki-client|routinator|all]
+  control_remote200_rp.sh start  <rpki-client|routinator>
+  control_remote200_rp.sh stop   <rpki-client|routinator|all>
+  control_remote200_rp.sh restart <rpki-client|routinator>
+
+Environment overrides:
+  REMOTE_HOST=root@43.110.128.200
+  REMOTE_CONFIG=/root/inter-rp-runners/inter-rp.env
+  REMOTE_SCRIPTS_DIR=/root/inter-rp-runners/scripts
+  RETAIN_RUNS=20
+  RSS_SAMPLE_MS=500
+
+Notes:
+  - start uses the remote run_single_rp_with_rss.sh loop and runs until stopped.
+  - stop only kills the selected RP loop branch and its current child processes.
+  - rpki-client and routinator are managed independently.
+USAGE
+}
+
+ACTION="${1:-status}"
+RP="${2:-all}"
+
+case "$ACTION" in
+  status|start|stop|restart) ;;
+  -h|--help|help)
+    usage
+    exit 0
+    ;;
+  *)
+    echo "unknown action: $ACTION" >&2
+    usage >&2
+    exit 2
+    ;;
+esac
+
+case "$RP" in
+  rpki-client|routinator|all) ;;
+  *)
+    echo "unknown RP: $RP" >&2
+    usage >&2
+    exit 2
+    ;;
+esac
+
+if [[ "$ACTION" == "start" || "$ACTION" == "restart" ]] && [[ "$RP" == "all" ]]; then
+  echo "start/restart requires one RP: rpki-client or routinator" >&2
+  exit 2
+fi
+
+ssh "$REMOTE_HOST" \
+  "REMOTE_CONFIG='$REMOTE_CONFIG' REMOTE_SCRIPTS_DIR='$REMOTE_SCRIPTS_DIR' RETAIN_RUNS='$DEFAULT_RETAIN_RUNS' RSS_SAMPLE_MS='$DEFAULT_RSS_SAMPLE_MS' ACTION='$ACTION' RP='$RP' bash -s" <<'REMOTE'
+set -euo pipefail
+
+load_config() {
+  if [[ -f "$REMOTE_CONFIG" ]]; then
+    # shellcheck disable=SC1090
+    source "$REMOTE_CONFIG"
+  fi
+  INTER_RP_ROOT="${INTER_RP_ROOT:-/var/lib/inter-rp-runners}"
+  RETAIN_RUNS="${RETAIN_RUNS:-20}"
+  RSS_SAMPLE_MS="${RSS_SAMPLE_MS:-500}"
+  ROUTINATOR_RUN_COMMAND="${ROUTINATOR_RUN_COMMAND:-/root/inter-rp-runners/scripts/run_routinator_once.sh}"
+  RPKI_CLIENT_RUN_COMMAND="${RPKI_CLIENT_RUN_COMMAND:-/root/inter-rp-runners/scripts/run_rpki_client_official_98_once.sh}"
+}
+
+rp_root_name() {
+  case "$1" in
+    routinator) echo "routinator" ;;
+    rpki-client) echo "rpki-client" ;;
+  esac
+}
+
+rp_pid_file() {
+  echo "$INTER_RP_ROOT/$(rp_root_name "$1").loop.pid"
+}
+
+rp_log_file() {
+  echo "$INTER_RP_ROOT/$(rp_root_name "$1").loop.log"
+}
+
+rp_command() {
+  case "$1" in
+    routinator) echo "$ROUTINATOR_RUN_COMMAND" ;;
+    rpki-client) echo "$RPKI_CLIENT_RUN_COMMAND" ;;
+  esac
+}
+
+rp_binary_pattern() {
+  case "$1" in
+    routinator) echo "/root/inter-rp-runners/bin/routinator|[[:space:]]routinator[[:space:]]" ;;
+    rpki-client) echo "rpki-client-official-9\\.8|[[:space:]]rpki-client[[:space:]]" ;;
+  esac
+}
+
+loop_pattern() {
+  local rp="$1"
+  echo "run_single_rp_with_rss\\.sh --rp $rp "
+}
+
+is_running_pid() {
+  local pid="${1:-}"
+  [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null
+}
+
+current_loop_pids() {
+  local rp="$1"
+  pgrep -af "$(loop_pattern "$rp")" 2>/dev/null | awk '{print $1}' || true
+}
+
+status_one() {
+  local rp="$1"
+  local pid_file
+  pid_file="$(rp_pid_file "$rp")"
+  local pid=""
+  [[ -f "$pid_file" ]] && pid="$(cat "$pid_file" 2>/dev/null || true)"
+  echo "== $rp =="
+  if is_running_pid "$pid"; then
+    echo "loop: running pid=$pid"
+  else
+    local detected
+    detected="$(current_loop_pids "$rp" | paste -sd, -)"
+    if [[ -n "$detected" ]]; then
+      echo "loop: running detected_pids=$detected"
+    else
+      echo "loop: stopped"
+    fi
+  fi
+  ps -eo pid,ppid,stat,etime,%cpu,%mem,rss,cmd --sort=-%cpu \
+    | grep -E "$(loop_pattern "$rp")|$(rp_binary_pattern "$rp")" \
+    | grep -v grep \
+    | head -20 || true
+  local latest="$INTER_RP_ROOT/$(rp_root_name "$rp")/latest/run-meta.json"
+  if [[ -f "$latest" ]]; then
+    python3 - "$latest" <<'PY'
+import json, sys
+path = sys.argv[1]
+with open(path, "r", encoding="utf-8") as handle:
+    meta = json.load(handle)
+print("latest:", "runSeq=", meta.get("runSeq"), "success=", meta.get("success"), "wallMs=", meta.get("wallMs"), "vrps=", meta.get("counts", {}).get("vrps"), "vaps=", meta.get("counts", {}).get("vaps"), "rssKb=", meta.get("maxRssKb", {}).get("aggregatePeak"))
+PY
+  else
+    echo "latest: none"
+  fi
+}
+
+start_one() {
+  local rp="$1"
+  local pid_file log_file root_name run_command
+  pid_file="$(rp_pid_file "$rp")"
+  log_file="$(rp_log_file "$rp")"
+  root_name="$(rp_root_name "$rp")"
+  run_command="$(rp_command "$rp")"
+  mkdir -p "$INTER_RP_ROOT/$root_name" "$INTER_RP_ROOT"
+  local existing=""
+  if [[ -f "$pid_file" ]]; then
+    existing="$(cat "$pid_file" 2>/dev/null || true)"
+  fi
+  if is_running_pid "$existing"; then
+    echo "$rp already running pid=$existing"
+    return 0
+  fi
+  local detected
+  detected="$(current_loop_pids "$rp" | head -1)"
+  if [[ -n "$detected" ]] && is_running_pid "$detected"; then
+    echo "$detected" >"$pid_file"
+    echo "$rp already running detected pid=$detected"
+    return 0
+  fi
+  nohup bash -lc '
+    set -euo pipefail
+    while true; do
+      "'"$REMOTE_SCRIPTS_DIR"'/run_single_rp_with_rss.sh" \
+        --rp "'"$rp"'" \
+        --root "'"$INTER_RP_ROOT/$root_name"'" \
+        --command "'"$run_command"'" \
+        --retain-runs "'"$RETAIN_RUNS"'" \
+        --sample-ms "'"$RSS_SAMPLE_MS"'" || true
+    done
+  ' >"$log_file" 2>&1 &
+  local pid="$!"
+  echo "$pid" >"$pid_file"
+  echo "$rp started pid=$pid log=$log_file"
+}
+
+stop_one() {
+  local rp="$1"
+  local candidates=()
+  local pid_file pid
+  pid_file="$(rp_pid_file "$rp")"
+  if [[ -f "$pid_file" ]]; then
+    pid="$(cat "$pid_file" 2>/dev/null || true)"
+    [[ -n "$pid" ]] && candidates+=("$pid")
+  fi
+  while read -r pid; do
+    [[ -n "$pid" ]] && candidates+=("$pid")
+  done < <(current_loop_pids "$rp")
+  while read -r pid; do
+    [[ -n "$pid" ]] && candidates+=("$pid")
+  done < <(pgrep -af "$(rp_binary_pattern "$rp")" 2>/dev/null | awk '{print $1}' || true)
+
+  if ((${#candidates[@]} == 0)); then
+    echo "$rp already stopped"
+    rm -f "$pid_file"
+    return 0
+  fi
+
+  local unique_pids
+  unique_pids="$(printf '%s\n' "${candidates[@]}" | awk '!seen[$0]++' | tr '\n' ' ')"
+  echo "stopping $rp pids=$unique_pids"
+  for pid in $unique_pids; do
+    kill -TERM "$pid" 2>/dev/null || true
+  done
+  sleep 3
+  for pid in $unique_pids; do
+    kill -KILL "$pid" 2>/dev/null || true
+  done
+  rm -f "$pid_file"
+  echo "$rp stopped"
+}
+
+load_config
+
+case "$ACTION:$RP" in
+  status:all)
+    date -Is
+    uptime
+    status_one routinator
+    status_one rpki-client
+    ;;
+  status:*)
+    date -Is
+    uptime
+    status_one "$RP"
+    ;;
+  start:*)
+    start_one "$RP"
+    status_one "$RP"
+    ;;
+  stop:all)
+    stop_one routinator
+    stop_one rpki-client
+    ;;
+  stop:*)
+    stop_one "$RP"
+    status_one "$RP"
+    ;;
+  restart:*)
+    stop_one "$RP"
+    start_one "$RP"
+    status_one "$RP"
+    ;;
+esac
+REMOTE

scripts/inter_rp/inter-rp.env.example

@@ -0,0 +1,26 @@
+# 远端200 runner 配置示例。
+# 该脚本只负责任务调度、RSS采样、run目录和run-meta.json；具体RP命令由下面两个变量提供。
+
+# 远端200上的运行根目录。
+# 建议把运行态数据放到 /var/lib，避免 rpki-client 降权后无法读写 /root。
+INTER_RP_ROOT=/var/lib/inter-rp-runners
+
+# -1 表示持续运行；正整数表示每个RP运行多少轮。
+MAX_RUNS=-1
+
+# 每轮结束后的等待秒数。
+RUN_INTERVAL_SECS=0
+
+# 每个RP保留最近多少轮。
+RETAIN_RUNS=20
+
+# RSS采样间隔，毫秒。
+RSS_SAMPLE_MS=500
+
+# Routinator 命令模板。命令运行时会自动导出 RUN_DIR/RP_ROOT/RUN_SEQ/RUN_ID/RP_NAME。
+# 命令必须把 vrps.csv/vaps.csv 写入 $RUN_DIR；Routinator 使用 --enable-aspa + JSON 输出转换出 VAP CSV。
+ROUTINATOR_RUN_COMMAND=/root/inter-rp-runners/scripts/run_routinator_once.sh
+
+# 官方 rpki-client 9.8 命令模板。
+# rpki-client 会降权运行，cache 和 output 必须预先放在可写目录；命令需复制 output/csv、output/rpki.ccr，并从 output/json 转换出 vaps.csv。
+RPKI_CLIENT_RUN_COMMAND=/root/inter-rp-runners/scripts/run_rpki_client_official_98_once.sh

scripts/inter_rp/run_inter_rp_metrics_sidecar.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+CONFIG_FILE="${INTER_RP_METRICS_CONFIG:-./inter-rp-metrics.env}"
+if [[ -f "$CONFIG_FILE" ]]; then
+  # shellcheck disable=SC1090
+  source "$CONFIG_FILE"
+fi
+
+RPKI_INTER_RP_METRICS_BIN="${RPKI_INTER_RP_METRICS_BIN:-./bin/rpki_inter_rp_metrics}"
+OURS_RUN_ROOT="${OURS_RUN_ROOT:?OURS_RUN_ROOT is required}"
+PEER_ROOT="${PEER_ROOT:-/root/inter-rp-aggregator/synced-from-200}"
+LISTEN="${INTER_RP_METRICS_LISTEN:-0.0.0.0:9557}"
+POLL_SECS="${INTER_RP_METRICS_POLL_SECS:-30}"
+INSTANCE="${INTER_RP_METRICS_INSTANCE:-remote231-inter-rp}"
+LOG_DIR="${INTER_RP_METRICS_LOG_DIR:-./logs}"
+
+mkdir -p "$LOG_DIR"
+exec "$RPKI_INTER_RP_METRICS_BIN" \
+  --ours-run-root "$OURS_RUN_ROOT" \
+  --peer-root "$PEER_ROOT" \
+  --listen "$LISTEN" \
+  --poll-secs "$POLL_SECS" \
+  --instance "$INSTANCE" \
+  >>"$LOG_DIR/inter-rp-metrics.log" 2>&1

scripts/inter_rp/run_remote200_rp_loops.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+CONFIG_FILE="${INTER_RP_CONFIG:-./inter-rp.env}"
+if [[ -f "$CONFIG_FILE" ]]; then
+  # shellcheck disable=SC1090
+  source "$CONFIG_FILE"
+fi
+
+INTER_RP_ROOT="${INTER_RP_ROOT:-/root/inter-rp-runners}"
+MAX_RUNS="${MAX_RUNS:--1}"
+RUN_INTERVAL_SECS="${RUN_INTERVAL_SECS:-0}"
+RETAIN_RUNS="${RETAIN_RUNS:-20}"
+RSS_SAMPLE_MS="${RSS_SAMPLE_MS:-500}"
+ROUTINATOR_RUN_COMMAND="${ROUTINATOR_RUN_COMMAND:-}"
+RPKI_CLIENT_RUN_COMMAND="${RPKI_CLIENT_RUN_COMMAND:-}"
+
+if [[ -z "$ROUTINATOR_RUN_COMMAND" || -z "$RPKI_CLIENT_RUN_COMMAND" ]]; then
+  cat >&2 <<'MSG'
+ROUTINATOR_RUN_COMMAND and RPKI_CLIENT_RUN_COMMAND are required.
+Each command runs with RUN_DIR, RP_ROOT, RUN_SEQ, RUN_ID and RP_NAME exported.
+MSG
+  exit 2
+fi
+
+mkdir -p "$INTER_RP_ROOT"
+
+run_loop() {
+  local rp_name="$1"
+  local run_command="$2"
+  local rp_root="$INTER_RP_ROOT/$rp_name"
+  local completed="0"
+  mkdir -p "$rp_root"
+  while true; do
+    if [[ "$MAX_RUNS" =~ ^[0-9]+$ ]] && (( completed >= MAX_RUNS )); then
+      break
+    fi
+    "$SCRIPT_DIR/run_single_rp_with_rss.sh" \
+      --rp "$rp_name" \
+      --root "$rp_root" \
+      --command "$run_command" \
+      --retain-runs "$RETAIN_RUNS" \
+      --sample-ms "$RSS_SAMPLE_MS" || true
+    completed=$((completed + 1))
+    if (( RUN_INTERVAL_SECS > 0 )); then
+      sleep "$RUN_INTERVAL_SECS"
+    fi
+  done
+}
+
+run_loop routinator "$ROUTINATOR_RUN_COMMAND" >"$INTER_RP_ROOT/routinator.loop.log" 2>&1 &
+ROUTINATOR_LOOP_PID=$!
+run_loop rpki-client "$RPKI_CLIENT_RUN_COMMAND" >"$INTER_RP_ROOT/rpki-client.loop.log" 2>&1 &
+RPKI_CLIENT_LOOP_PID=$!
+
+cat >"$INTER_RP_ROOT/runner-pids.env" <<PIDS
+ROUTINATOR_LOOP_PID=$ROUTINATOR_LOOP_PID
+RPKI_CLIENT_LOOP_PID=$RPKI_CLIENT_LOOP_PID
+PIDS
+
+wait "$ROUTINATOR_LOOP_PID" "$RPKI_CLIENT_LOOP_PID"

scripts/inter_rp/run_routinator_once.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${RUN_DIR:?RUN_DIR is required}"
+: "${RP_ROOT:?RP_ROOT is required}"
+
+timeout 1800 /root/inter-rp-runners/bin/routinator \
+  --repository-dir "$RP_ROOT/repository" \
+  --no-rir-tals \
+  --extra-tals-dir /var/lib/inter-rp-runners/fixtures/tal \
+  --enable-aspa \
+  vrps --format json --output "$RUN_DIR/routinator-output.json"
+
+python3 - "$RUN_DIR/routinator-output.json" "$RUN_DIR/vrps.csv" "$RUN_DIR/vaps.csv" <<'PY'
+import csv
+import json
+import sys
+
+json_path, vrps_path, vaps_path = sys.argv[1:4]
+
+with open(json_path, "r", encoding="utf-8") as handle:
+    payload = json.load(handle)
+
+def asn_text(value):
+    if isinstance(value, int):
+        return f"AS{value}"
+    text = str(value)
+    return text if text.startswith("AS") else f"AS{text}"
+
+def asn_sort_key(value):
+    text = asn_text(value)
+    try:
+        return int(text[2:])
+    except ValueError:
+        return text
+
+with open(vrps_path, "w", encoding="utf-8", newline="") as handle:
+    writer = csv.writer(handle)
+    writer.writerow(["ASN", "IP Prefix", "Max Length", "Trust Anchor"])
+    for row in payload.get("roas", []):
+        writer.writerow([
+            asn_text(row.get("asn", "")),
+            row.get("prefix", ""),
+            row.get("maxLength", row.get("max_length", "")),
+            row.get("ta", row.get("trust_anchor", "")),
+        ])
+
+with open(vaps_path, "w", encoding="utf-8", newline="") as handle:
+    writer = csv.writer(handle)
+    writer.writerow(["Customer ASN", "Providers", "Trust Anchor"])
+    for row in payload.get("aspas", payload.get("aspa", [])):
+        providers = sorted({asn_text(provider) for provider in row.get("providers", [])}, key=asn_sort_key)
+        writer.writerow([
+            asn_text(row.get("customer", row.get("customer_asid", ""))),
+            ";".join(providers),
+            row.get("ta", row.get("trust_anchor", "unknown")),
+        ])
+PY
+
+if [[ "${KEEP_INTER_RP_RAW_JSON:-0}" != "1" ]]; then
+  rm -f "$RUN_DIR/routinator-output.json"
+fi

scripts/inter_rp/run_rpki_client_official_98_once.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${RUN_DIR:?RUN_DIR is required}"
+: "${RP_ROOT:?RP_ROOT is required}"
+
+install -d -o nobody -g nogroup "$RP_ROOT/cache" "$RUN_DIR/output"
+chown nobody:nogroup "$RUN_DIR"
+
+timeout 1800 /root/inter-rp-runners/bin/rpki-client-official-9.8 \
+  -c \
+  -j \
+  -t /var/lib/inter-rp-runners/fixtures/tal/afrinic.tal \
+  -t /var/lib/inter-rp-runners/fixtures/tal/apnic-rfc7730-https.tal \
+  -t /var/lib/inter-rp-runners/fixtures/tal/arin.tal \
+  -t /var/lib/inter-rp-runners/fixtures/tal/lacnic.tal \
+  -t /var/lib/inter-rp-runners/fixtures/tal/ripe-ncc.tal \
+  -d "$RP_ROOT/cache" \
+  "$RUN_DIR/output"
+
+cp "$RUN_DIR/output/csv" "$RUN_DIR/vrps.csv"
+cp "$RUN_DIR/output/rpki.ccr" "$RUN_DIR/result.ccr"
+
+python3 - "$RUN_DIR/output/json" "$RUN_DIR/vaps.csv" <<'PY'
+import csv
+import json
+import sys
+
+json_path, vaps_path = sys.argv[1:3]
+
+with open(json_path, "r", encoding="utf-8") as handle:
+    payload = json.load(handle)
+
+def asn_text(value):
+    if isinstance(value, int):
+        return f"AS{value}"
+    text = str(value)
+    return text if text.startswith("AS") else f"AS{text}"
+
+def asn_sort_key(value):
+    text = asn_text(value)
+    try:
+        return int(text[2:])
+    except ValueError:
+        return text
+
+with open(vaps_path, "w", encoding="utf-8", newline="") as handle:
+    writer = csv.writer(handle)
+    writer.writerow(["Customer ASN", "Providers", "Trust Anchor"])
+    for row in payload.get("aspas", []):
+        providers = sorted({asn_text(provider) for provider in row.get("providers", [])}, key=asn_sort_key)
+        writer.writerow([
+            asn_text(row.get("customer_asid", row.get("customer", ""))),
+            ";".join(providers),
+            row.get("ta", row.get("trust_anchor", "unknown")),
+        ])
+PY
+
+rm -rf "$RUN_DIR/output"

scripts/inter_rp/run_single_rp_with_rss.sh

@@ -0,0 +1,266 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'USAGE'
+Usage: run_single_rp_with_rss.sh --rp <name> --root <path> --command <shell-command> [--retain-runs <n>] [--sample-ms <n>]
+
+The command runs with RUN_DIR, RP_ROOT, RUN_SEQ, RUN_ID, and RP_NAME exported.
+It must write artifacts into RUN_DIR. The wrapper writes run-meta.json and atomically updates latest.
+USAGE
+}
+
+RP_NAME=""
+RP_ROOT=""
+RUN_COMMAND=""
+RETAIN_RUNS="${RETAIN_RUNS:-20}"
+SAMPLE_MS="${RSS_SAMPLE_MS:-500}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rp)
+      RP_NAME="$2"
+      shift 2
+      ;;
+    --root)
+      RP_ROOT="$2"
+      shift 2
+      ;;
+    --command)
+      RUN_COMMAND="$2"
+      shift 2
+      ;;
+    --retain-runs)
+      RETAIN_RUNS="$2"
+      shift 2
+      ;;
+    --sample-ms)
+      SAMPLE_MS="$2"
+      shift 2
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "unknown argument: $1" >&2
+      usage >&2
+      exit 2
+      ;;
+  esac
+done
+
+if [[ -z "$RP_NAME" || -z "$RP_ROOT" || -z "$RUN_COMMAND" ]]; then
+  usage >&2
+  exit 2
+fi
+
+mkdir -p "$RP_ROOT/runs" "$RP_ROOT/logs"
+
+next_seq() {
+  local max_seq="0"
+  local run_path
+  shopt -s nullglob
+  for run_path in "$RP_ROOT"/runs/run_*; do
+    local run_name="${run_path##*/}"
+    local seq="${run_name#run_}"
+    if [[ "$seq" =~ ^[0-9]+$ ]] && (( 10#$seq > max_seq )); then
+      max_seq=$((10#$seq))
+    fi
+  done
+  shopt -u nullglob
+  printf '%06d' $((max_seq + 1))
+}
+
+rss_kb_for_pid() {
+  local pid="$1"
+  awk '/^VmRSS:/ {print $2; found=1} END {if (!found) print 0}' "/proc/$pid/status" 2>/dev/null || echo 0
+}
+
+collect_related_pids() {
+  local root_pid="$1"
+  local process_group="$2"
+  local queue=("$root_pid")
+  local pid
+  declare -A seen=()
+
+  if [[ -n "$process_group" ]]; then
+    while read -r pid; do
+      [[ -n "$pid" ]] && queue+=("$pid")
+    done < <(pgrep -g "$process_group" 2>/dev/null || true)
+  fi
+
+  while ((${#queue[@]} > 0)); do
+    pid="${queue[0]}"
+    queue=("${queue[@]:1}")
+    [[ -z "$pid" || -n "${seen[$pid]:-}" ]] && continue
+    seen[$pid]=1
+    echo "$pid"
+    while read -r child_pid; do
+      [[ -n "$child_pid" ]] && queue+=("$child_pid")
+    done < <(pgrep -P "$pid" 2>/dev/null || true)
+  done
+}
+
+sum_related_rss() {
+  local process_group="$1"
+  local parent_pid="$2"
+  local total_rss="0"
+  local child_max_rss="0"
+  local pid
+  while read -r pid; do
+    [[ -z "$pid" ]] && continue
+    local rss
+    rss="$(rss_kb_for_pid "$pid")"
+    total_rss=$((total_rss + rss))
+    if [[ "$pid" != "$parent_pid" ]] && (( rss > child_max_rss )); then
+      child_max_rss="$rss"
+    fi
+  done < <(collect_related_pids "$parent_pid" "$process_group")
+  printf '%s %s
+' "$total_rss" "$child_max_rss"
+}
+
+count_csv_rows() {
+  local path="$1"
+  if [[ ! -f "$path" ]]; then
+    echo 0
+    return
+  fi
+  awk 'BEGIN {count=0} /^[[:space:]]*$/ {next} /^#/ {next} NR==1 {next} {count++} END {print count}' "$path"
+}
+
+RUN_SEQ="$(next_seq)"
+RUN_ID="run_${RUN_SEQ}"
+RUN_DIR="$RP_ROOT/runs/$RUN_ID"
+mkdir -p "$RUN_DIR"
+
+STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+START_EPOCH_MS="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+
+export RP_NAME RP_ROOT RUN_SEQ RUN_ID RUN_DIR RUN_COMMAND
+set +e
+setsid bash -lc "$RUN_COMMAND" >"$RUN_DIR/stdout.log" 2>"$RUN_DIR/stderr.log" &
+CHILD_PID=$!
+set -e
+sleep 0.05
+PROCESS_GROUP="$(ps -o pgid= -p "$CHILD_PID" 2>/dev/null | tr -d '[:space:]' || true)"
+if [[ -z "$PROCESS_GROUP" ]]; then
+  PROCESS_GROUP="$CHILD_PID"
+fi
+
+PARENT_MAX_RSS_KB="0"
+CHILD_MAX_RSS_KB="0"
+AGGREGATE_PEAK_RSS_KB="0"
+SAMPLE_INTERVAL_SECONDS="$(python3 - <<PY
+print(max(float($SAMPLE_MS) / 1000.0, 0.05))
+PY
+)"
+
+while kill -0 "$CHILD_PID" 2>/dev/null; do
+  parent_rss="$(rss_kb_for_pid "$CHILD_PID")"
+  read -r aggregate_rss child_rss < <(sum_related_rss "$PROCESS_GROUP" "$CHILD_PID")
+  if (( parent_rss > PARENT_MAX_RSS_KB )); then
+    PARENT_MAX_RSS_KB="$parent_rss"
+  fi
+  if (( child_rss > CHILD_MAX_RSS_KB )); then
+    CHILD_MAX_RSS_KB="$child_rss"
+  fi
+  if (( aggregate_rss > AGGREGATE_PEAK_RSS_KB )); then
+    AGGREGATE_PEAK_RSS_KB="$aggregate_rss"
+  fi
+  sleep "$SAMPLE_INTERVAL_SECONDS"
+done
+
+set +e
+wait "$CHILD_PID"
+EXIT_CODE=$?
+set -e
+
+FINISHED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+END_EPOCH_MS="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+WALL_MS=$((END_EPOCH_MS - START_EPOCH_MS))
+SUCCESS=false
+if [[ "$EXIT_CODE" == "0" ]]; then
+  SUCCESS=true
+fi
+
+VRPS_COUNT="$(count_csv_rows "$RUN_DIR/vrps.csv")"
+VAPS_COUNT="$(count_csv_rows "$RUN_DIR/vaps.csv")"
+
+export STARTED_AT FINISHED_AT WALL_MS EXIT_CODE SUCCESS
+export PARENT_MAX_RSS_KB CHILD_MAX_RSS_KB AGGREGATE_PEAK_RSS_KB SAMPLE_MS
+export VRPS_COUNT VAPS_COUNT
+export CCR_ARTIFACT_PATH=""
+export VRPS_ARTIFACT_PATH=""
+export VAPS_ARTIFACT_PATH=""
+if [[ -f "$RUN_DIR/result.ccr" ]]; then
+  CCR_ARTIFACT_PATH="result.ccr"
+fi
+if [[ -f "$RUN_DIR/vrps.csv" ]]; then
+  VRPS_ARTIFACT_PATH="vrps.csv"
+fi
+if [[ -f "$RUN_DIR/vaps.csv" ]]; then
+  VAPS_ARTIFACT_PATH="vaps.csv"
+fi
+
+python3 - <<'PY' >"$RUN_DIR/run-meta.json"
+import os
+import json, socket
+def optional(name):
+    value = os.environ.get(name, "")
+    return value if value else None
+meta = {
+    "schemaVersion": 1,
+    "rp": os.environ["RP_NAME"],
+    "runSeq": int(os.environ["RUN_SEQ"]),
+    "runId": os.environ["RUN_ID"],
+    "host": socket.gethostname(),
+    "command": os.environ["RUN_COMMAND"],
+    "startedAtRfc3339Utc": os.environ["STARTED_AT"],
+    "finishedAtRfc3339Utc": os.environ["FINISHED_AT"],
+    "wallMs": int(os.environ["WALL_MS"]),
+    "exitCode": int(os.environ["EXIT_CODE"]),
+    "success": os.environ["SUCCESS"] == "true",
+    "maxRssKb": {
+        "parent": int(os.environ["PARENT_MAX_RSS_KB"]),
+        "childMax": int(os.environ["CHILD_MAX_RSS_KB"]),
+        "aggregatePeak": int(os.environ["AGGREGATE_PEAK_RSS_KB"]),
+        "sampleIntervalMs": int(os.environ["SAMPLE_MS"]),
+    },
+    "artifacts": {
+        "vrpsCsv": optional("VRPS_ARTIFACT_PATH"),
+        "vapsCsv": optional("VAPS_ARTIFACT_PATH"),
+        "ccr": optional("CCR_ARTIFACT_PATH"),
+        "stdout": "stdout.log",
+        "stderr": "stderr.log",
+    },
+    "counts": {
+        "vrps": int(os.environ["VRPS_COUNT"]),
+        "vaps": int(os.environ["VAPS_COUNT"]),
+    },
+}
+print(json.dumps(meta, indent=2, ensure_ascii=False))
+PY
+
+ln -sfn "runs/$RUN_ID" "$RP_ROOT/latest.tmp"
+mv -Tf "$RP_ROOT/latest.tmp" "$RP_ROOT/latest"
+
+echo "$RUN_ID $RP_NAME exit=$EXIT_CODE wall_ms=$WALL_MS vrps=$VRPS_COUNT vaps=$VAPS_COUNT rss_kb=$AGGREGATE_PEAK_RSS_KB"
+
+if [[ "$RETAIN_RUNS" =~ ^[0-9]+$ ]] && (( RETAIN_RUNS > 0 )); then
+  mapfile -t old_runs < <(find "$RP_ROOT/runs" -maxdepth 1 -type d -name 'run_*' -printf '%f\n' | sort | head -n -"$RETAIN_RUNS" || true)
+  for old_run in "${old_runs[@]}"; do
+    rm -rf "$RP_ROOT/runs/$old_run"
+  done
+fi
+
+exit "$EXIT_CODE"

scripts/inter_rp/sync_remote200_to_231.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+CONFIG_FILE="${INTER_RP_SYNC_CONFIG:-./inter-rp-sync.env}"
+if [[ -f "$CONFIG_FILE" ]]; then
+  # shellcheck disable=SC1090
+  source "$CONFIG_FILE"
+fi
+
+REMOTE200="${REMOTE200:-root@43.110.128.200}"
+REMOTE200_ROOT="${REMOTE200_ROOT:-/root/inter-rp-runners}"
+PEER_ROOT="${PEER_ROOT:-/root/inter-rp-aggregator/synced-from-200}"
+SYNC_INTERVAL_SECS="${SYNC_INTERVAL_SECS:-60}"
+MAX_SYNCS="${MAX_SYNCS:--1}"
+RPS="${RPS:-routinator rpki-client}"
+
+mkdir -p "$PEER_ROOT"
+
+write_status() {
+  local success="$1"
+  local message="$2"
+  export SYNC_SUCCESS="$success"
+  export SYNC_MESSAGE="$message"
+  export SYNC_REMOTE200="$REMOTE200"
+  export SYNC_REMOTE200_ROOT="$REMOTE200_ROOT"
+  python3 - <<'PY' >"$PEER_ROOT/sync-status.json"
+import json, socket, datetime
+print(json.dumps({
+    "schemaVersion": 1,
+    "success": __import__("os").environ["SYNC_SUCCESS"] == "true",
+    "lastSyncAtRfc3339Utc": datetime.datetime.now(datetime.UTC).replace(microsecond=0).isoformat().replace('+00:00', 'Z'),
+    "remoteHost": __import__("os").environ["SYNC_REMOTE200"],
+    "remoteRoot": __import__("os").environ["SYNC_REMOTE200_ROOT"],
+    "localHost": socket.gethostname(),
+    "message": __import__("os").environ["SYNC_MESSAGE"],
+}, indent=2))
+PY
+}
+
+sync_once() {
+  local rp_name
+  local temp_root="$PEER_ROOT/.sync-tmp-$$"
+  rm -rf "$temp_root"
+  mkdir -p "$temp_root"
+  for rp_name in $RPS; do
+    mkdir -p "$temp_root/$rp_name"
+    if ! rsync -aL --delete \
+      --include='run-meta.json' \
+      --include='result.ccr' \
+      --include='vrps.csv' \
+      --include='vaps.csv' \
+      --include='stdout.log' \
+      --include='stderr.log' \
+      --exclude='*' \
+      "$REMOTE200:$REMOTE200_ROOT/$rp_name/latest/" "$temp_root/$rp_name/latest/"; then
+      echo "sync failed for $rp_name; clearing stale local latest" >&2
+      rm -rf "$PEER_ROOT/$rp_name/latest" "$PEER_ROOT/$rp_name/latest.next" "$PEER_ROOT/$rp_name/latest.prev"
+      return 1
+    fi
+    if [[ ! -d "$temp_root/$rp_name/latest" ]]; then
+      echo "missing synced latest directory for $rp_name" >&2
+      rm -rf "$PEER_ROOT/$rp_name/latest" "$PEER_ROOT/$rp_name/latest.next" "$PEER_ROOT/$rp_name/latest.prev"
+      return 1
+    fi
+    rm -rf "$PEER_ROOT/$rp_name/latest.next" "$PEER_ROOT/$rp_name/latest.prev"
+    mkdir -p "$PEER_ROOT/$rp_name"
+    mv "$temp_root/$rp_name/latest" "$PEER_ROOT/$rp_name/latest.next"
+    if [[ -e "$PEER_ROOT/$rp_name/latest" ]]; then
+      mv "$PEER_ROOT/$rp_name/latest" "$PEER_ROOT/$rp_name/latest.prev"
+    fi
+    mv "$PEER_ROOT/$rp_name/latest.next" "$PEER_ROOT/$rp_name/latest"
+    rm -rf "$PEER_ROOT/$rp_name/latest.prev"
+  done
+  rm -rf "$temp_root"
+}
+
+completed="0"
+while true; do
+  if sync_once; then
+    write_status true "ok"
+  else
+    rm -rf "$PEER_ROOT/.sync-tmp-$$"
+    write_status false "rsync failed"
+  fi
+  completed=$((completed + 1))
+  if [[ "$MAX_SYNCS" =~ ^[0-9]+$ ]] && (( completed >= MAX_SYNCS )); then
+    break
+  fi
+  sleep "$SYNC_INTERVAL_SECS"
+done

scripts/local_repo_replay/build_local_repo_replay_package.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  build_local_repo_replay_package.sh --out <path> [--tar-gz]
+
+Build a standalone local repository tree replay package for Routinator and
+rpki-client. The package does not include repository data and does not include
+materialize tooling.
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SRC_DIR="$ROOT_DIR/scripts/local_repo_replay"
+OUT=""
+TAR_GZ=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --out) OUT="$2"; shift 2 ;;
+    --tar-gz) TAR_GZ=1; shift ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+[[ -n "$OUT" ]] || { usage >&2; exit 2; }
+
+if [[ "$TAR_GZ" -eq 1 ]]; then
+  PACKAGE_DIR="$(mktemp -d)"
+  TARGET_DIR="$PACKAGE_DIR/local-repo-replay-package"
+else
+  TARGET_DIR="$OUT"
+  rm -rf "$TARGET_DIR"
+fi
+
+mkdir -p "$TARGET_DIR/scripts" "$TARGET_DIR/docs" "$TARGET_DIR/examples"
+
+install -m 0755 "$SRC_DIR/run_routinator_from_local_tree.sh" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/run_rpki_client_from_local_tree.sh" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/run_dual_local_tree_replay.sh" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/prepare_tals.py" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/normalize_rp_outputs.py" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/compare_normalized_sets.py" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/summarize_replay.py" "$TARGET_DIR/scripts/"
+install -m 0755 "$ROOT_DIR/scripts/cir/cir-rsync-wrapper" "$TARGET_DIR/scripts/"
+install -m 0755 "$ROOT_DIR/scripts/cir/cir-local-link-sync.py" "$TARGET_DIR/scripts/"
+install -m 0644 "$SRC_DIR/templates/README.md" "$TARGET_DIR/README.md"
+install -m 0644 "$SRC_DIR/templates/docs/input_tree_requirements.md" "$TARGET_DIR/docs/"
+install -m 0644 "$SRC_DIR/templates/docs/offline_replay_limits.md" "$TARGET_DIR/docs/"
+install -m 0644 "$SRC_DIR/templates/docs/output_files.md" "$TARGET_DIR/docs/"
+install -m 0755 "$SRC_DIR/templates/examples/routinator_example.sh" "$TARGET_DIR/examples/"
+install -m 0755 "$SRC_DIR/templates/examples/rpki_client_example.sh" "$TARGET_DIR/examples/"
+install -m 0755 "$SRC_DIR/templates/examples/dual_compare_example.sh" "$TARGET_DIR/examples/"
+cat > "$TARGET_DIR/env.example" <<'EOF'
+# 本地目录树 replay 示例配置。目录树由使用者提前准备，不包含在 package 中。
+TAL_DIR=/data/replay/tals
+MIRROR_ROOT=/data/replay/mirror
+ROUTINATOR_BIN=/opt/routinator/target/release/routinator
+RPKI_CLIENT_BIN=/opt/rpki-client/src/rpki-client
+RPKI_CLIENT_CACHE_DIR=/data/replay/work/rpki-client-cache
+VALIDATION_TIME=2026-05-23T00:00:00Z
+EOF
+
+if grep -R -E 'cir_materialize|repo-bytes\\.db|\\.cir' "$TARGET_DIR/scripts" >/dev/null; then
+  echo "package contains forbidden materialize/repo-bytes implementation references" >&2
+  exit 1
+fi
+
+if [[ "$TAR_GZ" -eq 1 ]]; then
+  mkdir -p "$(dirname "$OUT")"
+  tar -C "$PACKAGE_DIR" -czf "$OUT" local-repo-replay-package
+  rm -rf "$PACKAGE_DIR"
+  echo "$OUT"
+else
+  echo "$TARGET_DIR"
+fi

scripts/local_repo_replay/compare_normalized_sets.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+def read_set(path: Path) -> set[str]:
+    if not path.is_file():
+        return set()
+    return {line.strip() for line in path.read_text(encoding="utf-8", errors="replace").splitlines() if line.strip()}
+
+
+def compare(left: set[str], right: set[str]) -> dict[str, object]:
+    union = left | right
+    inter = left & right
+    return {
+        "left": len(left),
+        "right": len(right),
+        "intersection": len(inter),
+        "onlyLeft": len(left - right),
+        "onlyRight": len(right - left),
+        "jaccard": round(len(inter) / len(union), 8) if union else 1.0,
+        "onlyLeftSamples": sorted(left - right)[:20],
+        "onlyRightSamples": sorted(right - left)[:20],
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--left", type=Path, required=True)
+    parser.add_argument("--right", type=Path, required=True)
+    parser.add_argument("--left-name", default="left")
+    parser.add_argument("--right-name", default="right")
+    parser.add_argument("--out", type=Path, required=True)
+    args = parser.parse_args()
+    result = {
+        "leftName": args.left_name,
+        "rightName": args.right_name,
+        "vrps": compare(read_set(args.left / "vrps.normalized.txt"), read_set(args.right / "vrps.normalized.txt")),
+        "vaps": compare(read_set(args.left / "vaps.normalized.txt"), read_set(args.right / "vaps.normalized.txt")),
+    }
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    print(args.out)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/local_repo_replay/normalize_rp_outputs.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+from pathlib import Path
+from typing import Any
+
+
+def normalize_asn(value: Any) -> str:
+    text = str(value).strip().upper()
+    if text.startswith("AS"):
+        text = text[2:]
+    return f"AS{int(text)}"
+
+
+def write_set(path: Path, rows: set[str]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("\n".join(sorted(rows)) + ("\n" if rows else ""), encoding="utf-8")
+
+
+def load_json(path: Path) -> Any:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def normalize_vrps_from_json(path: Path) -> set[str]:
+    data = load_json(path)
+    rows: set[str] = set()
+    objects: list[dict[str, Any]] = []
+    if isinstance(data, dict):
+        for key in ("roas", "routeOrigins", "valid_roas"):
+            value = data.get(key)
+            if isinstance(value, list):
+                objects.extend(item for item in value if isinstance(item, dict))
+    elif isinstance(data, list):
+        objects = [item for item in data if isinstance(item, dict)]
+    for item in objects:
+        asn = item.get("asn") or item.get("asID") or item.get("origin")
+        prefix = item.get("prefix")
+        max_length = item.get("maxLength") or item.get("max_length") or item.get("maxlen") or item.get("maxLengthPrefix")
+        if asn is None or prefix is None or max_length is None:
+            continue
+        rows.add(f"{normalize_asn(asn)}|{prefix}|{int(max_length)}")
+    return rows
+
+
+def normalize_vaps_from_json(path: Path) -> set[str]:
+    data = load_json(path)
+    rows: set[str] = set()
+    objects: list[dict[str, Any]] = []
+    if isinstance(data, dict):
+        for key in ("aspas", "aspaAssertions", "vaps"):
+            value = data.get(key)
+            if isinstance(value, list):
+                objects.extend(item for item in value if isinstance(item, dict))
+    elif isinstance(data, list):
+        objects = [item for item in data if isinstance(item, dict)]
+    for item in objects:
+        customer = (
+            item.get("customer")
+            or item.get("customer_asid")
+            or item.get("customerASID")
+            or item.get("customerAsid")
+            or item.get("customerAsn")
+        )
+        providers = item.get("providers") or item.get("provider_asns") or item.get("providerASNs") or []
+        if customer is None:
+            continue
+        provider_asns = [normalize_asn(provider) for provider in providers]
+        rows.add(f"{normalize_asn(customer)}|{','.join(sorted(set(provider_asns), key=lambda value: int(value[2:])))}")
+    return rows
+
+
+def normalize_vrps_from_csv(path: Path) -> set[str]:
+    rows: set[str] = set()
+    with path.open(newline="", encoding="utf-8", errors="replace") as handle:
+        for row in csv.DictReader(handle):
+            asn = row.get("ASN") or row.get("asn") or row.get("AS")
+            prefix = row.get("IP Prefix") or row.get("prefix") or row.get("Prefix")
+            max_length = row.get("Max Length") or row.get("maxLength") or row.get("max_length")
+            if asn and prefix and max_length:
+                rows.add(f"{normalize_asn(asn)}|{prefix}|{int(max_length)}")
+    return rows
+
+
+def normalize_routinator(input_path: Path) -> tuple[set[str], set[str]]:
+    return normalize_vrps_from_json(input_path), normalize_vaps_from_json(input_path)
+
+
+def normalize_rpki_client(input_path: Path) -> tuple[set[str], set[str]]:
+    json_path = input_path / "json" if input_path.is_dir() else input_path
+    csv_path = input_path / "csv" if input_path.is_dir() else input_path
+    vrps: set[str] = set()
+    vaps: set[str] = set()
+    if json_path.is_file():
+        vrps |= normalize_vrps_from_json(json_path)
+        vaps |= normalize_vaps_from_json(json_path)
+    if csv_path.is_file():
+        vrps |= normalize_vrps_from_csv(csv_path)
+    return vrps, vaps
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--kind", choices=["routinator", "rpki-client"], required=True)
+    parser.add_argument("--input", type=Path, required=True)
+    parser.add_argument("--vrps-out", type=Path, required=True)
+    parser.add_argument("--vaps-out", type=Path, required=True)
+    args = parser.parse_args()
+
+    if args.kind == "routinator":
+        vrps, vaps = normalize_routinator(args.input)
+    else:
+        vrps, vaps = normalize_rpki_client(args.input)
+    write_set(args.vrps_out, vrps)
+    write_set(args.vaps_out, vaps)
+    print(json.dumps({"vrps": len(vrps), "vaps": len(vaps)}, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/local_repo_replay/prepare_tals.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+def split_tal(text: str) -> tuple[list[str], list[str]]:
+    lines = text.splitlines()
+    for index, line in enumerate(lines):
+        if line.strip() == "":
+            return lines[:index], lines[index + 1 :]
+    return lines, []
+
+
+def rsync_only_tal(source: Path) -> tuple[str, dict[str, object]]:
+    text = source.read_text(encoding="utf-8", errors="replace")
+    uri_lines, key_lines = split_tal(text)
+    uris = [
+        line.strip()
+        for line in uri_lines
+        if line.strip() and not line.lstrip().startswith("#")
+    ]
+    rsync_uris = [
+        uri for uri in uris
+        if uri.lower().startswith("rsync://")
+    ]
+    if not rsync_uris:
+        return text if text.endswith("\n") else text + "\n", {
+            "file": source.name,
+            "mode": "unchanged_no_rsync_uri",
+            "uris": len(uris),
+            "rsyncUris": 0,
+        }
+    out = "\n".join(rsync_uris) + "\n\n" + "\n".join(key_lines).strip() + "\n"
+    return out, {
+        "file": source.name,
+        "mode": "rsync_only",
+        "uris": len(uris),
+        "rsyncUris": len(rsync_uris),
+    }
+
+
+def collect_tals(tal_dir: Path | None, tals: list[Path]) -> list[Path]:
+    paths: list[Path] = []
+    if tal_dir is not None:
+        paths.extend(sorted(tal_dir.glob("*.tal")))
+    paths.extend(tals)
+    unique: dict[str, Path] = {}
+    for path in paths:
+        unique[path.name] = path
+    return [unique[name] for name in sorted(unique)]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Prepare TALs for local rsync-tree replay.")
+    parser.add_argument("--tal-dir", type=Path)
+    parser.add_argument("--tal", type=Path, action="append", default=[])
+    parser.add_argument("--out-dir", type=Path, required=True)
+    parser.add_argument("--summary", type=Path)
+    args = parser.parse_args()
+
+    sources = collect_tals(args.tal_dir, args.tal)
+    if not sources:
+        raise SystemExit("no TAL files provided")
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    rows = []
+    for source in sources:
+        if not source.is_file():
+            raise SystemExit(f"TAL file not found: {source}")
+        content, row = rsync_only_tal(source)
+        (args.out_dir / source.name).write_text(content, encoding="utf-8")
+        rows.append(row)
+
+    if args.summary is not None:
+        args.summary.parent.mkdir(parents=True, exist_ok=True)
+        args.summary.write_text(json.dumps(rows, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/local_repo_replay/run_dual_local_tree_replay.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  run_dual_local_tree_replay.sh \
+    --routinator-bin <path> --routinator-mirror-root <dir> \
+    --rpki-client-bin <path> --rpki-client-mirror-root <dir> \
+    --tal-dir <dir> --out-dir <dir> [--validation-time <RFC3339>]
+EOF
+}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROUTINATOR_BIN=""
+ROUTINATOR_MIRROR_ROOT=""
+RPKI_CLIENT_BIN=""
+RPKI_CLIENT_MIRROR_ROOT=""
+RPKI_CLIENT_CACHE_DIR=""
+TAL_DIR=""
+OUT_DIR=""
+VALIDATION_TIME=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
+    --routinator-mirror-root) ROUTINATOR_MIRROR_ROOT="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --rpki-client-mirror-root) RPKI_CLIENT_MIRROR_ROOT="$2"; shift 2 ;;
+    --rpki-client-cache-dir) RPKI_CLIENT_CACHE_DIR="$2"; shift 2 ;;
+    --tal-dir) TAL_DIR="$2"; shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --validation-time) VALIDATION_TIME="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+if [[ -z "$RPKI_CLIENT_MIRROR_ROOT" ]]; then
+  RPKI_CLIENT_MIRROR_ROOT="$ROUTINATOR_MIRROR_ROOT"
+fi
+[[ -n "$ROUTINATOR_BIN" && -n "$ROUTINATOR_MIRROR_ROOT" && -n "$RPKI_CLIENT_BIN" && -n "$RPKI_CLIENT_MIRROR_ROOT" && -n "$TAL_DIR" && -n "$OUT_DIR" ]] || { usage >&2; exit 2; }
+
+mkdir -p "$OUT_DIR"
+TIME_ARGS=()
+if [[ -n "$VALIDATION_TIME" ]]; then
+  TIME_ARGS=(--validation-time "$VALIDATION_TIME")
+fi
+CACHE_ARGS=()
+if [[ -n "$RPKI_CLIENT_CACHE_DIR" ]]; then
+  CACHE_ARGS=(--cache-dir "$RPKI_CLIENT_CACHE_DIR")
+fi
+
+"$SCRIPT_DIR/run_routinator_from_local_tree.sh" \
+  --routinator-bin "$ROUTINATOR_BIN" \
+  --mirror-root "$ROUTINATOR_MIRROR_ROOT" \
+  --tal-dir "$TAL_DIR" \
+  --out-dir "$OUT_DIR/routinator" \
+  --enable-aspa \
+  "${TIME_ARGS[@]}"
+
+"$SCRIPT_DIR/run_rpki_client_from_local_tree.sh" \
+  --rpki-client-bin "$RPKI_CLIENT_BIN" \
+  --mirror-root "$RPKI_CLIENT_MIRROR_ROOT" \
+  --tal-dir "$TAL_DIR" \
+  --out-dir "$OUT_DIR/rpki-client" \
+  "${CACHE_ARGS[@]}" \
+  "${TIME_ARGS[@]}"
+
+python3 "$SCRIPT_DIR/compare_normalized_sets.py" \
+  --left "$OUT_DIR/routinator" \
+  --right "$OUT_DIR/rpki-client" \
+  --left-name routinator \
+  --right-name rpki-client \
+  --out "$OUT_DIR/compare-summary.json"
+echo "done: $OUT_DIR"

scripts/local_repo_replay/run_routinator_from_local_tree.sh

@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  run_routinator_from_local_tree.sh \
+    --routinator-bin <path> \
+    --mirror-root <local-rsync-mirror-root> \
+    --tal-dir <dir> | --tal <file>... \
+    --out-dir <path> \
+    [--validation-time <RFC3339>] \
+    [--real-rsync-bin <path>] \
+    [--enable-aspa]
+
+The input mirror is prepared by the caller. This script does not generate it.
+Pass --validation-time only if FAKETIME_LIB points to a working libfaketime
+library; otherwise Routinator will validate at wall-clock time.
+Example:
+  export FAKETIME_LIB=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1
+EOF
+}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROUTINATOR_BIN=""
+MIRROR_ROOT=""
+OUT_DIR=""
+VALIDATION_TIME=""
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+ENABLE_ASPA=0
+TAL_DIR=""
+TALS=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
+    --mirror-root) MIRROR_ROOT="$2"; shift 2 ;;
+    --tal-dir) TAL_DIR="$2"; shift 2 ;;
+    --tal) TALS+=("$2"); shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --validation-time) VALIDATION_TIME="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    --enable-aspa) ENABLE_ASPA=1; shift ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+[[ -n "$ROUTINATOR_BIN" && -n "$MIRROR_ROOT" && -n "$OUT_DIR" ]] || { usage >&2; exit 2; }
+[[ -x "$ROUTINATOR_BIN" ]] || { echo "routinator binary not executable: $ROUTINATOR_BIN" >&2; exit 2; }
+[[ -d "$MIRROR_ROOT" ]] || { echo "mirror root not found: $MIRROR_ROOT" >&2; exit 2; }
+if [[ -z "$TAL_DIR" && "${#TALS[@]}" -eq 0 ]]; then
+  echo "provide --tal-dir or at least one --tal" >&2
+  exit 2
+fi
+
+mkdir -p "$OUT_DIR"
+WORK_DIR="$OUT_DIR/work"
+REPO_DIR="$WORK_DIR/repository"
+EXTRA_TALS="$WORK_DIR/tals"
+CONFIG_FILE="$WORK_DIR/routinator.conf"
+rm -rf "$WORK_DIR"
+mkdir -p "$REPO_DIR" "$EXTRA_TALS"
+
+PREPARE_TAL_ARGS=(--out-dir "$EXTRA_TALS" --summary "$OUT_DIR/prepared-tals.json")
+if [[ -n "$TAL_DIR" ]]; then
+  PREPARE_TAL_ARGS+=(--tal-dir "$TAL_DIR")
+fi
+for tal in "${TALS[@]}"; do
+  PREPARE_TAL_ARGS+=(--tal "$tal")
+done
+python3 "$SCRIPT_DIR/prepare_tals.py" "${PREPARE_TAL_ARGS[@]}"
+
+cat > "$CONFIG_FILE" <<EOF
+repository-dir = "$REPO_DIR"
+no-rir-tals = true
+extra-tals-dir = "$EXTRA_TALS"
+disable-rrdp = true
+rrdp-fallback = "never"
+rsync-command = "$SCRIPT_DIR/cir-rsync-wrapper"
+EOF
+
+export CIR_MIRROR_ROOT="$(cd "$MIRROR_ROOT" && pwd)"
+export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
+export CIR_LOCAL_LINK_MODE=1
+export LOCAL_REPO_REPLAY_VALIDATION_TIME="$VALIDATION_TIME"
+export LOCAL_REPO_REPLAY_OUT_DIR="$OUT_DIR"
+
+ARGS=(
+  "$ROUTINATOR_BIN"
+  --config "$CONFIG_FILE"
+  --repository-dir "$REPO_DIR"
+  --disable-rrdp
+  --rrdp-fallback never
+  --rsync-command "$SCRIPT_DIR/cir-rsync-wrapper"
+  --no-rir-tals
+  --extra-tals-dir "$EXTRA_TALS"
+)
+if [[ "$ENABLE_ASPA" -eq 1 ]]; then
+  echo 'enable-aspa = true' >> "$CONFIG_FILE"
+  ARGS+=(--enable-aspa)
+fi
+if [[ -n "$VALIDATION_TIME" && -z "${FAKETIME_LIB:-}" ]]; then
+  echo "warning: --validation-time is ignored for Routinator because FAKETIME_LIB is not set" >&2
+fi
+VRP_ARGS=(vrps -o "$OUT_DIR/vrps.csv")
+JSON_ARGS=(vrps -f jsonext -o "$OUT_DIR/routinator.json")
+
+/usr/bin/time -v -o "$OUT_DIR/process-time.txt" bash -c '
+  set -euo pipefail
+  if [[ -n "$LOCAL_REPO_REPLAY_VALIDATION_TIME" && -n "${FAKETIME_LIB:-}" ]]; then
+    faketime_value="$(python3 - <<'"'"'PY'"'"' "$LOCAL_REPO_REPLAY_VALIDATION_TIME"
+from datetime import datetime, timezone
+import sys
+dt = datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).astimezone(timezone.utc)
+print("@" + dt.strftime("%Y-%m-%d %H:%M:%S"))
+PY
+)"
+    export LD_PRELOAD="$FAKETIME_LIB"
+    export TZ=UTC
+    unset FAKETIME_FMT
+    export FAKETIME="$faketime_value"
+    export FAKETIME_DONT_FAKE_MONOTONIC=1
+  fi
+  "$@" update --complete >"$LOCAL_REPO_REPLAY_OUT_DIR/update.log" 2>&1 || true
+  "$@" vrps -o "$LOCAL_REPO_REPLAY_OUT_DIR/vrps.csv" >"$LOCAL_REPO_REPLAY_OUT_DIR/vrps.log" 2>&1
+  "$@" vrps -f jsonext -o "$LOCAL_REPO_REPLAY_OUT_DIR/routinator.json" >"$LOCAL_REPO_REPLAY_OUT_DIR/json.log" 2>&1
+' local_repo_replay "${ARGS[@]}"
+python3 "$SCRIPT_DIR/normalize_rp_outputs.py" --kind routinator --input "$OUT_DIR/routinator.json" --vrps-out "$OUT_DIR/vrps.normalized.txt" --vaps-out "$OUT_DIR/vaps.normalized.txt"
+python3 "$SCRIPT_DIR/summarize_replay.py" --rp routinator --out-dir "$OUT_DIR" --summary "$OUT_DIR/summary.json"
+echo "done: $OUT_DIR"

scripts/local_repo_replay/run_rpki_client_from_local_tree.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  run_rpki_client_from_local_tree.sh \
+    --rpki-client-bin <path> \
+    --mirror-root <local-rsync-mirror-root> \
+    --tal-dir <dir> | --tal <file>... \
+    --out-dir <path> \
+    [--cache-dir <work-cache-dir>] \
+    [--validation-time <RFC3339>] \
+    [--real-rsync-bin <path>] \
+    [--parser-workers <n>]
+
+The input mirror is prepared by the caller. This script disables RRDP, points
+rpki-client's rsync command at a local URI mapper, and fetches only from the
+local filesystem tree.
+EOF
+}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+RPKI_CLIENT_BIN=""
+CACHE_DIR=""
+MIRROR_ROOT=""
+OUT_DIR=""
+TAL_DIR=""
+VALIDATION_TIME=""
+PARSER_WORKERS="${PARSER_WORKERS:-4}"
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+TALS=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --mirror-root) MIRROR_ROOT="$2"; shift 2 ;;
+    --cache-dir) CACHE_DIR="$2"; shift 2 ;;
+    --tal-dir) TAL_DIR="$2"; shift 2 ;;
+    --tal) TALS+=("$2"); shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --validation-time) VALIDATION_TIME="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    --parser-workers) PARSER_WORKERS="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+[[ -n "$RPKI_CLIENT_BIN" && -n "$MIRROR_ROOT" && -n "$OUT_DIR" ]] || { usage >&2; exit 2; }
+[[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
+[[ -d "$MIRROR_ROOT" ]] || { echo "mirror root not found: $MIRROR_ROOT" >&2; exit 2; }
+if [[ -z "$TAL_DIR" && "${#TALS[@]}" -eq 0 ]]; then
+  echo "provide --tal-dir or at least one --tal" >&2
+  exit 2
+fi
+
+mkdir -p "$OUT_DIR"
+if [[ -z "$CACHE_DIR" ]]; then
+  CACHE_DIR="$OUT_DIR/cache"
+fi
+mkdir -p "$CACHE_DIR" "$OUT_DIR/out"
+chmod a+rwx "$OUT_DIR" "$CACHE_DIR" "$OUT_DIR/out"
+WORK_DIR="$OUT_DIR/work"
+PREPARED_TALS="$WORK_DIR/tals"
+rm -rf "$WORK_DIR"
+mkdir -p "$PREPARED_TALS"
+PREPARE_TAL_ARGS=(--out-dir "$PREPARED_TALS" --summary "$OUT_DIR/prepared-tals.json")
+if [[ -n "$TAL_DIR" ]]; then
+  PREPARE_TAL_ARGS+=(--tal-dir "$TAL_DIR")
+fi
+for tal in "${TALS[@]}"; do
+  PREPARE_TAL_ARGS+=(--tal "$tal")
+done
+python3 "$SCRIPT_DIR/prepare_tals.py" "${PREPARE_TAL_ARGS[@]}"
+
+CLIENT_TAL_ARGS=()
+while IFS= read -r tal; do
+  CLIENT_TAL_ARGS+=(-t "$tal")
+done < <(find "$PREPARED_TALS" -maxdepth 1 -type f -name '*.tal' | sort)
+
+TIME_ARGS=()
+if [[ -n "$VALIDATION_TIME" ]]; then
+  epoch="$(python3 - <<'PY' "$VALIDATION_TIME"
+from datetime import datetime, timezone
+import sys
+print(int(datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).astimezone(timezone.utc).timestamp()))
+PY
+)"
+  TIME_ARGS=(-P "$epoch")
+fi
+
+export CIR_MIRROR_ROOT="$(cd "$MIRROR_ROOT" && pwd)"
+export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
+export CIR_LOCAL_LINK_MODE=1
+
+/usr/bin/time -v -o "$OUT_DIR/process-time.txt" \
+  "$RPKI_CLIENT_BIN" \
+  -R -e "$SCRIPT_DIR/cir-rsync-wrapper" -j -c -p "$PARSER_WORKERS" \
+  "${TIME_ARGS[@]}" \
+  "${CLIENT_TAL_ARGS[@]}" \
+  -d "$CACHE_DIR" \
+  "$OUT_DIR/out" >"$OUT_DIR/stdout.log" 2>"$OUT_DIR/stderr.log"
+
+python3 "$SCRIPT_DIR/normalize_rp_outputs.py" --kind rpki-client --input "$OUT_DIR/out" --vrps-out "$OUT_DIR/vrps.normalized.txt" --vaps-out "$OUT_DIR/vaps.normalized.txt"
+python3 "$SCRIPT_DIR/summarize_replay.py" --rp rpki-client --out-dir "$OUT_DIR" --summary "$OUT_DIR/summary.json"
+echo "done: $OUT_DIR"

scripts/local_repo_replay/summarize_replay.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+
+
+def count_lines(path: Path) -> int:
+    if not path.is_file():
+        return 0
+    return sum(1 for line in path.read_text(encoding="utf-8", errors="replace").splitlines() if line.strip())
+
+
+def parse_time(path: Path) -> dict[str, object]:
+    if not path.is_file():
+        return {}
+    text = path.read_text(encoding="utf-8", errors="replace")
+    elapsed = re.search(r"Elapsed \(wall clock\) time .*: (.+)", text)
+    rss = re.search(r"Maximum resident set size \(kbytes\): (\d+)", text)
+    return {
+        "elapsed": elapsed.group(1).strip() if elapsed else "",
+        "maxRssKb": int(rss.group(1)) if rss else 0,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--rp", required=True)
+    parser.add_argument("--out-dir", type=Path, required=True)
+    parser.add_argument("--summary", type=Path, required=True)
+    args = parser.parse_args()
+    summary = {
+        "rp": args.rp,
+        "vrps": count_lines(args.out_dir / "vrps.normalized.txt"),
+        "vaps": count_lines(args.out_dir / "vaps.normalized.txt"),
+        "time": parse_time(args.out_dir / "process-time.txt"),
+        "artifacts": sorted(path.name for path in args.out_dir.iterdir() if path.is_file()),
+    }
+    args.summary.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    print(args.summary)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/local_repo_replay/templates/README.md

@@ -0,0 +1,118 @@
+# Local Repository Tree Replay Package
+
+This package replays already prepared local RPKI repository trees with
+Routinator and rpki-client.
+
+It is intentionally independent from CIR:
+
+- it does not read `.cir`;
+- it does not read `repo-bytes.db`;
+- it does not call `cir_materialize`;
+- it does not generate a local repository tree.
+
+The caller must prepare the local repository/cache tree before running these
+scripts.
+
+## Contents
+
+```text
+local-repo-replay-package/
+  scripts/
+    run_routinator_from_local_tree.sh
+    run_rpki_client_from_local_tree.sh
+    run_dual_local_tree_replay.sh
+    prepare_tals.py
+    cir-rsync-wrapper
+    cir-local-link-sync.py
+    normalize_rp_outputs.py
+    compare_normalized_sets.py
+    summarize_replay.py
+  docs/
+    input_tree_requirements.md
+    offline_replay_limits.md
+    output_files.md
+  examples/
+    routinator_example.sh
+    rpki_client_example.sh
+    dual_compare_example.sh
+  env.example
+```
+
+## Routinator replay
+
+```bash
+./scripts/run_routinator_from_local_tree.sh \
+  --routinator-bin /opt/routinator/target/release/routinator \
+  --mirror-root /data/replay/mirror \
+  --tal-dir /data/replay/tals \
+  --out-dir /data/replay/out/routinator \
+  --enable-aspa
+```
+
+The script uses `--disable-rrdp`, `--rsync-command ./scripts/cir-rsync-wrapper`,
+and the local mirror root to satisfy rsync fetches from the local filesystem.
+The wrapper name is historical; in this package it is only a generic
+`rsync://` to local-path mapper.
+
+If `--validation-time` is needed for Routinator, set `FAKETIME_LIB` to a working
+libfaketime shared library. Otherwise Routinator validates at wall-clock time.
+
+On Ubuntu, install and use faketime like this:
+
+```bash
+sudo apt-get install -y libfaketime
+export FAKETIME_LIB=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1
+./scripts/run_routinator_from_local_tree.sh \
+  --routinator-bin /opt/routinator/target/release/routinator \
+  --mirror-root /data/replay/mirror \
+  --tal-dir /data/replay/tals \
+  --out-dir /data/replay/out/routinator \
+  --validation-time 2026-05-14T06:48:00Z \
+  --enable-aspa
+```
+
+Without `FAKETIME_LIB`, old local trees can produce empty or smaller output
+because Routinator validates manifests and CRLs against current wall-clock time.
+
+## rpki-client replay
+
+```bash
+./scripts/run_rpki_client_from_local_tree.sh \
+  --rpki-client-bin /opt/rpki-client/src/rpki-client \
+  --mirror-root /data/replay/mirror \
+  --tal-dir /data/replay/tals \
+  --out-dir /data/replay/out/rpki-client \
+  --parser-workers 4
+```
+
+The script uses `rpki-client -R -e ./scripts/cir-rsync-wrapper` so RRDP is
+disabled and rsync fetches are served from the local mirror. `--cache-dir` is an
+optional working cache directory used by rpki-client during this local replay.
+
+## Dual replay
+
+```bash
+./scripts/run_dual_local_tree_replay.sh \
+  --routinator-bin /opt/routinator/target/release/routinator \
+  --routinator-mirror-root /data/replay/mirror \
+  --rpki-client-bin /opt/rpki-client/src/rpki-client \
+  --rpki-client-mirror-root /data/replay/mirror \
+  --tal-dir /data/replay/tals \
+  --out-dir /data/replay/out/dual
+```
+
+If `--validation-time` is passed to dual replay, remember to export
+`FAKETIME_LIB` first so Routinator and rpki-client use the same logical
+validation time.
+
+## Outputs
+
+Each run writes normalized output:
+
+- `vrps.normalized.txt`
+- `vaps.normalized.txt`
+- `summary.json`
+- raw RP output and logs
+- `process-time.txt`
+
+See `docs/output_files.md`.

scripts/local_repo_replay/templates/docs/input_tree_requirements.md

@@ -0,0 +1,39 @@
+# Input Tree Requirements
+
+The input tree is not part of this package. The caller must prepare it before
+running replay.
+
+## Mirror root
+
+The mirror root must map rsync URIs to local paths:
+
+```text
+rsync://rpki.example.net/repo/a/b/c.roa
+=> <mirror-root>/rpki.example.net/repo/a/b/c.roa
+```
+
+The tree must contain all objects needed by the selected TALs: TA certificates,
+manifests, CRLs, ROAs, ASPAs, router certs, and child CA certificates.
+
+Both Routinator and rpki-client scripts consume this same mirror root through a
+local rsync command wrapper.
+
+## rpki-client working cache
+
+For rpki-client replay, `--cache-dir` is only rpki-client's working cache
+directory for this local run. It is not the input dataset. The authoritative
+input is `--mirror-root`.
+
+## TALs
+
+Provide either `--tal-dir <dir>` or repeated `--tal <file>`.
+
+The scripts prepare a replay-local TAL copy that prefers `rsync://` TA
+certificate URIs. This prevents a TAL with an HTTPS URI listed first from
+escaping to the network during local replay. The TAL set should match the local
+tree. Mixing a tree from one run with different TALs may produce meaningless
+differences.
+
+## No generation
+
+This package does not generate the tree and does not repair missing objects.

scripts/local_repo_replay/templates/docs/offline_replay_limits.md

@@ -0,0 +1,38 @@
+# Offline Replay Limits
+
+## Routinator
+
+The Routinator script disables RRDP and uses an rsync command wrapper to map
+rsync URLs to local paths. It still runs Routinator's normal validation logic.
+
+If the local mirror does not contain required objects, validation can fail or
+produce fewer outputs.
+
+## rpki-client
+
+The rpki-client script uses `-R` to disable RRDP and `-e` to point rsync at the
+local mapper. rpki-client still builds its normal working cache, but every
+rsync source is rewritten to the local mirror.
+
+If the mirror was incomplete or produced by a different TAL set, replay results
+may differ from the original run.
+
+## Validation time
+
+rpki-client supports `-P <posix-seconds>`. Routinator does not expose the same
+simple command-line evaluation-time option in the tested version; if `FAKETIME_LIB`
+is configured, the script can run Routinator under faketime. Without
+`FAKETIME_LIB`, `--validation-time` is intentionally ignored for Routinator and
+current wall-clock validation can reject stale manifests or CRLs.
+
+Ubuntu example:
+
+```bash
+sudo apt-get install -y libfaketime
+export FAKETIME_LIB=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1
+```
+
+The script sets `TZ=UTC` and converts RFC3339 validation time to libfaketime
+absolute UTC format, for example `2026-05-14T06:48:00Z` becomes
+`@2026-05-14 06:48:00`. Setting `TZ=UTC` is required because libfaketime parses
+absolute timestamps in the process timezone.

scripts/local_repo_replay/templates/docs/output_files.md

@@ -0,0 +1,16 @@
+# Output Files
+
+Each replay output directory can contain:
+
+- `vrps.normalized.txt`: one normalized VRP per line.
+- `vaps.normalized.txt`: one normalized VAP/ASPA per line.
+- `summary.json`: counts and resource summary.
+- `process-time.txt`: `/usr/bin/time -v` output.
+- RP-specific raw output:
+  - Routinator: `routinator.json`, `vrps.csv`.
+  - rpki-client: `out/json`, `out/csv`, and other native files.
+- logs:
+  - `stdout.log` / `stderr.log` for rpki-client.
+  - `update.log`, `vrps.log`, `json.log` for Routinator.
+
+Dual replay additionally writes `compare-summary.json`.

scripts/local_repo_replay/templates/examples/dual_compare_example.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+./scripts/run_dual_local_tree_replay.sh \
+  --routinator-bin "${ROUTINATOR_BIN:-/opt/routinator/target/release/routinator}" \
+  --routinator-mirror-root "${ROUTINATOR_MIRROR_ROOT:-/data/replay/mirror}" \
+  --rpki-client-bin "${RPKI_CLIENT_BIN:-/opt/rpki-client/src/rpki-client}" \
+  --rpki-client-mirror-root "${RPKI_CLIENT_MIRROR_ROOT:-/data/replay/mirror}" \
+  --rpki-client-cache-dir "${RPKI_CLIENT_CACHE_DIR:-/data/replay/work/rpki-client-cache}" \
+  --tal-dir "${TAL_DIR:-/data/replay/tals}" \
+  --out-dir "${OUT_DIR:-/data/replay/out/dual}"

scripts/local_repo_replay/templates/examples/routinator_example.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+./scripts/run_routinator_from_local_tree.sh \
+  --routinator-bin "${ROUTINATOR_BIN:-/opt/routinator/target/release/routinator}" \
+  --mirror-root "${MIRROR_ROOT:-/data/replay/mirror}" \
+  --tal-dir "${TAL_DIR:-/data/replay/tals}" \
+  --out-dir "${OUT_DIR:-/data/replay/out/routinator}" \
+  --enable-aspa

scripts/local_repo_replay/templates/examples/rpki_client_example.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+./scripts/run_rpki_client_from_local_tree.sh \
+  --rpki-client-bin "${RPKI_CLIENT_BIN:-/opt/rpki-client/src/rpki-client}" \
+  --mirror-root "${MIRROR_ROOT:-/data/replay/mirror}" \
+  --cache-dir "${RPKI_CLIENT_CACHE_DIR:-/data/replay/work/rpki-client-cache}" \
+  --tal-dir "${TAL_DIR:-/data/replay/tals}" \
+  --out-dir "${OUT_DIR:-/data/replay/out/rpki-client}" \
+  --parser-workers "${PARSER_WORKERS:-4}"

scripts/manifest_perf_compare_m2.sh

@@ -0,0 +1,195 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# M2: Run per-sample decode+profile benchmark (Ours vs Routinator) on selected_der fixtures.
+#
+# Outputs:
+# - specs/develop/20260224/data/m2_manifest_decode_profile_compare.csv
+# - specs/develop/20260224/data/m2_raw.log
+#
+# Note: This script assumes Routinator benchmark repo exists at:
+#   /home/yuyr/dev/rust_playground/routinator/benchmark
+#
+# It also assumes fixtures exist under:
+#   rpki/tests/benchmark/selected_der/*.mft
+#   routinator/benchmark/fixtures/selected_der/*.mft
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+RPKI_DIR="$ROOT_DIR"
+OURS_BENCH_DIR="$RPKI_DIR/benchmark/ours_manifest_bench"
+
+ROUT_BENCH_DIR="${ROUT_BENCH_DIR:-/home/yuyr/dev/rust_playground/routinator/benchmark}"
+ROUT_BIN="$ROUT_BENCH_DIR/target/release/routinator-manifest-benchmark"
+
+DATE_TAG="${DATE_TAG:-20260224}"
+OUT_DIR="$RPKI_DIR/../specs/develop/${DATE_TAG}/data"
+OUT_CSV="${OUT_CSV:-$OUT_DIR/m2_manifest_decode_profile_compare.csv}"
+OUT_RAW="${OUT_RAW:-$OUT_DIR/m2_raw.log}"
+
+REPEATS="${REPEATS:-3}"
+
+# Iterations / warmups (kept moderate for interactive iteration).
+ITER_SMALL="${ITER_SMALL:-20000}"
+ITER_MEDIUM="${ITER_MEDIUM:-20000}"
+ITER_LARGE="${ITER_LARGE:-20000}"
+ITER_XLARGE="${ITER_XLARGE:-2000}"
+
+WARM_SMALL="${WARM_SMALL:-2000}"
+WARM_MEDIUM="${WARM_MEDIUM:-2000}"
+WARM_LARGE="${WARM_LARGE:-2000}"
+WARM_XLARGE="${WARM_XLARGE:-200}"
+
+SAMPLES=(
+  small-01
+  small-02
+  medium-01
+  medium-02
+  large-01
+  large-02
+  xlarge-01
+  xlarge-02
+)
+
+mkdir -p "$OUT_DIR"
+: > "$OUT_RAW"
+
+echo "sample,bucket,manifest_file_count,ours_avg_ns_per_op,ours_ops_per_s,rout_avg_ns_per_op,rout_ops_per_s,ratio_ours_over_rout,iterations,repeats,warmup" > "$OUT_CSV"
+
+echo "[1/3] Build ours benchmark (release)..." | tee -a "$OUT_RAW"
+(cd "$OURS_BENCH_DIR" && cargo build --release -q)
+OURS_BIN="$OURS_BENCH_DIR/target/release/ours-manifest-bench"
+
+echo "[2/3] Build routinator benchmark (release)..." | tee -a "$OUT_RAW"
+(cd "$ROUT_BENCH_DIR" && cargo build --release -q)
+
+taskset_prefix=""
+if command -v taskset >/dev/null 2>&1; then
+  if [[ -n "${TASKSET_CPU:-}" ]]; then
+    taskset_prefix="taskset -c ${TASKSET_CPU}"
+  fi
+fi
+
+bucket_for() {
+  local s="$1"
+  case "$s" in
+    small-*) echo "small" ;;
+    medium-*) echo "medium" ;;
+    large-*) echo "large" ;;
+    xlarge-*) echo "xlarge" ;;
+    *) echo "unknown" ;;
+  esac
+}
+
+iters_for() {
+  local b="$1"
+  case "$b" in
+    small) echo "$ITER_SMALL" ;;
+    medium) echo "$ITER_MEDIUM" ;;
+    large) echo "$ITER_LARGE" ;;
+    xlarge) echo "$ITER_XLARGE" ;;
+    *) echo "$ITER_MEDIUM" ;;
+  esac
+}
+
+warm_for() {
+  local b="$1"
+  case "$b" in
+    small) echo "$WARM_SMALL" ;;
+    medium) echo "$WARM_MEDIUM" ;;
+    large) echo "$WARM_LARGE" ;;
+    xlarge) echo "$WARM_XLARGE" ;;
+    *) echo "$WARM_MEDIUM" ;;
+  esac
+}
+
+run_ours() {
+  local sample="$1"
+  local iters="$2"
+  local warm="$3"
+  local ours_fixture="$RPKI_DIR/tests/benchmark/selected_der/${sample}.mft"
+  if [[ ! -f "$ours_fixture" ]]; then
+    echo "ours fixture not found: $ours_fixture" >&2
+    exit 1
+  fi
+
+  echo "### ours $sample" >> "$OUT_RAW"
+  local out
+  out=$($taskset_prefix "$OURS_BIN" --manifest "$ours_fixture" --iterations "$iters" --warmup-iterations "$warm" --repeats "$REPEATS")
+  echo "$out" >> "$OUT_RAW"
+
+  local line
+  line=$(echo "$out" | rg "^\\| ${sample} \\|" | tail -n 1)
+  if [[ -z "${line:-}" ]]; then
+    echo "failed to parse ours output for $sample" >&2
+    exit 1
+  fi
+  # Expected final row: | sample | avg ns/op | ops/s | file count |
+  local avg ops cnt
+  avg=$(echo "$line" | awk -F'|' '{gsub(/^ +| +$/,"",$3); print $3}')
+  ops=$(echo "$line" | awk -F'|' '{gsub(/^ +| +$/,"",$4); print $4}')
+  cnt=$(echo "$line" | awk -F'|' '{gsub(/^ +| +$/,"",$5); print $5}')
+  echo "$avg,$ops,$cnt"
+}
+
+run_rout() {
+  local sample="$1"
+  local iters="$2"
+  local warm="$3"
+  local rout_fixture="$ROUT_BENCH_DIR/fixtures/selected_der/${sample}.mft"
+  if [[ ! -f "$rout_fixture" ]]; then
+    echo "routinator fixture not found: $rout_fixture" >&2
+    exit 1
+  fi
+
+  echo "### routinator $sample" >> "$OUT_RAW"
+  local out
+  out=$(
+    $taskset_prefix "$ROUT_BIN" \
+      --target decode_only \
+      --manifest "$rout_fixture" \
+      --issuer "$ROUT_BENCH_DIR/fixtures/ta.cer" \
+      --iterations "$iters" \
+      --repeats "$REPEATS" \
+      --warmup-iterations "$warm" \
+      --strict false
+  )
+  echo "$out" >> "$OUT_RAW"
+
+  local avg_line cnt_line
+  avg_line=$(echo "$out" | rg "^  avg:")
+  cnt_line=$(echo "$out" | rg "^  manifest_file_count:")
+
+  local avg_ns ops_s cnt
+  avg_ns=$(echo "$avg_line" | awk '{print $2}')
+  ops_s=$(echo "$avg_line" | awk '{gsub(/[()]/,"",$4); print $4}')
+  cnt=$(echo "$cnt_line" | awk '{print $2}')
+  echo "$avg_ns,$ops_s,$cnt"
+}
+
+echo "[3/3] Run per-sample benchmarks..." | tee -a "$OUT_RAW"
+for s in "${SAMPLES[@]}"; do
+  b=$(bucket_for "$s")
+  it=$(iters_for "$b")
+  warm=$(warm_for "$b")
+
+  IFS=, read -r ours_avg ours_ops ours_cnt < <(run_ours "$s" "$it" "$warm")
+  IFS=, read -r rout_avg rout_ops rout_cnt < <(run_rout "$s" "$it" "$warm")
+
+  if [[ "$ours_cnt" != "$rout_cnt" ]]; then
+    echo "WARNING: file count differs for $s (ours=$ours_cnt rout=$rout_cnt)" | tee -a "$OUT_RAW"
+  fi
+
+  ratio=$(python3 - <<PY
+o=float("$ours_avg")
+r=float("$rout_avg")
+print(f"{(o/r):.4f}" if r != 0 else "inf")
+PY
+)
+
+  echo "$s,$b,$ours_cnt,$ours_avg,$ours_ops,$rout_avg,$rout_ops,$ratio,$it,$REPEATS,$warm" >> "$OUT_CSV"
+  echo >> "$OUT_RAW"
+done
+
+echo "Done."
+echo "- CSV: $OUT_CSV"
+echo "- Raw: $OUT_RAW"

scripts/manifest_perf_profile_m3.sh

@@ -0,0 +1,142 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# M3: Generate flamegraphs + top hotspots for Manifest decode+profile (Ours vs Routinator).
+#
+# Outputs under:
+#   specs/develop/20260224/flamegraph/
+#   specs/develop/20260224/hotspots/
+#   specs/develop/20260224/perf/
+#
+# Notes:
+# - On WSL2, /usr/bin/perf is often a wrapper that fails. This script uses a real perf binary
+#   from /usr/lib/linux-tools/*/perf (if present).
+# - Ours profiling uses perf + flamegraph --perfdata to avoid rebuilding the whole crate graph
+#   with RocksDB.
+
+ROOT_REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RPKI_DIR="$ROOT_REPO/rpki"
+
+DATE_TAG="${DATE_TAG:-20260224}"
+OUT_BASE="$ROOT_REPO/specs/develop/${DATE_TAG}"
+OUT_FLAME="$OUT_BASE/flamegraph"
+OUT_HOT="$OUT_BASE/hotspots"
+OUT_PERF="$OUT_BASE/perf"
+
+RUN_TAG="${RUN_TAG:-p2}"
+
+OURS_BENCH_DIR="$RPKI_DIR/benchmark/ours_manifest_bench"
+OURS_BIN="$OURS_BENCH_DIR/target/release/ours-manifest-bench"
+
+ROUT_BENCH_DIR="${ROUT_BENCH_DIR:-/home/yuyr/dev/rust_playground/routinator/benchmark}"
+ROUT_BIN="$ROUT_BENCH_DIR/target/release/routinator-manifest-benchmark"
+ROUT_ISSUER="$ROUT_BENCH_DIR/fixtures/ta.cer"
+
+PROFILE_HZ="${PROFILE_HZ:-99}"
+
+mkdir -p "$OUT_FLAME" "$OUT_HOT" "$OUT_PERF"
+
+PERF_WRAPPER_OUT="$(perf --version 2>&1 || true)"
+PERF_REAL=""
+if echo "${PERF_WRAPPER_OUT}" | grep -q "WARNING: perf not found for kernel"; then
+  PERF_REAL="$(ls -1 /usr/lib/linux-tools/*/perf 2>/dev/null | head -n 1 || true)"
+else
+  PERF_REAL="$(command -v perf || true)"
+fi
+
+if [[ -z "${PERF_REAL}" ]]; then
+  echo "ERROR: usable perf binary not found (wrapper detected and no /usr/lib/linux-tools/*/perf)." >&2
+  exit 2
+fi
+
+SHIM_DIR="$RPKI_DIR/target/bench/tools"
+mkdir -p "$SHIM_DIR"
+cat > "$SHIM_DIR/perf" <<EOF
+#!/usr/bin/env bash
+exec "${PERF_REAL}" "\$@"
+EOF
+chmod +x "$SHIM_DIR/perf"
+export PATH="$SHIM_DIR:$PATH"
+
+echo "Using perf: $PERF_REAL"
+
+echo "[1/3] Build ours benchmark with frame pointers..."
+(cd "$OURS_BENCH_DIR" && RUSTFLAGS="-C force-frame-pointers=yes" cargo build --release -q)
+
+echo "[2/3] Build routinator benchmark (release)..."
+(cd "$ROUT_BENCH_DIR" && cargo build --release -q)
+
+taskset_prefix=""
+if command -v taskset >/dev/null 2>&1; then
+  taskset_prefix="taskset -c 0"
+fi
+
+profile_ours() {
+  local sample="$1"
+  local iters="$2"
+  local warm="$3"
+  local fixture="$RPKI_DIR/tests/benchmark/selected_der/${sample}.mft"
+  if [[ ! -f "$fixture" ]]; then
+    echo "ERROR: ours fixture not found: $fixture" >&2
+    exit 1
+  fi
+
+  local perfdata="$OUT_PERF/ours_${sample}_${RUN_TAG}.perf.data"
+  local svg="$OUT_FLAME/ours_${sample}_${RUN_TAG}.svg"
+  local tsv="$OUT_HOT/ours_${sample}_${RUN_TAG}.tsv"
+
+  echo "== ours $sample (iters=$iters warmup=$warm hz=$PROFILE_HZ)"
+  $taskset_prefix perf record -o "$perfdata" -F "$PROFILE_HZ" -g -- \
+    "$OURS_BIN" --manifest "$fixture" --iterations "$iters" --warmup-iterations "$warm" --repeats 1 >/dev/null
+
+  flamegraph --perfdata "$perfdata" --output "$svg" --title "ours ${sample} ManifestObject::decode_der" --deterministic >/dev/null
+
+  perf report -i "$perfdata" --stdio --no-children --sort symbol --percent-limit 0.5 \
+    | awk '/^[[:space:]]*[0-9.]+%/ {pct=$1; sub(/%/,"",pct); $1=""; sub(/^[[:space:]]+/,""); print pct "\t" $0}' \
+    > "$tsv"
+}
+
+profile_routinator() {
+  local sample="$1"
+  local iters="$2"
+  local warm="$3"
+  local fixture="$ROUT_BENCH_DIR/fixtures/selected_der/${sample}.mft"
+  if [[ ! -f "$fixture" ]]; then
+    echo "ERROR: routinator fixture not found: $fixture" >&2
+    exit 1
+  fi
+
+  local svg="$OUT_FLAME/routinator_${sample}_${RUN_TAG}.svg"
+  local tsv="$OUT_HOT/routinator_${sample}_${RUN_TAG}.tsv"
+
+  echo "== routinator $sample (iters=$iters warmup=$warm hz=$PROFILE_HZ)"
+  $taskset_prefix "$ROUT_BIN" \
+    --target decode_only \
+    --manifest "$fixture" \
+    --issuer "$ROUT_ISSUER" \
+    --iterations "$iters" \
+    --repeats 1 \
+    --warmup-iterations "$warm" \
+    --strict false \
+    --profile-hz "$PROFILE_HZ" \
+    --flamegraph "$svg" \
+    --hotspots "$tsv" \
+    >/dev/null
+}
+
+echo "[3/3] Profile samples..."
+
+# Choose iterations so each capture runs ~10-20s serially.
+profile_ours small-01 200000 0
+profile_routinator small-01 200000 0
+
+profile_ours large-02 50000 0
+profile_routinator large-02 50000 0
+
+profile_ours xlarge-02 5000 0
+profile_routinator xlarge-02 5000 0
+
+echo "Done."
+echo "- Flamegraphs: $OUT_FLAME/"
+echo "- Hotspots:    $OUT_HOT/"
+echo "- Perf data:   $OUT_PERF/"

scripts/manual_sync/README.md

@@ -0,0 +1,97 @@
+# Manual RRDP sync (APNIC-focused)
+
+This directory contains **manual, command-line** scripts to reproduce the workflow described in:
+
+- `specs/develop/20260226/apnic_rrdp_delta_analysis_after_manifest_revalidation_fix_20260227T022606Z.md`
+
+They are meant for **hands-on validation / acceptance runs**, not for CI.
+
+## Prerequisites
+
+- Rust toolchain (`cargo`)
+- `rsync` available on PATH (for rsync fallback/objects)
+- Network access (RRDP over HTTPS)
+
+## What the scripts do
+
+### `full_sync.sh`
+
+- Creates a fresh RocksDB directory
+- Runs a **full serial** validation from a TAL URL (default: APNIC RFC7730 TAL)
+- Writes:
+  - run log
+  - audit report JSON
+  - run meta JSON (includes durations + download_stats)
+  - short summary Markdown (includes durations + download_stats)
+  - RocksDB key statistics (`db_stats --exact`)
+  - RRDP legacy session/serial dump (`rrdp_state_dump --view legacy-state`)
+
+### `delta_sync.sh`
+
+- Copies an existing “baseline snapshot DB” to a new DB directory (so the baseline is not modified)
+- Runs another validation against the copied DB (RRDP will prefer **delta** when available)
+- Produces the same artifacts as `full_sync.sh`
+- Additionally generates a Markdown **delta analysis** report by comparing:
+  - base vs delta report JSON
+  - base vs delta `rrdp_state_dump --view legacy-state` TSV
+  - and includes a **duration comparison** (base vs delta) if the base meta JSON is available
+  - delta meta JSON includes download_stats copied from delta report JSON
+
+## Audit report fields (report.json)
+
+The `rpki` binary writes an audit report JSON with:
+
+- `format_version: 2`
+- `downloads`: per-download RRDP/rsync events (URI, timestamps, duration, ok/fail, error, bytes, objects stats)
+- `download_stats`: aggregate counters (by kind)
+
+These are useful for diagnosing why a run is slow (e.g. RRDP snapshot vs delta vs rsync fallback).
+
+The standalone `rrdp_state_dump` tool also supports `source`, `members`, `owners`, and `all` views.
+The manual sync scripts intentionally call `--view legacy-state` so delta analysis keeps using a stable session/serial TSV format.
+
+## Meta fields (meta.json)
+
+The scripts generate `*_meta.json` next to `*_report.json` and include:
+
+- `durations_secs`: wall-clock duration breakdown for the script steps
+- `download_stats`: copied from `report_json.download_stats`
+
+## Usage
+
+Run from `rpki/`:
+
+```bash
+./scripts/manual_sync/full_sync.sh
+```
+
+After you have a baseline run, run delta against it:
+
+```bash
+./scripts/manual_sync/delta_sync.sh target/live/manual_sync/apnic_full_db_YYYYMMDDTHHMMSSZ \
+  target/live/manual_sync/apnic_full_report_YYYYMMDDTHHMMSSZ.json
+```
+
+If the baseline was produced by `full_sync.sh`, the delta script will auto-discover the base meta JSON
+next to the base report (by replacing `_report.json` with `_meta.json`) and include base durations in
+the delta analysis report.
+
+## Configuration (env vars)
+
+Both scripts accept overrides via env vars:
+
+- `TAL_URL` (default: APNIC TAL URL)
+- `HTTP_TIMEOUT_SECS` (default: 1800)
+- `RSYNC_TIMEOUT_SECS` (default: 1800)
+- `RSYNC_MIRROR_ROOT` (default: disabled; when set, passes `--rsync-mirror-root` to `rpki`)
+- `VALIDATION_TIME` (RFC3339; default: now UTC)
+- `OUT_DIR` (default: `rpki/target/live/manual_sync`)
+- `RUN_NAME` (default: auto timestamped)
+
+Example:
+
+```bash
+TAL_URL="https://tal.apnic.net/tal-archive/apnic-rfc7730-https.tal" \
+HTTP_TIMEOUT_SECS=1800 RSYNC_TIMEOUT_SECS=1800 \
+./scripts/manual_sync/full_sync.sh
+```

scripts/manual_sync/delta_sync.sh

@@ -0,0 +1,542 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Delta sync + validation starting from a baseline snapshot DB.
+#
+# This script:
+#  1) Copies BASE_DB_DIR -> DELTA_DB_DIR (so baseline is not modified)
+#  2) Runs rpki validation again (RRDP will prefer delta if available)
+#  3) Writes artifacts + a markdown delta analysis report
+#
+# Usage:
+#   ./scripts/manual_sync/delta_sync.sh <base_db_dir> <base_report_json>
+#
+# Outputs under OUT_DIR (default: target/live/manual_sync):
+# - *_delta_db_*               copied RocksDB directory
+# - *_delta_report_*.json      audit report
+# - *_delta_run_*.log          stdout/stderr log (includes summary)
+# - *_delta_db_stats_*.txt     db_stats --exact output
+# - *_delta_rrdp_state_*.tsv   rrdp_state_dump --view legacy-state output
+# - *_delta_analysis_*.md      base vs delta comparison report
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+BASE_DB_DIR="${1:-}"
+BASE_REPORT_JSON="${2:-}"
+if [[ -z "${BASE_DB_DIR}" || -z "${BASE_REPORT_JSON}" ]]; then
+  echo "Usage: $0 <base_db_dir> <base_report_json>" >&2
+  exit 2
+fi
+if [[ ! -d "${BASE_DB_DIR}" ]]; then
+  echo "ERROR: base_db_dir is not a directory: ${BASE_DB_DIR}" >&2
+  exit 2
+fi
+if [[ ! -f "${BASE_REPORT_JSON}" ]]; then
+  echo "ERROR: base_report_json does not exist: ${BASE_REPORT_JSON}" >&2
+  exit 2
+fi
+
+TAL_URL="${TAL_URL:-https://tal.apnic.net/tal-archive/apnic-rfc7730-https.tal}"
+HTTP_TIMEOUT_SECS="${HTTP_TIMEOUT_SECS:-1800}"
+RSYNC_TIMEOUT_SECS="${RSYNC_TIMEOUT_SECS:-1800}"
+RSYNC_MIRROR_ROOT="${RSYNC_MIRROR_ROOT:-}"
+VALIDATION_TIME="${VALIDATION_TIME:-}"
+
+OUT_DIR="${OUT_DIR:-$ROOT_DIR/target/live/manual_sync}"
+mkdir -p "$OUT_DIR"
+
+TS="$(date -u +%Y%m%dT%H%M%SZ)"
+RUN_NAME="${RUN_NAME:-apnic_delta_${TS}}"
+
+DELTA_DB_DIR="${DELTA_DB_DIR:-$OUT_DIR/${RUN_NAME}_db}"
+DELTA_REPORT_JSON="${DELTA_REPORT_JSON:-$OUT_DIR/${RUN_NAME}_report.json}"
+DELTA_RUN_LOG="${DELTA_RUN_LOG:-$OUT_DIR/${RUN_NAME}_run.log}"
+
+BASE_DB_STATS_TXT="${BASE_DB_STATS_TXT:-$OUT_DIR/${RUN_NAME}_base_db_stats.txt}"
+DELTA_DB_STATS_TXT="${DELTA_DB_STATS_TXT:-$OUT_DIR/${RUN_NAME}_delta_db_stats.txt}"
+
+BASE_RRDP_STATE_TSV="${BASE_RRDP_STATE_TSV:-$OUT_DIR/${RUN_NAME}_base_rrdp_state.tsv}"
+DELTA_RRDP_STATE_TSV="${DELTA_RRDP_STATE_TSV:-$OUT_DIR/${RUN_NAME}_delta_rrdp_state.tsv}"
+
+DELTA_ANALYSIS_MD="${DELTA_ANALYSIS_MD:-$OUT_DIR/${RUN_NAME}_delta_analysis.md}"
+DELTA_META_JSON="${DELTA_META_JSON:-$OUT_DIR/${RUN_NAME}_meta.json}"
+
+# Best-effort base meta discovery (produced by `full_sync.sh`).
+BASE_META_JSON="${BASE_META_JSON:-}"
+if [[ -z "${BASE_META_JSON}" ]]; then
+  guess="${BASE_REPORT_JSON%_report.json}_meta.json"
+  if [[ -f "${guess}" ]]; then
+    BASE_META_JSON="${guess}"
+  fi
+fi
+
+echo "== rpki manual delta sync ==" >&2
+echo "tal_url=$TAL_URL" >&2
+echo "base_db=$BASE_DB_DIR" >&2
+echo "base_report=$BASE_REPORT_JSON" >&2
+echo "delta_db=$DELTA_DB_DIR" >&2
+echo "delta_report=$DELTA_REPORT_JSON" >&2
+
+echo "== copying base DB (baseline is not modified) ==" >&2
+cp -a "$BASE_DB_DIR" "$DELTA_DB_DIR"
+
+script_start_s="$(date +%s)"
+run_start_s="$(date +%s)"
+cmd=(cargo run --release --bin rpki -- \
+  --db "$DELTA_DB_DIR" \
+  --tal-url "$TAL_URL" \
+  --http-timeout-secs "$HTTP_TIMEOUT_SECS" \
+  --rsync-timeout-secs "$RSYNC_TIMEOUT_SECS" \
+  --report-json "$DELTA_REPORT_JSON")
+if [[ -n "${RSYNC_MIRROR_ROOT}" ]]; then
+  cmd+=(--rsync-mirror-root "$RSYNC_MIRROR_ROOT")
+fi
+if [[ -n "${VALIDATION_TIME}" ]]; then
+  cmd+=(--validation-time "$VALIDATION_TIME")
+fi
+
+(
+  echo "# command:"
+  printf '%q ' "${cmd[@]}"
+  echo
+  echo
+  "${cmd[@]}"
+) 2>&1 | tee "$DELTA_RUN_LOG" >/dev/null
+run_end_s="$(date +%s)"
+run_duration_s="$((run_end_s - run_start_s))"
+
+echo "== db_stats (exact) ==" >&2
+db_stats_start_s="$(date +%s)"
+cargo run --release --bin db_stats -- --db "$BASE_DB_DIR" --exact 2>&1 | tee "$BASE_DB_STATS_TXT" >/dev/null
+cargo run --release --bin db_stats -- --db "$DELTA_DB_DIR" --exact 2>&1 | tee "$DELTA_DB_STATS_TXT" >/dev/null
+db_stats_end_s="$(date +%s)"
+db_stats_duration_s="$((db_stats_end_s - db_stats_start_s))"
+
+echo "== rrdp_state_dump (legacy-state) ==" >&2
+state_start_s="$(date +%s)"
+cargo run --release --bin rrdp_state_dump -- --db "$BASE_DB_DIR" --view legacy-state >"$BASE_RRDP_STATE_TSV"
+cargo run --release --bin rrdp_state_dump -- --db "$DELTA_DB_DIR" --view legacy-state >"$DELTA_RRDP_STATE_TSV"
+state_end_s="$(date +%s)"
+state_duration_s="$((state_end_s - state_start_s))"
+
+script_end_s="$(date +%s)"
+total_duration_s="$((script_end_s - script_start_s))"
+
+echo "== delta analysis report ==" >&2
+TAL_URL="$TAL_URL" \
+BASE_DB_DIR="$BASE_DB_DIR" \
+DELTA_DB_DIR="$DELTA_DB_DIR" \
+DELTA_RUN_LOG="$DELTA_RUN_LOG" \
+VALIDATION_TIME_ARG="$VALIDATION_TIME" \
+HTTP_TIMEOUT_SECS="$HTTP_TIMEOUT_SECS" \
+RSYNC_TIMEOUT_SECS="$RSYNC_TIMEOUT_SECS" \
+RUN_DURATION_S="$run_duration_s" \
+DB_STATS_DURATION_S="$db_stats_duration_s" \
+STATE_DURATION_S="$state_duration_s" \
+TOTAL_DURATION_S="$total_duration_s" \
+  python3 - "$BASE_REPORT_JSON" "$DELTA_REPORT_JSON" "$BASE_RRDP_STATE_TSV" "$DELTA_RRDP_STATE_TSV" \
+  "$BASE_DB_STATS_TXT" "$DELTA_DB_STATS_TXT" "$BASE_META_JSON" "$DELTA_META_JSON" "$DELTA_ANALYSIS_MD" <<'PY'
+import json
+import os
+import sys
+from collections import Counter, defaultdict
+from datetime import datetime, timezone
+from pathlib import Path
+
+base_report_path = Path(sys.argv[1])
+delta_report_path = Path(sys.argv[2])
+base_state_path = Path(sys.argv[3])
+delta_state_path = Path(sys.argv[4])
+base_db_stats_path = Path(sys.argv[5])
+delta_db_stats_path = Path(sys.argv[6])
+base_meta_path_s = sys.argv[7]
+delta_meta_path = Path(sys.argv[8])
+out_md_path = Path(sys.argv[9])
+
+def load_json(p: Path):
+    s = p.read_text(encoding="utf-8")
+    try:
+        return json.loads(s)
+    except json.JSONDecodeError:
+        # Backwards-compat / robustness: tolerate accidental literal trailing "\\n".
+        s2 = s.strip()
+        if s2.endswith("\\n"):
+            s2 = s2[:-2].rstrip()
+        return json.loads(s2)
+
+def load_optional_json(path_s: str):
+    if not path_s:
+        return None
+    p = Path(path_s)
+    if not p.exists():
+        return None
+    return load_json(p)
+
+def parse_rrdp_state_tsv(p: Path):
+    # format from `rrdp_state_dump --view legacy-state`:
+    #   [legacy-state]
+    #   notify_uri	serial	session_id
+    #   <notify_uri>	<serial>	<session_id>
+    out = {}
+    for line in p.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("["):
+            continue
+        if line == "notify_uri	serial	session_id":
+            continue
+        parts = line.split("	")
+        if len(parts) != 3:
+            raise SystemExit(f"invalid rrdp_state_dump line in {p}: {line!r}")
+        uri, serial, session = parts
+        out[uri] = (int(serial), session)
+    return out
+
+def parse_db_stats(p: Path):
+    # lines: key=value
+    out = {}
+    for line in p.read_text(encoding="utf-8").splitlines():
+        if "=" not in line:
+            continue
+        k, v = line.split("=", 1)
+        k = k.strip()
+        v = v.strip()
+        if v.isdigit():
+            out[k] = int(v)
+        else:
+            out[k] = v
+    return out
+
+def warnings_total(rep: dict) -> int:
+    return len(rep["tree"]["warnings"]) + sum(len(pp["warnings"]) for pp in rep["publication_points"])
+
+def report_summary(rep: dict) -> dict:
+    return {
+        "validation_time": rep["meta"]["validation_time_rfc3339_utc"],
+        "publication_points_processed": rep["tree"]["instances_processed"],
+        "publication_points_failed": rep["tree"]["instances_failed"],
+        "rrdp_repos_unique": len({pp.get("rrdp_notification_uri") for pp in rep["publication_points"] if pp.get("rrdp_notification_uri")}),
+        "vrps": len(rep["vrps"]),
+        "aspas": len(rep["aspas"]),
+        "audit_publication_points": len(rep["publication_points"]),
+        "warnings_total": warnings_total(rep),
+    }
+
+def count_repo_sync_failed(rep: dict) -> int:
+    # Best-effort heuristic (we don't currently expose a structured counter in the audit report).
+    # Keep the match conservative to avoid false positives.
+    def is_repo_sync_failed(msg: str) -> bool:
+        m = msg.lower()
+        return "repo sync failed" in m or "rrdp fetch failed" in m or "rsync fetch failed" in m
+
+    n = 0
+    for w in rep["tree"]["warnings"]:
+        if is_repo_sync_failed(w.get("message", "")):
+            n += 1
+    for pp in rep["publication_points"]:
+        for w in pp.get("warnings", []):
+            if is_repo_sync_failed(w.get("message", "")):
+                n += 1
+    return n
+
+def pp_manifest_sha(pp: dict) -> str:
+    # In our audit format, the first object is the manifest (synthetic entry) with sha256 of manifest_bytes.
+    for o in pp["objects"]:
+        if o["kind"] == "manifest":
+            return o["sha256_hex"]
+    return ""
+
+def pp_objects_by_uri(rep: dict):
+    m = {}
+    for pp in rep["publication_points"]:
+        for o in pp["objects"]:
+            m[o["rsync_uri"]] = (o["sha256_hex"], o["kind"])
+    return m
+
+def vrp_set(rep: dict):
+    return {(v["asn"], v["prefix"], v["max_length"]) for v in rep["vrps"]}
+
+def rfc_refs_str(w: dict) -> str:
+    refs = w.get("rfc_refs") or []
+    return ", ".join(refs) if refs else ""
+
+base = load_json(base_report_path)
+delta = load_json(delta_report_path)
+base_sum = report_summary(base)
+delta_sum = report_summary(delta)
+
+base_db = parse_db_stats(base_db_stats_path)
+delta_db = parse_db_stats(delta_db_stats_path)
+
+base_state = parse_rrdp_state_tsv(base_state_path)
+delta_state = parse_rrdp_state_tsv(delta_state_path)
+
+base_meta = load_optional_json(base_meta_path_s)
+download_stats = delta.get("download_stats") or {}
+delta_meta = {
+  "recorded_at_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+  "tal_url": os.environ["TAL_URL"],
+  "base_db_dir": os.environ["BASE_DB_DIR"],
+  "delta_db_dir": os.environ["DELTA_DB_DIR"],
+  "base_report_json": str(base_report_path),
+  "delta_report_json": str(delta_report_path),
+  "delta_run_log": os.environ["DELTA_RUN_LOG"],
+  "validation_time_arg": os.environ.get("VALIDATION_TIME_ARG",""),
+  "http_timeout_secs": int(os.environ["HTTP_TIMEOUT_SECS"]),
+  "rsync_timeout_secs": int(os.environ["RSYNC_TIMEOUT_SECS"]),
+  "durations_secs": {
+    "rpki_run": int(os.environ["RUN_DURATION_S"]),
+    "db_stats_exact": int(os.environ["DB_STATS_DURATION_S"]),
+    "rrdp_state_dump": int(os.environ["STATE_DURATION_S"]),
+    "total_script": int(os.environ["TOTAL_DURATION_S"]),
+  },
+  "download_stats": download_stats,
+}
+delta_meta_path.write_text(json.dumps(delta_meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
+now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+# RRDP state changes
+serial_changed = 0
+session_changed = 0
+serial_deltas = []
+for uri, (old_serial, old_sess) in base_state.items():
+    if uri not in delta_state:
+        continue
+    new_serial, new_sess = delta_state[uri]
+    if new_serial != old_serial:
+        serial_changed += 1
+        serial_deltas.append((uri, old_serial, new_serial, new_serial - old_serial))
+    if new_sess != old_sess:
+        session_changed += 1
+
+serial_deltas.sort(key=lambda x: x[3], reverse=True)
+
+# Publication point diffs
+base_pp = {pp["manifest_rsync_uri"]: pp for pp in base["publication_points"]}
+delta_pp = {pp["manifest_rsync_uri"]: pp for pp in delta["publication_points"]}
+
+base_keys = set(base_pp.keys())
+delta_keys = set(delta_pp.keys())
+
+new_pp = sorted(delta_keys - base_keys)
+missing_pp = sorted(base_keys - delta_keys)
+
+updated_pp = 0
+unchanged_pp = 0
+for k in sorted(base_keys & delta_keys):
+    if pp_manifest_sha(base_pp[k]) != pp_manifest_sha(delta_pp[k]):
+        updated_pp += 1
+    else:
+        unchanged_pp += 1
+
+# Cache usage + repo sync failure hints
+def source_counts(rep: dict) -> Counter:
+    c = Counter()
+    for pp in rep["publication_points"]:
+        c[pp.get("source","")] += 1
+    return c
+
+base_sources = source_counts(base)
+delta_sources = source_counts(delta)
+base_repo_sync_failed = count_repo_sync_failed(base)
+delta_repo_sync_failed = count_repo_sync_failed(delta)
+
+def cache_reason_counts(rep: dict) -> Counter:
+    c = Counter()
+    for pp in rep.get("publication_points", []):
+        if pp.get("source") != "vcir_current_instance":
+            continue
+        # Use warning messages as "reason". If missing, emit a fallback bucket.
+        ws = pp.get("warnings", [])
+        if not ws:
+            c["(no warnings recorded)"] += 1
+            continue
+        for w in ws:
+            msg = w.get("message", "").strip() or "(empty warning message)"
+            c[msg] += 1
+    return c
+
+base_cache_reasons = cache_reason_counts(base)
+delta_cache_reasons = cache_reason_counts(delta)
+
+# Object change stats (by rsync URI, sha256)
+base_obj = pp_objects_by_uri(base)
+delta_obj = pp_objects_by_uri(delta)
+
+kind_stats = {k: {"added": 0, "changed": 0, "removed": 0} for k in ["manifest","crl","certificate","roa","aspa","other"]}
+all_uris = set(base_obj.keys()) | set(delta_obj.keys())
+for uri in all_uris:
+    b = base_obj.get(uri)
+    d = delta_obj.get(uri)
+    if b is None and d is not None:
+        kind_stats[d[1]]["added"] += 1
+    elif b is not None and d is None:
+        kind_stats[b[1]]["removed"] += 1
+    else:
+        if b[0] != d[0]:
+            kind_stats[d[1]]["changed"] += 1
+
+# VRP diff
+base_v = vrp_set(base)
+delta_v = vrp_set(delta)
+added_v = delta_v - base_v
+removed_v = base_v - delta_v
+
+def fmt_db_stats(db: dict) -> str:
+    ordered = [
+        "mode",
+        "repository_view",
+        "raw_by_hash",
+        "vcir",
+        "rrdp_source",
+        "rrdp_source_member",
+        "rrdp_uri_owner",
+        "rrdp_state",
+        "raw_objects",
+        "rrdp_object_index",
+        "group_current_repository_view",
+        "group_current_validation_state",
+        "group_current_rrdp_state",
+        "group_legacy_compatibility",
+        "total",
+        "sst_files",
+    ]
+    out = []
+    seen = set()
+    for k in ordered:
+        if k in db:
+            out.append(f"- `{k}={db[k]}`")
+            seen.add(k)
+    for k in sorted(set(db) - seen):
+        out.append(f"- `{k}={db[k]}`")
+    return "\n".join(out) if out else "_(missing db_stats keys)_"
+
+lines = []
+lines.append("# APNIC RRDP 增量同步验收（manual_sync）\n\n")
+lines.append(f"时间戳：`{now}`（UTC）\n\n")
+
+lines.append("## 复现信息\n\n")
+lines.append(f"- base_report：`{base_report_path}`\n")
+lines.append(f"- delta_report：`{delta_report_path}`\n")
+lines.append(f"- base_db_stats：`{base_db_stats_path}`\n")
+lines.append(f"- delta_db_stats：`{delta_db_stats_path}`\n")
+lines.append(f"- base_rrdp_state：`{base_state_path}`\n")
+lines.append(f"- delta_rrdp_state：`{delta_state_path}`\n\n")
+
+lines.append("## 运行结果概览\n\n")
+lines.append("| metric | base | delta |\n")
+lines.append("|---|---:|---:|\n")
+for k in [
+    "validation_time",
+    "publication_points_processed",
+    "publication_points_failed",
+    "rrdp_repos_unique",
+    "vrps",
+    "aspas",
+    "audit_publication_points",
+    "warnings_total",
+]:
+    lines.append(f"| {k} | {base_sum[k]} | {delta_sum[k]} |\n")
+lines.append("\n")
+
+def dur(meta: dict | None, key: str):
+    if not meta:
+        return None
+    return (meta.get("durations_secs") or {}).get(key)
+
+base_rpki_run = dur(base_meta, "rpki_run")
+delta_rpki_run = delta_meta["durations_secs"]["rpki_run"]
+base_total = dur(base_meta, "total_script")
+delta_total = delta_meta["durations_secs"]["total_script"]
+
+lines.append("## 持续时间（seconds）\n\n")
+lines.append("| step | base | delta |\n")
+lines.append("|---|---:|---:|\n")
+lines.append(f"| rpki_run | {base_rpki_run if base_rpki_run is not None else 'unknown'} | {delta_rpki_run} |\n")
+lines.append(f"| total_script | {base_total if base_total is not None else 'unknown'} | {delta_total} |\n")
+lines.append("\n")
+if base_meta is None and base_meta_path_s:
+    lines.append(f"> 注：未能读取 base meta：`{base_meta_path_s}`（文件不存在或不可读）。建议用 `full_sync.sh` 生成 baseline 以获得 base 时长对比。\n\n")
+
+lines.append("RocksDB KV（`db_stats --exact`）：\n\n")
+lines.append("### 基线（base）\n\n")
+lines.append(fmt_db_stats(base_db) + "\n\n")
+lines.append("### 增量（delta）\n\n")
+lines.append(fmt_db_stats(delta_db) + "\n\n")
+
+lines.append("## RRDP 增量是否发生（基于 `rrdp_state` 变化）\n\n")
+lines.append(f"- repo_total(base)={len(base_state)}\n")
+lines.append(f"- repo_total(delta)={len(delta_state)}\n")
+lines.append(f"- serial_changed={serial_changed}\n")
+lines.append(f"- session_changed={session_changed}\n\n")
+if serial_deltas:
+    lines.append("serial 增长最大的 10 个 RRDP repo（old -> new）：\n\n")
+    for uri, old, new, diff in serial_deltas[:10]:
+        lines.append(f"- `{uri}`：`{old} -> {new}`（+{diff}）\n")
+    lines.append("\n")
+
+lines.append("## 发布点（Publication Point）变化统计\n\n")
+lines.append("以 `manifest_rsync_uri` 作为发布点 key，对比 base vs delta：\n\n")
+lines.append(f"- base PP：`{len(base_keys)}`\n")
+lines.append(f"- delta PP：`{len(delta_keys)}`\n")
+lines.append(f"- `new_pp={len(new_pp)}`\n")
+lines.append(f"- `missing_pp={len(missing_pp)}`\n")
+lines.append(f"- `updated_pp={updated_pp}`\n")
+lines.append(f"- `unchanged_pp={unchanged_pp}`\n\n")
+lines.append("> 注：`new_pp/missing_pp/updated_pp` 会混入“遍历范围变化”的影响（例如 validation_time 不同、或 base 中存在更多失败 PP）。\n\n")
+
+lines.append("## fail fetch / VCIR 当前实例缓存复用情况\n\n")
+lines.append(f"- repo sync failed（启发式：warning contains 'repo sync failed'/'rrdp fetch failed'/'rsync fetch failed'）\n")
+lines.append(f"  - base：`{base_repo_sync_failed}`\n")
+lines.append(f"  - delta：`{delta_repo_sync_failed}`\n\n")
+
+lines.append("- source 计数（按 `PublicationPointAudit.source`）：\n\n")
+lines.append(f"  - base：`{dict(base_sources)}`\n")
+lines.append(f"  - delta：`{dict(delta_sources)}`\n\n")
+
+def render_cache_reasons(title: str, c: Counter) -> str:
+    if not c:
+        return f"{title}：`0`（未使用 VCIR 当前实例缓存复用）\n\n"
+    lines = []
+    total = sum(c.values())
+    lines.append(f"{title}：`{total}`\n\n")
+    lines.append("Top reasons（按 warning message 聚合，可能一条 PP 有多条 warning）：\n\n")
+    for msg, n in c.most_common(10):
+        lines.append(f"- `{n}` × {msg}\n")
+    lines.append("\n")
+    return "".join(lines)
+
+lines.append(render_cache_reasons("- base `source=vcir_current_instance`", base_cache_reasons))
+lines.append(render_cache_reasons("- delta `source=vcir_current_instance`", delta_cache_reasons))
+
+lines.append("## 文件变更统计（按对象类型）\n\n")
+lines.append("按 `ObjectAuditEntry.sha256_hex` 对比（同一 rsync URI 前后 hash 变化记为 `~changed`）：\n\n")
+lines.append("| kind | added | changed | removed |\n")
+lines.append("|---|---:|---:|---:|\n")
+for kind in ["manifest","crl","certificate","roa","aspa","other"]:
+    st = kind_stats[kind]
+    lines.append(f"| {kind} | {st['added']} | {st['changed']} | {st['removed']} |\n")
+lines.append("\n")
+
+lines.append("## VRP 影响（去重后集合 diff）\n\n")
+lines.append("以 `(asn, prefix, max_length)` 为 key：\n\n")
+lines.append(f"- base unique VRP：`{len(base_v)}`\n")
+lines.append(f"- delta unique VRP：`{len(delta_v)}`\n")
+lines.append(f"- `added={len(added_v)}`\n")
+lines.append(f"- `removed={len(removed_v)}`\n")
+lines.append(f"- `net={len(added_v) - len(removed_v)}`\n\n")
+
+out_md_path.write_text("".join(lines), encoding="utf-8")
+print(out_md_path)
+PY
+
+echo "== done ==" >&2
+echo "artifacts:" >&2
+echo "- delta db:        $DELTA_DB_DIR" >&2
+echo "- delta report:    $DELTA_REPORT_JSON" >&2
+echo "- delta run log:   $DELTA_RUN_LOG" >&2
+echo "- delta meta json: $DELTA_META_JSON" >&2
+echo "- analysis md:     $DELTA_ANALYSIS_MD" >&2
+echo "- base state tsv:  $BASE_RRDP_STATE_TSV" >&2
+echo "- delta state tsv: $DELTA_RRDP_STATE_TSV" >&2

scripts/manual_sync/full_sync.sh

@@ -0,0 +1,189 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Full sync + validation from a TAL URL (default: APNIC).
+#
+# Produces artifacts under OUT_DIR (default: target/live/manual_sync):
+# - *_db_*               RocksDB directory
+# - *_report_*.json      audit report
+# - *_run_*.log          stdout/stderr log (includes summary)
+# - *_db_stats_*.txt     db_stats --exact output
+# - *_rrdp_state_*.tsv   rrdp_state_dump --view legacy-state output
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+TAL_URL="${TAL_URL:-https://tal.apnic.net/tal-archive/apnic-rfc7730-https.tal}"
+HTTP_TIMEOUT_SECS="${HTTP_TIMEOUT_SECS:-1800}"
+RSYNC_TIMEOUT_SECS="${RSYNC_TIMEOUT_SECS:-1800}"
+RSYNC_MIRROR_ROOT="${RSYNC_MIRROR_ROOT:-}"
+VALIDATION_TIME="${VALIDATION_TIME:-}"
+
+OUT_DIR="${OUT_DIR:-$ROOT_DIR/target/live/manual_sync}"
+mkdir -p "$OUT_DIR"
+
+TS="$(date -u +%Y%m%dT%H%M%SZ)"
+RUN_NAME="${RUN_NAME:-apnic_full_${TS}}"
+
+DB_DIR="${DB_DIR:-$OUT_DIR/${RUN_NAME}_db}"
+REPORT_JSON="${REPORT_JSON:-$OUT_DIR/${RUN_NAME}_report.json}"
+RUN_LOG="${RUN_LOG:-$OUT_DIR/${RUN_NAME}_run.log}"
+DB_STATS_TXT="${DB_STATS_TXT:-$OUT_DIR/${RUN_NAME}_db_stats.txt}"
+RRDP_STATE_TSV="${RRDP_STATE_TSV:-$OUT_DIR/${RUN_NAME}_rrdp_state.tsv}"
+RUN_META_JSON="${RUN_META_JSON:-$OUT_DIR/${RUN_NAME}_meta.json}"
+SUMMARY_MD="${SUMMARY_MD:-$OUT_DIR/${RUN_NAME}_summary.md}"
+
+echo "== rpki manual full sync ==" >&2
+echo "tal_url=$TAL_URL" >&2
+echo "db=$DB_DIR" >&2
+echo "report_json=$REPORT_JSON" >&2
+echo "out_dir=$OUT_DIR" >&2
+
+cmd=(cargo run --release --bin rpki -- \
+  --db "$DB_DIR" \
+  --tal-url "$TAL_URL" \
+  --http-timeout-secs "$HTTP_TIMEOUT_SECS" \
+  --rsync-timeout-secs "$RSYNC_TIMEOUT_SECS" \
+  --report-json "$REPORT_JSON")
+
+if [[ -n "${RSYNC_MIRROR_ROOT}" ]]; then
+  cmd+=(--rsync-mirror-root "$RSYNC_MIRROR_ROOT")
+fi
+if [[ -n "${VALIDATION_TIME}" ]]; then
+  cmd+=(--validation-time "$VALIDATION_TIME")
+fi
+
+script_start_s="$(date +%s)"
+run_start_s="$(date +%s)"
+(
+  echo "# command:"
+  printf '%q ' "${cmd[@]}"
+  echo
+  echo
+  "${cmd[@]}"
+) 2>&1 | tee "$RUN_LOG" >/dev/null
+run_end_s="$(date +%s)"
+run_duration_s="$((run_end_s - run_start_s))"
+
+echo "== db_stats (exact) ==" >&2
+db_stats_start_s="$(date +%s)"
+cargo run --release --bin db_stats -- --db "$DB_DIR" --exact 2>&1 | tee "$DB_STATS_TXT" >/dev/null
+db_stats_end_s="$(date +%s)"
+db_stats_duration_s="$((db_stats_end_s - db_stats_start_s))"
+
+echo "== rrdp_state_dump (legacy-state) ==" >&2
+state_start_s="$(date +%s)"
+cargo run --release --bin rrdp_state_dump -- --db "$DB_DIR" --view legacy-state >"$RRDP_STATE_TSV"
+state_end_s="$(date +%s)"
+state_duration_s="$((state_end_s - state_start_s))"
+
+script_end_s="$(date +%s)"
+total_duration_s="$((script_end_s - script_start_s))"
+
+echo "== write run meta + summary ==" >&2
+TAL_URL="$TAL_URL" \
+DB_DIR="$DB_DIR" \
+REPORT_JSON="$REPORT_JSON" \
+RUN_LOG="$RUN_LOG" \
+HTTP_TIMEOUT_SECS="$HTTP_TIMEOUT_SECS" \
+RSYNC_TIMEOUT_SECS="$RSYNC_TIMEOUT_SECS" \
+VALIDATION_TIME_ARG="$VALIDATION_TIME" \
+RUN_DURATION_S="$run_duration_s" \
+DB_STATS_DURATION_S="$db_stats_duration_s" \
+STATE_DURATION_S="$state_duration_s" \
+TOTAL_DURATION_S="$total_duration_s" \
+  python3 - "$REPORT_JSON" "$RUN_META_JSON" "$SUMMARY_MD" <<'PY'
+import json
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+report_path = Path(sys.argv[1])
+meta_path = Path(sys.argv[2])
+summary_path = Path(sys.argv[3])
+
+rep = json.loads(report_path.read_text(encoding="utf-8"))
+
+now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+download_stats = rep.get("download_stats") or {}
+meta = {
+  "recorded_at_utc": now,
+  "tal_url": os.environ["TAL_URL"],
+  "db_dir": os.environ["DB_DIR"],
+  "report_json": os.environ["REPORT_JSON"],
+  "run_log": os.environ["RUN_LOG"],
+  "validation_time_rfc3339_utc": rep["meta"]["validation_time_rfc3339_utc"],
+  "http_timeout_secs": int(os.environ["HTTP_TIMEOUT_SECS"]),
+  "rsync_timeout_secs": int(os.environ["RSYNC_TIMEOUT_SECS"]),
+  "validation_time_arg": os.environ.get("VALIDATION_TIME_ARG",""),
+  "durations_secs": {
+    "rpki_run": int(os.environ["RUN_DURATION_S"]),
+    "db_stats_exact": int(os.environ["DB_STATS_DURATION_S"]),
+    "rrdp_state_dump": int(os.environ["STATE_DURATION_S"]),
+    "total_script": int(os.environ["TOTAL_DURATION_S"]),
+  },
+  "counts": {
+    "publication_points_processed": rep["tree"]["instances_processed"],
+    "publication_points_failed": rep["tree"]["instances_failed"],
+    "vrps": len(rep["vrps"]),
+    "aspas": len(rep["aspas"]),
+    "audit_publication_points": len(rep["publication_points"]),
+  },
+  "download_stats": download_stats,
+}
+
+meta_path.write_text(json.dumps(meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
+lines = []
+lines.append("# Manual full sync summary\\n\\n")
+lines.append(f"- recorded_at_utc: `{now}`\\n")
+lines.append(f"- tal_url: `{meta['tal_url']}`\\n")
+lines.append(f"- db: `{meta['db_dir']}`\\n")
+lines.append(f"- report_json: `{meta['report_json']}`\\n")
+lines.append(f"- validation_time: `{meta['validation_time_rfc3339_utc']}`\\n\\n")
+lines.append("## Results\\n\\n")
+lines.append("| metric | value |\\n")
+lines.append("|---|---:|\\n")
+for k in ["publication_points_processed","publication_points_failed","vrps","aspas","audit_publication_points"]:
+  lines.append(f"| {k} | {meta['counts'][k]} |\\n")
+lines.append("\\n")
+lines.append("## Durations (seconds)\\n\\n")
+lines.append("| step | seconds |\\n")
+lines.append("|---|---:|\\n")
+for k,v in meta["durations_secs"].items():
+  lines.append(f"| {k} | {v} |\\n")
+lines.append("\\n")
+
+lines.append("## Download Stats\\n\\n")
+lines.append("- raw events: `report_json.downloads`\\n")
+lines.append("- aggregated: `report_json.download_stats` (copied into meta.json)\\n\\n")
+
+def fmt_u(v):
+  if v is None:
+    return ""
+  return str(v)
+
+by_kind = download_stats.get("by_kind") or {}
+lines.append("| kind | ok | fail | duration_ms_total | bytes_total | objects_count_total | objects_bytes_total |\\n")
+lines.append("|---|---:|---:|---:|---:|---:|---:|\\n")
+for kind in sorted(by_kind.keys()):
+  st = by_kind[kind] or {}
+  lines.append(
+    f"| {kind} | {st.get('ok_total',0)} | {st.get('fail_total',0)} | {st.get('duration_ms_total',0)} | {fmt_u(st.get('bytes_total'))} | {fmt_u(st.get('objects_count_total'))} | {fmt_u(st.get('objects_bytes_total'))} |\\n"
+  )
+lines.append("\\n")
+
+summary_path.write_text("".join(lines), encoding="utf-8")
+print(summary_path)
+PY
+
+echo "== done ==" >&2
+echo "artifacts:" >&2
+echo "- db:         $DB_DIR" >&2
+echo "- report:     $REPORT_JSON" >&2
+echo "- run log:    $RUN_LOG" >&2
+echo "- db stats:   $DB_STATS_TXT" >&2
+echo "- rrdp state: $RRDP_STATE_TSV" >&2
+echo "- meta json:  $RUN_META_JSON" >&2
+echo "- summary md: $SUMMARY_MD" >&2

scripts/payload_replay/README.md

@@ -0,0 +1,268 @@
+# Payload Replay Scripts
+
+本目录提供基于本地 payload archive 的手工 replay 入口。
+
+## `multi_rir_case_info.py`
+
+用于从 multi-RIR bundle 中解析指定 `rir` 的输入路径、对照 CSV、fixture、以及 Routinator replay timing 基线。
+
+示例：
+
+```bash
+python3 scripts/payload_replay/multi_rir_case_info.py \
+  --bundle-root ../../rpki/target/live/20260316-112341-multi-final3 \
+  --rir afrinic
+```
+
+也支持输出 shell 环境变量：
+
+```bash
+python3 scripts/payload_replay/multi_rir_case_info.py \
+  --bundle-root ../../rpki/target/live/20260316-112341-multi-final3 \
+  --rir afrinic \
+  --format env
+```
+
+## `run_multi_rir_replay_case.sh`
+
+统一的 multi-RIR 入口。给定 `rir` 和模式后，它会自动选择该 RIR 的：
+
+- snapshot/base replay 输入
+- delta replay 输入
+- 对照 CSV
+- TAL / TA fixture
+- trust anchor 名称
+
+用法：
+
+```bash
+./scripts/payload_replay/run_multi_rir_replay_case.sh <rir> [describe|snapshot|delta|both]
+```
+
+示例：
+
+```bash
+./scripts/payload_replay/run_multi_rir_replay_case.sh afrinic describe
+./scripts/payload_replay/run_multi_rir_replay_case.sh lacnic snapshot
+./scripts/payload_replay/run_multi_rir_replay_case.sh arin delta
+./scripts/payload_replay/run_multi_rir_replay_case.sh ripe both
+```
+
+
+脚本会自动：
+
+- 从 multi-RIR bundle 中选择指定 RIR 的 snapshot/base 与 delta 输入
+- 读取该 RIR 的 Routinator `base-replay` / `delta-replay` timing 基线
+- 优先使用 `base-locks.json.validationTime` 与 `locks-delta.json.validationTime` 作为 replay `--validation-time`；若缺失才回退到 `timings/base-replay.json` 与 `timings/delta-replay.json` 的 `startedAt`
+- 在 `target/live/multi_rir_replay_runs/<rir>/` 下生成：
+  - snapshot replay 产物
+  - delta replay 产物
+  - per-RIR 合并 case report（含 correctness + timing compare）
+
+默认 bundle 根目录为：
+
+- `../../rpki/target/live/20260316-112341-multi-final3`
+
+也可以通过 `BUNDLE_ROOT` 覆盖。
+
+## `run_apnic_snapshot_replay_profile.sh`
+
+基于 multi-RIR bundle 中的 APNIC snapshot 输入，使用当前 replay 主流程执行一次带 `--analyze` 和 `--profile-cpu` 的离线 profile。
+
+```bash
+./scripts/payload_replay/run_apnic_snapshot_replay_profile.sh
+```
+
+作用：
+
+- 使用 `APNIC` 的 snapshot/base replay 输入
+- 自动开启：
+  - `--analyze`
+  - `--profile-cpu`
+- 自动记录：
+  - replay wall-clock 时长
+  - Routinator baseline (`base-replay`)
+  - analyze 目录路径
+- 生成：
+  - `report.json`
+  - `run.log`
+  - `meta.json`
+  - `summary.md`
+  - 以及 `target/live/analyze/<timestamp>/` 下的：
+    - `timing.json`
+    - `flamegraph.svg`
+    - `pprof.pb.gz`
+
+支持：
+
+- `DRY_RUN=1`：只打印命令，不真正执行
+- `MAX_DEPTH` / `MAX_INSTANCES`：用于限定 replay 范围
+- `PROFILE_RUN_ROOT`：覆盖 wrapper 产物输出目录
+
+## `run_apnic_replay.sh`
+
+默认使用：
+
+- `tests/fixtures/tal/apnic-rfc7730-https.tal`
+- `tests/fixtures/ta/apnic-ta.cer`
+- `target/live/payload_replay/payload-archive`
+- `target/live/payload_replay/locks.json`
+
+运行：
+
+```bash
+./scripts/payload_replay/run_apnic_replay.sh
+```
+
+产物默认输出到：
+
+- `target/live/payload_replay_runs/`
+
+包含：
+
+- replay DB 目录
+- `report.json`
+- `run.log`
+- `meta.json`
+- `summary.md`
+
+## 环境变量
+
+可覆盖：
+
+- `TAL_PATH`
+- `TA_PATH`
+- `PAYLOAD_REPLAY_ARCHIVE`
+- `PAYLOAD_REPLAY_LOCKS`
+- `VALIDATION_TIME`
+- `MAX_DEPTH`
+- `MAX_INSTANCES`
+- `OUT_DIR`
+- `RUN_NAME`
+- `DB_DIR`
+- `REPORT_JSON`
+- `RUN_LOG`
+- `META_JSON`
+- `SUMMARY_MD`
+
+## 说明
+
+- 该脚本依赖 `rpki` CLI 已支持：
+  - `--payload-replay-archive`
+  - `--payload-replay-locks`
+- replay 模式必须搭配离线 TAL/TA 输入，不会去访问真实 RRDP / rsync 网络源。
+
+## `report_to_routinator_csv.py`
+
+把 `rpki` 生成的 `report.json` 转成 Routinator 风格的 VRP CSV：
+
+```bash
+python3 scripts/payload_replay/report_to_routinator_csv.py \
+  --report target/live/payload_replay_runs/<run>_report.json \
+  --out target/live/payload_replay_runs/<run>_vrps.csv \
+  --trust-anchor apnic
+```
+
+输出列为：
+
+- `ASN`
+- `IP Prefix`
+- `Max Length`
+- `Trust Anchor`
+
+## `compare_with_routinator_record.sh`
+
+把 ours 生成的 VRP CSV 与 Routinator 的 `record.csv` 做对比：
+
+```bash
+./scripts/payload_replay/compare_with_routinator_record.sh \
+  target/live/payload_replay_runs/<run>_vrps.csv \
+  target/live/payload_replay/record.csv
+```
+
+会产出：
+
+- compare summary Markdown
+- `only_in_ours.csv`
+- `only_in_record.csv`
+
+## `run_apnic_replay.sh` 现有额外产物
+
+脚本现在除了 `report/meta/summary`，还会额外生成：
+
+- `vrps.csv`
+- 若 `ROUTINATOR_RECORD_CSV` 存在，则生成：
+  - compare summary
+  - `only_in_ours.csv`
+  - `only_in_record.csv`
+
+## `run_apnic_delta_replay.sh`
+
+使用 APNIC delta demo 数据集运行 base + delta replay：
+
+```bash
+./scripts/payload_replay/run_apnic_delta_replay.sh
+```
+
+默认输入：
+
+- `target/live/apnic_delta_demo/20260315-170223-autoplay/base-payload-archive`
+- `target/live/apnic_delta_demo/20260315-170223-autoplay/base-locks.json`
+- `target/live/apnic_delta_demo/20260315-170223-autoplay/payload-delta-archive`
+- `target/live/apnic_delta_demo/20260315-170223-autoplay/locks-delta.json`
+- `tests/fixtures/tal/apnic-rfc7730-https.tal`
+- `tests/fixtures/ta/apnic-ta.cer`
+
+输出目录默认：`target/live/payload_delta_replay_runs/`
+
+## `run_apnic_delta_replay.sh` compare outputs
+
+脚本现在在 delta replay 结束后还会额外生成：
+
+- `vrps.csv`
+- compare summary Markdown
+- `only_in_ours.csv`
+- `only_in_record.csv`
+
+默认 compare 输入是：
+
+- `target/live/apnic_delta_demo/20260315-170223-autoplay/record-delta.csv`
+
+也可以通过环境变量覆盖：
+
+- `TRUST_ANCHOR`
+- `ROUTINATOR_RECORD_CSV`
+- `VRPS_CSV`
+- `COMPARE_SUMMARY_MD`
+- `ONLY_IN_OURS_CSV`
+- `ONLY_IN_RECORD_CSV`
+
+## `write_multi_rir_case_report.py`
+
+把某个 RIR 的 snapshot replay 与 delta replay 的 `meta.json`、compare summary 以及 Routinator timing 基线合并成一个 per-RIR Markdown/JSON 报告。
+
+该脚本通常由 `run_multi_rir_replay_case.sh <rir> both` 自动调用。
+
+## `run_multi_rir_replay_suite.sh`
+
+顺序执行 5 个 RIR（或环境变量 `RIRS` 指定的子集）的 `both` 模式，并最终生成 multi-RIR 汇总报告。
+
+```bash
+./scripts/payload_replay/run_multi_rir_replay_suite.sh
+```
+
+可覆盖环境变量：
+
+- `BUNDLE_ROOT`
+- `SUITE_OUT_DIR`
+- `RIRS`
+
+最终输出：
+
+- `<suite_out_dir>/<rir>/<rir>_case_report.md`
+- `<suite_out_dir>/multi_rir_summary.md`
+- `<suite_out_dir>/multi_rir_summary.json`
+
+## `write_multi_rir_summary.py`
+
+汇总 5 个 RIR 的 per-RIR case report，生成 correctness + timing 总表与几何平均比值。

scripts/payload_replay/compare_with_routinator_record.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -lt 2 || $# -gt 5 ]]; then
+  echo "Usage: $0 <ours.csv> <record.csv> [summary.md] [only_in_ours.csv] [only_in_record.csv]" >&2
+  exit 2
+fi
+
+OURS_CSV="$1"
+RECORD_CSV="$2"
+SUMMARY_MD="${3:-}"
+ONLY_IN_OURS_CSV="${4:-}"
+ONLY_IN_RECORD_CSV="${5:-}"
+
+if [[ -z "$SUMMARY_MD" ]]; then
+  SUMMARY_MD="$(dirname "$OURS_CSV")/$(basename "$OURS_CSV" .csv)_vs_routinator_summary.md"
+fi
+if [[ -z "$ONLY_IN_OURS_CSV" ]]; then
+  ONLY_IN_OURS_CSV="$(dirname "$OURS_CSV")/$(basename "$OURS_CSV" .csv)_only_in_ours.csv"
+fi
+if [[ -z "$ONLY_IN_RECORD_CSV" ]]; then
+  ONLY_IN_RECORD_CSV="$(dirname "$OURS_CSV")/$(basename "$OURS_CSV" .csv)_only_in_record.csv"
+fi
+
+python3 - "$OURS_CSV" "$RECORD_CSV" "$SUMMARY_MD" "$ONLY_IN_OURS_CSV" "$ONLY_IN_RECORD_CSV" <<'PY'
+import csv
+import ipaddress
+import sys
+from pathlib import Path
+
+ours_csv = Path(sys.argv[1])
+record_csv = Path(sys.argv[2])
+summary_md = Path(sys.argv[3])
+only_in_ours_csv = Path(sys.argv[4])
+only_in_record_csv = Path(sys.argv[5])
+
+
+def normalize_row(row: dict):
+    asn = row["ASN"].strip().upper()
+    prefix = row["IP Prefix"].strip()
+    max_len = str(int(row["Max Length"]))
+    ta = row["Trust Anchor"].strip()
+    network = ipaddress.ip_network(prefix, strict=False)
+    return {
+        "ASN": asn,
+        "IP Prefix": str(network),
+        "Max Length": max_len,
+        "Trust Anchor": ta,
+    }
+
+
+def read_rows(path: Path):
+    with path.open(encoding="utf-8", newline="") as f:
+        rows = [normalize_row(r) for r in csv.DictReader(f)]
+    return rows
+
+
+def row_key(row: dict):
+    network = ipaddress.ip_network(row["IP Prefix"], strict=False)
+    return (
+        row["ASN"],
+        network.version,
+        int(network.network_address),
+        network.prefixlen,
+        int(row["Max Length"]),
+        row["Trust Anchor"],
+    )
+
+
+def write_rows(path: Path, rows):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=["ASN", "IP Prefix", "Max Length", "Trust Anchor"])
+        writer.writeheader()
+        for row in rows:
+            writer.writerow(row)
+
+ours = read_rows(ours_csv)
+record = read_rows(record_csv)
+ours_map = {row_key(r): r for r in ours}
+record_map = {row_key(r): r for r in record}
+only_in_ours = [ours_map[k] for k in sorted(set(ours_map) - set(record_map))]
+only_in_record = [record_map[k] for k in sorted(set(record_map) - set(ours_map))]
+intersection = len(set(ours_map) & set(record_map))
+
+write_rows(only_in_ours_csv, only_in_ours)
+write_rows(only_in_record_csv, only_in_record)
+
+summary_md.parent.mkdir(parents=True, exist_ok=True)
+summary = []
+summary.append("# Replay vs Routinator VRP Compare\n\n")
+summary.append(f"- ours_csv: `{ours_csv}`\n")
+summary.append(f"- record_csv: `{record_csv}`\n")
+summary.append(f"- only_in_ours_csv: `{only_in_ours_csv}`\n")
+summary.append(f"- only_in_record_csv: `{only_in_record_csv}`\n\n")
+summary.append("| metric | value |\n")
+summary.append("|---|---:|\n")
+summary.append(f"| ours_total | {len(ours_map)} |\n")
+summary.append(f"| record_total | {len(record_map)} |\n")
+summary.append(f"| intersection | {intersection} |\n")
+summary.append(f"| only_in_ours | {len(only_in_ours)} |\n")
+summary.append(f"| only_in_record | {len(only_in_record)} |\n")
+summary_md.write_text("".join(summary), encoding="utf-8")
+print(summary_md)
+PY
+
+echo "== compare complete ==" >&2
+echo "- summary: $SUMMARY_MD" >&2
+echo "- only_in_ours: $ONLY_IN_OURS_CSV" >&2
+echo "- only_in_record: $ONLY_IN_RECORD_CSV" >&2

scripts/payload_replay/multi_rir_case_info.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import shlex
+import sys
+from pathlib import Path
+
+
+RIR_CONFIG = {
+    "afrinic": {
+        "tal": "tests/fixtures/tal/afrinic.tal",
+        "ta": "tests/fixtures/ta/afrinic-ta.cer",
+        "trust_anchor": "afrinic",
+    },
+    "apnic": {
+        "tal": "tests/fixtures/tal/apnic-rfc7730-https.tal",
+        "ta": "tests/fixtures/ta/apnic-ta.cer",
+        "trust_anchor": "apnic",
+    },
+    "arin": {
+        "tal": "tests/fixtures/tal/arin.tal",
+        "ta": "tests/fixtures/ta/arin-ta.cer",
+        "trust_anchor": "arin",
+    },
+    "lacnic": {
+        "tal": "tests/fixtures/tal/lacnic.tal",
+        "ta": "tests/fixtures/ta/lacnic-ta.cer",
+        "trust_anchor": "lacnic",
+    },
+    "ripe": {
+        "tal": "tests/fixtures/tal/ripe-ncc.tal",
+        "ta": "tests/fixtures/ta/ripe-ncc-ta.cer",
+        "trust_anchor": "ripe",
+    },
+}
+
+
+def default_repo_root() -> Path:
+    return Path(__file__).resolve().parents[2]
+
+
+def default_bundle_root(repo_root: Path) -> Path:
+    return (repo_root / "../../rpki/target/live/20260316-112341-multi-final3").resolve()
+
+
+def require_path(path: Path, kind: str) -> Path:
+    if kind == "dir" and not path.is_dir():
+        raise SystemExit(f"missing directory: {path}")
+    if kind == "file" and not path.is_file():
+        raise SystemExit(f"missing file: {path}")
+    return path
+
+
+def load_timing_summary(bundle_root: Path) -> dict:
+    timing_path = require_path(bundle_root / "timing-summary.json", "file")
+    return json.loads(timing_path.read_text(encoding="utf-8"))
+
+
+def load_json(path: Path) -> dict:
+    return json.loads(require_path(path, "file").read_text(encoding="utf-8"))
+
+
+def lock_validation_time(lock_obj: dict, fallback_started_at: str) -> str:
+    return lock_obj.get("validationTime") or lock_obj.get("validation_time") or fallback_started_at
+
+
+def build_case(bundle_root: Path, repo_root: Path, rir: str) -> dict:
+    if rir not in RIR_CONFIG:
+        raise SystemExit(
+            f"unsupported rir: {rir}; expected one of: {', '.join(sorted(RIR_CONFIG))}"
+        )
+
+    rir_root = require_path(bundle_root / rir, "dir")
+    cfg = RIR_CONFIG[rir]
+    timing_summary = load_timing_summary(bundle_root)
+    if rir not in timing_summary:
+        raise SystemExit(f"timing-summary.json missing entry for rir: {rir}")
+    timing_entry = timing_summary[rir]
+    durations = timing_entry.get("durations") or {}
+
+    base_timing = require_path(rir_root / "timings" / "base-replay.json", "file")
+    delta_timing = require_path(rir_root / "timings" / "delta-replay.json", "file")
+    base_timing_obj = json.loads(base_timing.read_text(encoding="utf-8"))
+    delta_timing_obj = json.loads(delta_timing.read_text(encoding="utf-8"))
+    base_locks_obj = load_json(rir_root / "base-locks.json")
+    delta_locks_obj = load_json(rir_root / "locks-delta.json")
+
+    case = {
+        "bundle_root": str(bundle_root),
+        "repo_root": str(repo_root),
+        "rir": rir,
+        "trust_anchor": cfg["trust_anchor"],
+        "rir_root": str(rir_root),
+        "base_archive": str(require_path(rir_root / "base-payload-archive", "dir")),
+        "base_locks": str(require_path(rir_root / "base-locks.json", "file")),
+        "base_vrps_csv": str(require_path(rir_root / "base-vrps.csv", "file")),
+        "delta_archive": str(require_path(rir_root / "payload-delta-archive", "dir")),
+        "delta_locks": str(require_path(rir_root / "locks-delta.json", "file")),
+        "delta_record_csv": str(require_path(rir_root / "record-delta.csv", "file")),
+        "replay_delta_csv": str(require_path(rir_root / "replay-delta.csv", "file")),
+        "verification_json": str(require_path(rir_root / "verification.json", "file")),
+        "readme": str(require_path(rir_root / "README.md", "file")),
+        "timings_dir": str(require_path(rir_root / "timings", "dir")),
+        "base_timing_json": str(base_timing),
+        "delta_timing_json": str(delta_timing),
+        "tal_path": str(require_path(repo_root / cfg["tal"], "file")),
+        "ta_path": str(require_path(repo_root / cfg["ta"], "file")),
+        "validation_times": {
+            "snapshot": lock_validation_time(base_locks_obj, base_timing_obj["startedAt"]),
+            "delta": lock_validation_time(delta_locks_obj, delta_timing_obj["startedAt"]),
+        },
+        "timing_started_at": {
+            "snapshot_replay": base_timing_obj["startedAt"],
+            "delta_replay": delta_timing_obj["startedAt"],
+        },
+        "routinator_timings": {
+            "base_replay_seconds": float(durations["base-replay"]),
+            "delta_replay_seconds": float(durations["delta-replay"]),
+        },
+    }
+    return case
+
+
+def emit_env(case: dict) -> str:
+    ordered = {
+        "BUNDLE_ROOT": case["bundle_root"],
+        "RIR": case["rir"],
+        "TRUST_ANCHOR": case["trust_anchor"],
+        "RIR_ROOT": case["rir_root"],
+        "TAL_PATH": case["tal_path"],
+        "TA_PATH": case["ta_path"],
+        "PAYLOAD_REPLAY_ARCHIVE": case["base_archive"],
+        "PAYLOAD_REPLAY_LOCKS": case["base_locks"],
+        "ROUTINATOR_BASE_RECORD_CSV": case["base_vrps_csv"],
+        "PAYLOAD_BASE_ARCHIVE": case["base_archive"],
+        "PAYLOAD_BASE_LOCKS": case["base_locks"],
+        "PAYLOAD_DELTA_ARCHIVE": case["delta_archive"],
+        "PAYLOAD_DELTA_LOCKS": case["delta_locks"],
+        "ROUTINATOR_DELTA_RECORD_CSV": case["delta_record_csv"],
+        "SNAPSHOT_VALIDATION_TIME": case["validation_times"]["snapshot"],
+        "DELTA_VALIDATION_TIME": case["validation_times"]["delta"],
+        "ROUTINATOR_BASE_REPLAY_SECONDS": str(case["routinator_timings"]["base_replay_seconds"]),
+        "ROUTINATOR_DELTA_REPLAY_SECONDS": str(case["routinator_timings"]["delta_replay_seconds"]),
+    }
+    return "\n".join(
+        f"export {key}={shlex.quote(value)}" for key, value in ordered.items()
+    )
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Resolve one RIR case inside a multi-RIR replay bundle")
+    parser.add_argument("--bundle-root", type=Path, default=None)
+    parser.add_argument("--repo-root", type=Path, default=None)
+    parser.add_argument("--rir", required=True, choices=sorted(RIR_CONFIG))
+    parser.add_argument("--format", choices=["json", "env"], default="json")
+    args = parser.parse_args()
+
+    repo_root = (args.repo_root or default_repo_root()).resolve()
+    bundle_root = (args.bundle_root or default_bundle_root(repo_root)).resolve()
+    case = build_case(bundle_root, repo_root, args.rir)
+
+    if args.format == "env":
+        print(emit_env(case))
+    else:
+        print(json.dumps(case, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/payload_replay/report_to_routinator_csv.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+import argparse
+import csv
+import ipaddress
+import json
+from pathlib import Path
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(
+        description="Convert rpki report.json VRPs into Routinator-compatible CSV"
+    )
+    p.add_argument("--report", required=True, help="path to rpki report.json")
+    p.add_argument("--out", required=True, help="output CSV path")
+    p.add_argument(
+        "--trust-anchor",
+        default="unknown",
+        help="Trust Anchor column value (default: unknown)",
+    )
+    return p.parse_args()
+
+
+def sort_key(vrp: dict):
+    network = ipaddress.ip_network(vrp["prefix"], strict=False)
+    return (
+        int(vrp["asn"]),
+        network.version,
+        int(network.network_address),
+        network.prefixlen,
+        int(vrp["max_length"]),
+    )
+
+
+def main() -> int:
+    args = parse_args()
+    report = json.loads(Path(args.report).read_text(encoding="utf-8"))
+    vrps = list(report.get("vrps") or [])
+    vrps.sort(key=sort_key)
+
+    out_path = Path(args.out)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    with out_path.open("w", encoding="utf-8", newline="") as f:
+        w = csv.writer(f)
+        w.writerow(["ASN", "IP Prefix", "Max Length", "Trust Anchor"])
+        for vrp in vrps:
+            w.writerow([
+                f"AS{vrp['asn']}",
+                vrp["prefix"],
+                vrp["max_length"],
+                args.trust_anchor,
+            ])
+    print(out_path)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/payload_replay/run_apnic_delta_replay.sh

@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+DELTA_ROOT="${DELTA_ROOT:-$ROOT_DIR/target/live/apnic_delta_demo/20260315-170223-autoplay}"
+TAL_PATH="${TAL_PATH:-$ROOT_DIR/tests/fixtures/tal/apnic-rfc7730-https.tal}"
+TA_PATH="${TA_PATH:-$ROOT_DIR/tests/fixtures/ta/apnic-ta.cer}"
+PAYLOAD_BASE_ARCHIVE="${PAYLOAD_BASE_ARCHIVE:-$DELTA_ROOT/base-payload-archive}"
+PAYLOAD_BASE_LOCKS="${PAYLOAD_BASE_LOCKS:-$DELTA_ROOT/base-locks.json}"
+PAYLOAD_DELTA_ARCHIVE="${PAYLOAD_DELTA_ARCHIVE:-$DELTA_ROOT/payload-delta-archive}"
+PAYLOAD_DELTA_LOCKS="${PAYLOAD_DELTA_LOCKS:-$DELTA_ROOT/locks-delta.json}"
+VALIDATION_TIME="${VALIDATION_TIME:-}"
+PAYLOAD_BASE_VALIDATION_TIME="${PAYLOAD_BASE_VALIDATION_TIME:-}"
+TRUST_ANCHOR="${TRUST_ANCHOR:-apnic}"
+ROUTINATOR_RECORD_CSV="${ROUTINATOR_RECORD_CSV:-$DELTA_ROOT/record-delta.csv}"
+MAX_DEPTH="${MAX_DEPTH:-}"
+MAX_INSTANCES="${MAX_INSTANCES:-}"
+OUT_DIR="${OUT_DIR:-$ROOT_DIR/target/live/payload_delta_replay_runs}"
+mkdir -p "$OUT_DIR"
+
+if [[ -z "$PAYLOAD_BASE_VALIDATION_TIME" ]]; then
+  PAYLOAD_BASE_VALIDATION_TIME="$(python3 - "$PAYLOAD_BASE_LOCKS" <<'LOCKPY'
+import json, sys
+from pathlib import Path
+path = Path(sys.argv[1])
+data = json.loads(path.read_text(encoding='utf-8'))
+print(data.get('validationTime') or data.get('validation_time') or '')
+LOCKPY
+)"
+fi
+
+if [[ -z "$VALIDATION_TIME" ]]; then
+  VALIDATION_TIME="$(python3 - "$PAYLOAD_DELTA_LOCKS" <<'LOCKPY'
+import json, sys
+from pathlib import Path
+path = Path(sys.argv[1])
+data = json.loads(path.read_text(encoding='utf-8'))
+print(data.get('validationTime') or data.get('validation_time') or '2026-03-15T10:00:00Z')
+LOCKPY
+)"
+fi
+
+TS="$(date -u +%Y%m%dT%H%M%SZ)"
+RUN_NAME="${RUN_NAME:-apnic_delta_replay_${TS}}"
+DB_DIR="${DB_DIR:-$OUT_DIR/${RUN_NAME}_db}"
+REPORT_JSON="${REPORT_JSON:-$OUT_DIR/${RUN_NAME}_report.json}"
+RUN_LOG="${RUN_LOG:-$OUT_DIR/${RUN_NAME}_run.log}"
+META_JSON="${META_JSON:-$OUT_DIR/${RUN_NAME}_meta.json}"
+SUMMARY_MD="${SUMMARY_MD:-$OUT_DIR/${RUN_NAME}_summary.md}"
+VRPS_CSV="${VRPS_CSV:-$OUT_DIR/${RUN_NAME}_vrps.csv}"
+COMPARE_SUMMARY_MD="${COMPARE_SUMMARY_MD:-$OUT_DIR/${RUN_NAME}_compare_summary.md}"
+ONLY_IN_OURS_CSV="${ONLY_IN_OURS_CSV:-$OUT_DIR/${RUN_NAME}_only_in_ours.csv}"
+ONLY_IN_RECORD_CSV="${ONLY_IN_RECORD_CSV:-$OUT_DIR/${RUN_NAME}_only_in_record.csv}"
+
+cmd=(cargo run --release --bin rpki --
+  --db "$DB_DIR"
+  --tal-path "$TAL_PATH"
+  --ta-path "$TA_PATH"
+  --payload-base-archive "$PAYLOAD_BASE_ARCHIVE"
+  --payload-base-locks "$PAYLOAD_BASE_LOCKS"
+  --payload-delta-archive "$PAYLOAD_DELTA_ARCHIVE"
+  --payload-delta-locks "$PAYLOAD_DELTA_LOCKS"
+  --validation-time "$VALIDATION_TIME"
+  --report-json "$REPORT_JSON")
+
+if [[ -n "$MAX_DEPTH" ]]; then
+  cmd+=(--max-depth "$MAX_DEPTH")
+fi
+if [[ -n "$MAX_INSTANCES" ]]; then
+  cmd+=(--max-instances "$MAX_INSTANCES")
+fi
+
+run_start_s="$(date +%s)"
+(
+  echo "# command:"
+  printf '%q ' "${cmd[@]}"
+  echo
+  echo
+  "${cmd[@]}"
+) 2>&1 | tee "$RUN_LOG" >/dev/null
+run_end_s="$(date +%s)"
+run_duration_s="$((run_end_s - run_start_s))"
+
+PAYLOAD_BASE_ARCHIVE="$PAYLOAD_BASE_ARCHIVE" \
+PAYLOAD_BASE_LOCKS="$PAYLOAD_BASE_LOCKS" \
+PAYLOAD_DELTA_ARCHIVE="$PAYLOAD_DELTA_ARCHIVE" \
+PAYLOAD_DELTA_LOCKS="$PAYLOAD_DELTA_LOCKS" \
+PAYLOAD_BASE_VALIDATION_TIME="$PAYLOAD_BASE_VALIDATION_TIME" \
+DB_DIR="$DB_DIR" \
+REPORT_JSON="$REPORT_JSON" \
+RUN_LOG="$RUN_LOG" \
+VALIDATION_TIME="$VALIDATION_TIME" \
+RUN_DURATION_S="$run_duration_s" \
+python3 - "$REPORT_JSON" "$META_JSON" "$SUMMARY_MD" <<'PY'
+import json
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+report_path = Path(sys.argv[1])
+meta_path = Path(sys.argv[2])
+summary_path = Path(sys.argv[3])
+rep = json.loads(report_path.read_text(encoding='utf-8'))
+now = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+meta = {
+  'recorded_at_utc': now,
+  'payload_base_archive': os.environ['PAYLOAD_BASE_ARCHIVE'],
+  'payload_base_locks': os.environ['PAYLOAD_BASE_LOCKS'],
+  'payload_delta_archive': os.environ['PAYLOAD_DELTA_ARCHIVE'],
+  'payload_delta_locks': os.environ['PAYLOAD_DELTA_LOCKS'],
+  'db_dir': os.environ['DB_DIR'],
+  'report_json': os.environ['REPORT_JSON'],
+  'run_log': os.environ['RUN_LOG'],
+  'validation_time_arg': os.environ['VALIDATION_TIME'],
+  'base_validation_time_arg': os.environ.get('PAYLOAD_BASE_VALIDATION_TIME') or os.environ['VALIDATION_TIME'],
+  'durations_secs': {'rpki_run': int(os.environ['RUN_DURATION_S'])},
+  'counts': {
+    'publication_points_processed': rep['tree']['instances_processed'],
+    'publication_points_failed': rep['tree']['instances_failed'],
+    'vrps': len(rep['vrps']),
+    'aspas': len(rep['aspas']),
+    'audit_publication_points': len(rep['publication_points']),
+  },
+}
+meta_path.write_text(json.dumps(meta, ensure_ascii=False, indent=2)+'\n', encoding='utf-8')
+summary = []
+summary.append('# Payload Delta Replay Summary\n\n')
+for key in ['payload_base_archive','payload_base_locks','payload_delta_archive','payload_delta_locks','db_dir','report_json','base_validation_time_arg','validation_time_arg']:
+    summary.append(f'- {key}: `{meta[key]}`\n')
+summary.append('\n## Results\n\n| metric | value |\n|---|---:|\n')
+for k,v in meta['counts'].items():
+    summary.append(f'| {k} | {v} |\n')
+summary.append('\n## Durations\n\n| step | seconds |\n|---|---:|\n')
+for k,v in meta['durations_secs'].items():
+    summary.append(f'| {k} | {v} |\n')
+summary_path.write_text(''.join(summary), encoding='utf-8')
+print(summary_path)
+PY
+
+python3 scripts/payload_replay/report_to_routinator_csv.py \
+  --report "$REPORT_JSON" \
+  --out "$VRPS_CSV" \
+  --trust-anchor "$TRUST_ANCHOR" >/dev/null
+
+if [[ -f "$ROUTINATOR_RECORD_CSV" ]]; then
+  ./scripts/payload_replay/compare_with_routinator_record.sh \
+    "$VRPS_CSV" \
+    "$ROUTINATOR_RECORD_CSV" \
+    "$COMPARE_SUMMARY_MD" \
+    "$ONLY_IN_OURS_CSV" \
+    "$ONLY_IN_RECORD_CSV" >/dev/null
+fi
+
+echo "== payload delta replay run complete ==" >&2
+echo "- db: $DB_DIR" >&2
+echo "- report: $REPORT_JSON" >&2
+echo "- run log: $RUN_LOG" >&2
+echo "- meta json: $META_JSON" >&2
+echo "- summary md: $SUMMARY_MD" >&2
+echo "- vrps csv: $VRPS_CSV" >&2
+if [[ -f "$COMPARE_SUMMARY_MD" ]]; then
+  echo "- compare summary: $COMPARE_SUMMARY_MD" >&2
+  echo "- only in ours: $ONLY_IN_OURS_CSV" >&2
+  echo "- only in record: $ONLY_IN_RECORD_CSV" >&2
+fi

scripts/payload_replay/run_apnic_replay.sh

@@ -0,0 +1,161 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+TAL_PATH="${TAL_PATH:-$ROOT_DIR/tests/fixtures/tal/apnic-rfc7730-https.tal}"
+TA_PATH="${TA_PATH:-$ROOT_DIR/tests/fixtures/ta/apnic-ta.cer}"
+PAYLOAD_REPLAY_ARCHIVE="${PAYLOAD_REPLAY_ARCHIVE:-$ROOT_DIR/target/live/payload_replay/payload-archive}"
+PAYLOAD_REPLAY_LOCKS="${PAYLOAD_REPLAY_LOCKS:-$ROOT_DIR/target/live/payload_replay/locks.json}"
+VALIDATION_TIME="${VALIDATION_TIME:-}"
+TRUST_ANCHOR="${TRUST_ANCHOR:-apnic}"
+ROUTINATOR_RECORD_CSV="${ROUTINATOR_RECORD_CSV:-$ROOT_DIR/target/live/payload_replay/record.csv}"
+MAX_DEPTH="${MAX_DEPTH:-}"
+MAX_INSTANCES="${MAX_INSTANCES:-}"
+OUT_DIR="${OUT_DIR:-$ROOT_DIR/target/live/payload_replay_runs}"
+mkdir -p "$OUT_DIR"
+
+if [[ -z "$VALIDATION_TIME" ]]; then
+  VALIDATION_TIME="$(python3 - "$PAYLOAD_REPLAY_LOCKS" <<'LOCKPY'
+import json, sys
+from pathlib import Path
+path = Path(sys.argv[1])
+data = json.loads(path.read_text(encoding='utf-8'))
+print(data.get('validationTime') or data.get('validation_time') or '2026-03-13T02:30:00Z')
+LOCKPY
+)"
+fi
+
+TS="$(date -u +%Y%m%dT%H%M%SZ)"
+RUN_NAME="${RUN_NAME:-apnic_replay_${TS}}"
+DB_DIR="${DB_DIR:-$OUT_DIR/${RUN_NAME}_db}"
+REPORT_JSON="${REPORT_JSON:-$OUT_DIR/${RUN_NAME}_report.json}"
+RUN_LOG="${RUN_LOG:-$OUT_DIR/${RUN_NAME}_run.log}"
+META_JSON="${META_JSON:-$OUT_DIR/${RUN_NAME}_meta.json}"
+SUMMARY_MD="${SUMMARY_MD:-$OUT_DIR/${RUN_NAME}_summary.md}"
+VRPS_CSV="${VRPS_CSV:-$OUT_DIR/${RUN_NAME}_vrps.csv}"
+COMPARE_SUMMARY_MD="${COMPARE_SUMMARY_MD:-$OUT_DIR/${RUN_NAME}_compare_summary.md}"
+ONLY_IN_OURS_CSV="${ONLY_IN_OURS_CSV:-$OUT_DIR/${RUN_NAME}_only_in_ours.csv}"
+ONLY_IN_RECORD_CSV="${ONLY_IN_RECORD_CSV:-$OUT_DIR/${RUN_NAME}_only_in_record.csv}"
+
+cmd=(cargo run --release --bin rpki --
+  --db "$DB_DIR"
+  --tal-path "$TAL_PATH"
+  --ta-path "$TA_PATH"
+  --payload-replay-archive "$PAYLOAD_REPLAY_ARCHIVE"
+  --payload-replay-locks "$PAYLOAD_REPLAY_LOCKS"
+  --validation-time "$VALIDATION_TIME"
+  --report-json "$REPORT_JSON")
+
+if [[ -n "$MAX_DEPTH" ]]; then
+  cmd+=(--max-depth "$MAX_DEPTH")
+fi
+if [[ -n "$MAX_INSTANCES" ]]; then
+  cmd+=(--max-instances "$MAX_INSTANCES")
+fi
+
+run_start_s="$(date +%s)"
+(
+  echo "# command:"
+  printf '%q ' "${cmd[@]}"
+  echo
+  echo
+  "${cmd[@]}"
+) 2>&1 | tee "$RUN_LOG" >/dev/null
+run_end_s="$(date +%s)"
+run_duration_s="$((run_end_s - run_start_s))"
+
+TAL_PATH="$TAL_PATH" \
+TA_PATH="$TA_PATH" \
+PAYLOAD_REPLAY_ARCHIVE="$PAYLOAD_REPLAY_ARCHIVE" \
+PAYLOAD_REPLAY_LOCKS="$PAYLOAD_REPLAY_LOCKS" \
+DB_DIR="$DB_DIR" \
+REPORT_JSON="$REPORT_JSON" \
+RUN_LOG="$RUN_LOG" \
+VALIDATION_TIME="$VALIDATION_TIME" \
+RUN_DURATION_S="$run_duration_s" \
+python3 - "$REPORT_JSON" "$META_JSON" "$SUMMARY_MD" <<'PY'
+import json
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+report_path = Path(sys.argv[1])
+meta_path = Path(sys.argv[2])
+summary_path = Path(sys.argv[3])
+rep = json.loads(report_path.read_text(encoding="utf-8"))
+now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+meta = {
+  "recorded_at_utc": now,
+  "tal_path": os.environ["TAL_PATH"],
+  "ta_path": os.environ["TA_PATH"],
+  "payload_replay_archive": os.environ["PAYLOAD_REPLAY_ARCHIVE"],
+  "payload_replay_locks": os.environ["PAYLOAD_REPLAY_LOCKS"],
+  "db_dir": os.environ["DB_DIR"],
+  "report_json": os.environ["REPORT_JSON"],
+  "run_log": os.environ["RUN_LOG"],
+  "validation_time_arg": os.environ["VALIDATION_TIME"],
+  "durations_secs": {
+    "rpki_run": int(os.environ["RUN_DURATION_S"]),
+  },
+  "counts": {
+    "publication_points_processed": rep["tree"]["instances_processed"],
+    "publication_points_failed": rep["tree"]["instances_failed"],
+    "vrps": len(rep["vrps"]),
+    "aspas": len(rep["aspas"]),
+    "audit_publication_points": len(rep["publication_points"]),
+  },
+}
+meta_path.write_text(json.dumps(meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+summary = []
+summary.append("# Payload Replay Summary\n\n")
+summary.append(f"- recorded_at_utc: `{now}`\n")
+summary.append(f"- tal_path: `{meta['tal_path']}`\n")
+summary.append(f"- ta_path: `{meta['ta_path']}`\n")
+summary.append(f"- payload_replay_archive: `{meta['payload_replay_archive']}`\n")
+summary.append(f"- payload_replay_locks: `{meta['payload_replay_locks']}`\n")
+summary.append(f"- db: `{meta['db_dir']}`\n")
+summary.append(f"- report_json: `{meta['report_json']}`\n")
+summary.append(f"- validation_time_arg: `{meta['validation_time_arg']}`\n\n")
+summary.append("## Results\n\n")
+summary.append("| metric | value |\n")
+summary.append("|---|---:|\n")
+for k, v in meta["counts"].items():
+    summary.append(f"| {k} | {v} |\n")
+summary.append("\n## Durations\n\n")
+summary.append("| step | seconds |\n")
+summary.append("|---|---:|\n")
+for k, v in meta["durations_secs"].items():
+    summary.append(f"| {k} | {v} |\n")
+summary_path.write_text("".join(summary), encoding="utf-8")
+print(summary_path)
+PY
+
+python3 scripts/payload_replay/report_to_routinator_csv.py \
+  --report "$REPORT_JSON" \
+  --out "$VRPS_CSV" \
+  --trust-anchor "$TRUST_ANCHOR" >/dev/null
+
+if [[ -f "$ROUTINATOR_RECORD_CSV" ]]; then
+  ./scripts/payload_replay/compare_with_routinator_record.sh \
+    "$VRPS_CSV" \
+    "$ROUTINATOR_RECORD_CSV" \
+    "$COMPARE_SUMMARY_MD" \
+    "$ONLY_IN_OURS_CSV" \
+    "$ONLY_IN_RECORD_CSV" >/dev/null
+fi
+
+echo "== payload replay run complete ==" >&2
+echo "- db: $DB_DIR" >&2
+echo "- report: $REPORT_JSON" >&2
+echo "- run log: $RUN_LOG" >&2
+echo "- meta json: $META_JSON" >&2
+echo "- summary md: $SUMMARY_MD" >&2
+echo "- vrps csv: $VRPS_CSV" >&2
+if [[ -f "$COMPARE_SUMMARY_MD" ]]; then
+  echo "- compare summary: $COMPARE_SUMMARY_MD" >&2
+  echo "- only in ours: $ONLY_IN_OURS_CSV" >&2
+  echo "- only in record: $ONLY_IN_RECORD_CSV" >&2
+fi

scripts/payload_replay/run_apnic_snapshot_replay_profile.sh

@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+BUNDLE_ROOT="${BUNDLE_ROOT:-$ROOT_DIR/../../rpki/target/live/20260316-112341-multi-final3}"
+CASE_INFO_SCRIPT="$ROOT_DIR/scripts/payload_replay/multi_rir_case_info.py"
+PROFILE_RUN_ROOT="${PROFILE_RUN_ROOT:-$ROOT_DIR/target/live/analyze_runs}"
+mkdir -p "$PROFILE_RUN_ROOT"
+
+TS="$(date -u +%Y%m%dT%H%M%SZ)"
+RUN_NAME="${RUN_NAME:-apnic_snapshot_profile_${TS}}"
+RUN_DIR="$PROFILE_RUN_ROOT/$RUN_NAME"
+mkdir -p "$RUN_DIR"
+
+ANALYZE_ROOT="$ROOT_DIR/target/live/analyze"
+mkdir -p "$ANALYZE_ROOT"
+mapfile -t ANALYZE_BEFORE < <(find "$ANALYZE_ROOT" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort)
+
+if [[ "${DRY_RUN:-0}" == "1" && ! -d "$BUNDLE_ROOT" ]]; then
+  TRUST_ANCHOR="apnic"
+  TAL_PATH="$ROOT_DIR/tests/fixtures/tal/apnic-rfc7730-https.tal"
+  TA_PATH="$ROOT_DIR/tests/fixtures/ta/apnic-ta.cer"
+  PAYLOAD_REPLAY_ARCHIVE="$BUNDLE_ROOT/apnic/base-payload-archive"
+  PAYLOAD_REPLAY_LOCKS="$BUNDLE_ROOT/apnic/base-locks.json"
+  SNAPSHOT_VALIDATION_TIME="2026-03-16T00:00:00Z"
+  ROUTINATOR_BASE_REPLAY_SECONDS="0"
+else
+  eval "$(python3 "$CASE_INFO_SCRIPT" --bundle-root "$BUNDLE_ROOT" --rir apnic --format env)"
+fi
+
+DB_DIR="${DB_DIR:-$RUN_DIR/db}"
+REPORT_JSON="${REPORT_JSON:-$RUN_DIR/report.json}"
+RUN_LOG="${RUN_LOG:-$RUN_DIR/run.log}"
+META_JSON="${META_JSON:-$RUN_DIR/meta.json}"
+SUMMARY_MD="${SUMMARY_MD:-$RUN_DIR/summary.md}"
+
+rm -rf "$DB_DIR"
+
+cmd=(cargo run --release --features profile --bin rpki --
+  --db "$DB_DIR"
+  --tal-path "$TAL_PATH"
+  --ta-path "$TA_PATH"
+  --payload-replay-archive "$PAYLOAD_REPLAY_ARCHIVE"
+  --payload-replay-locks "$PAYLOAD_REPLAY_LOCKS"
+  --validation-time "$SNAPSHOT_VALIDATION_TIME"
+  --analyze
+  --profile-cpu
+  --report-json "$REPORT_JSON")
+
+if [[ -n "${MAX_DEPTH:-}" ]]; then
+  cmd+=(--max-depth "$MAX_DEPTH")
+fi
+if [[ -n "${MAX_INSTANCES:-}" ]]; then
+  cmd+=(--max-instances "$MAX_INSTANCES")
+fi
+
+if [[ "${DRY_RUN:-0}" == "1" ]]; then
+  printf '%q ' "${cmd[@]}"
+  echo
+  exit 0
+fi
+
+run_start_s="$(date +%s)"
+(
+  echo '# command:'
+  printf '%q ' "${cmd[@]}"
+  echo
+  echo
+  "${cmd[@]}"
+) 2>&1 | tee "$RUN_LOG" >/dev/null
+run_end_s="$(date +%s)"
+run_duration_s="$((run_end_s - run_start_s))"
+
+mapfile -t ANALYZE_AFTER < <(find "$ANALYZE_ROOT" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort)
+ANALYZE_DIR=""
+for candidate in "${ANALYZE_AFTER[@]}"; do
+  seen=0
+  for old in "${ANALYZE_BEFORE[@]}"; do
+    if [[ "$candidate" == "$old" ]]; then
+      seen=1
+      break
+    fi
+  done
+  if [[ "$seen" == "0" ]]; then
+    ANALYZE_DIR="$candidate"
+  fi
+done
+if [[ -z "$ANALYZE_DIR" ]]; then
+  ANALYZE_DIR="$(find "$ANALYZE_ROOT" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort | tail -n 1)"
+fi
+
+BUNDLE_ROOT="$BUNDLE_ROOT" \
+TRUST_ANCHOR="$TRUST_ANCHOR" \
+TAL_PATH="$TAL_PATH" \
+TA_PATH="$TA_PATH" \
+PAYLOAD_REPLAY_ARCHIVE="$PAYLOAD_REPLAY_ARCHIVE" \
+PAYLOAD_REPLAY_LOCKS="$PAYLOAD_REPLAY_LOCKS" \
+SNAPSHOT_VALIDATION_TIME="$SNAPSHOT_VALIDATION_TIME" \
+ROUTINATOR_BASE_REPLAY_SECONDS="$ROUTINATOR_BASE_REPLAY_SECONDS" \
+DB_DIR="$DB_DIR" \
+REPORT_JSON="$REPORT_JSON" \
+RUN_LOG="$RUN_LOG" \
+ANALYZE_DIR="$ANALYZE_DIR" \
+RUN_DURATION_S="$run_duration_s" \
+python3 - "$META_JSON" "$SUMMARY_MD" <<'PY'
+import json
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+meta_path = Path(sys.argv[1])
+summary_path = Path(sys.argv[2])
+report_path = Path(os.environ['REPORT_JSON'])
+report = json.loads(report_path.read_text(encoding='utf-8'))
+recorded = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+meta = {
+    'recorded_at_utc': recorded,
+    'bundle_root': os.environ['BUNDLE_ROOT'],
+    'trust_anchor': os.environ['TRUST_ANCHOR'],
+    'tal_path': os.environ['TAL_PATH'],
+    'ta_path': os.environ['TA_PATH'],
+    'payload_replay_archive': os.environ['PAYLOAD_REPLAY_ARCHIVE'],
+    'payload_replay_locks': os.environ['PAYLOAD_REPLAY_LOCKS'],
+    'validation_time_arg': os.environ['SNAPSHOT_VALIDATION_TIME'],
+    'routinator_base_replay_seconds': float(os.environ['ROUTINATOR_BASE_REPLAY_SECONDS']),
+    'db_dir': os.environ['DB_DIR'],
+    'report_json': os.environ['REPORT_JSON'],
+    'run_log': os.environ['RUN_LOG'],
+    'analyze_dir': os.environ.get('ANALYZE_DIR') or '',
+    'durations_secs': {
+        'rpki_run_wall': int(os.environ['RUN_DURATION_S']),
+    },
+    'counts': {
+        'publication_points_processed': report['tree']['instances_processed'],
+        'publication_points_failed': report['tree']['instances_failed'],
+        'vrps': len(report['vrps']),
+        'aspas': len(report['aspas']),
+        'audit_publication_points': len(report['publication_points']),
+    },
+}
+meta_path.write_text(json.dumps(meta, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
+ratio = meta['durations_secs']['rpki_run_wall'] / meta['routinator_base_replay_seconds'] if meta['routinator_base_replay_seconds'] else None
+lines = []
+lines.append('# APNIC Snapshot Replay Profile Summary\n\n')
+lines.append(f"- recorded_at_utc: `{recorded}`\n")
+lines.append(f"- bundle_root: `{meta['bundle_root']}`\n")
+lines.append(f"- tal_path: `{meta['tal_path']}`\n")
+lines.append(f"- ta_path: `{meta['ta_path']}`\n")
+lines.append(f"- payload_replay_archive: `{meta['payload_replay_archive']}`\n")
+lines.append(f"- payload_replay_locks: `{meta['payload_replay_locks']}`\n")
+lines.append(f"- validation_time_arg: `{meta['validation_time_arg']}`\n")
+lines.append(f"- db_dir: `{meta['db_dir']}`\n")
+lines.append(f"- report_json: `{meta['report_json']}`\n")
+lines.append(f"- run_log: `{meta['run_log']}`\n")
+lines.append(f"- analyze_dir: `{meta['analyze_dir']}`\n\n")
+lines.append('## Timing\n\n')
+lines.append('| metric | value |\n')
+lines.append('|---|---:|\n')
+lines.append(f"| ours_snapshot_replay_wall_s | {meta['durations_secs']['rpki_run_wall']} |\n")
+lines.append(f"| routinator_base_replay_s | {meta['routinator_base_replay_seconds']:.3f} |\n")
+if ratio is not None:
+    lines.append(f"| ratio_ours_over_routinator | {ratio:.3f} |\n")
+lines.append('\n## Counts\n\n')
+for key, value in meta['counts'].items():
+    lines.append(f"- {key}: `{value}`\n")
+summary_path.write_text(''.join(lines), encoding='utf-8')
+PY
+
+echo "== APNIC snapshot replay profiling complete ==" >&2
+echo "- run_dir: $RUN_DIR" >&2
+echo "- analyze_dir: $ANALYZE_DIR" >&2
+echo "- report_json: $REPORT_JSON" >&2
+echo "- run_log: $RUN_LOG" >&2
+echo "- meta_json: $META_JSON" >&2
+echo "- summary_md: $SUMMARY_MD" >&2

scripts/payload_replay/run_multi_rir_replay_case.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+if [[ $# -lt 1 || $# -gt 2 ]]; then
+  echo "usage: $0 <rir> [describe|snapshot|delta|both]" >&2
+  exit 2
+fi
+
+RIR="$1"
+MODE="${2:-both}"
+BUNDLE_ROOT="${BUNDLE_ROOT:-$ROOT_DIR/../../rpki/target/live/20260316-112341-multi-final3}"
+CASE_INFO_SCRIPT="$ROOT_DIR/scripts/payload_replay/multi_rir_case_info.py"
+CASE_REPORT_SCRIPT="$ROOT_DIR/scripts/payload_replay/write_multi_rir_case_report.py"
+MULTI_RIR_OUT_DIR="${MULTI_RIR_OUT_DIR:-$ROOT_DIR/target/live/multi_rir_replay_runs/$RIR}"
+mkdir -p "$MULTI_RIR_OUT_DIR"
+
+eval "$(python3 "$CASE_INFO_SCRIPT" --bundle-root "$BUNDLE_ROOT" --rir "$RIR" --format env)"
+
+SNAPSHOT_DB_DIR="${SNAPSHOT_DB_DIR:-$MULTI_RIR_OUT_DIR/${RIR}_snapshot_replay_db}"
+SNAPSHOT_REPORT_MD="${SNAPSHOT_REPORT_MD:-$MULTI_RIR_OUT_DIR/${RIR}_snapshot_compare_summary.md}"
+SNAPSHOT_META_JSON="${SNAPSHOT_META_JSON:-$MULTI_RIR_OUT_DIR/${RIR}_snapshot_meta.json}"
+SNAPSHOT_RUN_LOG="${SNAPSHOT_RUN_LOG:-$MULTI_RIR_OUT_DIR/${RIR}_snapshot_run.log}"
+SNAPSHOT_REPORT_JSON="${SNAPSHOT_REPORT_JSON:-$MULTI_RIR_OUT_DIR/${RIR}_snapshot_report.json}"
+SNAPSHOT_VRPS_CSV="${SNAPSHOT_VRPS_CSV:-$MULTI_RIR_OUT_DIR/${RIR}_snapshot_vrps.csv}"
+SNAPSHOT_ONLY_OURS="${SNAPSHOT_ONLY_OURS:-$MULTI_RIR_OUT_DIR/${RIR}_snapshot_only_in_ours.csv}"
+SNAPSHOT_ONLY_RECORD="${SNAPSHOT_ONLY_RECORD:-$MULTI_RIR_OUT_DIR/${RIR}_snapshot_only_in_record.csv}"
+
+DELTA_DB_DIR="${DELTA_DB_DIR:-$MULTI_RIR_OUT_DIR/${RIR}_delta_replay_db}"
+DELTA_REPORT_MD="${DELTA_REPORT_MD:-$MULTI_RIR_OUT_DIR/${RIR}_delta_compare_summary.md}"
+DELTA_META_JSON="${DELTA_META_JSON:-$MULTI_RIR_OUT_DIR/${RIR}_delta_meta.json}"
+DELTA_RUN_LOG="${DELTA_RUN_LOG:-$MULTI_RIR_OUT_DIR/${RIR}_delta_run.log}"
+DELTA_REPORT_JSON="${DELTA_REPORT_JSON:-$MULTI_RIR_OUT_DIR/${RIR}_delta_report.json}"
+DELTA_VRPS_CSV="${DELTA_VRPS_CSV:-$MULTI_RIR_OUT_DIR/${RIR}_delta_vrps.csv}"
+DELTA_ONLY_OURS="${DELTA_ONLY_OURS:-$MULTI_RIR_OUT_DIR/${RIR}_delta_only_in_ours.csv}"
+DELTA_ONLY_RECORD="${DELTA_ONLY_RECORD:-$MULTI_RIR_OUT_DIR/${RIR}_delta_only_in_record.csv}"
+
+CASE_REPORT_JSON="${CASE_REPORT_JSON:-$MULTI_RIR_OUT_DIR/${RIR}_case_report.json}"
+CASE_REPORT_MD="${CASE_REPORT_MD:-$MULTI_RIR_OUT_DIR/${RIR}_case_report.md}"
+
+case "$MODE" in
+  describe)
+    python3 "$CASE_INFO_SCRIPT" --bundle-root "$BUNDLE_ROOT" --rir "$RIR"
+    ;;
+  snapshot)
+    rm -rf "$SNAPSHOT_DB_DIR"
+    ROUTINATOR_RECORD_CSV="$ROUTINATOR_BASE_RECORD_CSV" \
+      VALIDATION_TIME="$SNAPSHOT_VALIDATION_TIME" \
+      OUT_DIR="$MULTI_RIR_OUT_DIR" \
+      DB_DIR="$SNAPSHOT_DB_DIR" \
+      RUN_NAME="${RUN_NAME:-${RIR}_snapshot_replay}" \
+      META_JSON="$SNAPSHOT_META_JSON" \
+      RUN_LOG="$SNAPSHOT_RUN_LOG" \
+      REPORT_JSON="$SNAPSHOT_REPORT_JSON" \
+      VRPS_CSV="$SNAPSHOT_VRPS_CSV" \
+      COMPARE_SUMMARY_MD="$SNAPSHOT_REPORT_MD" \
+      ONLY_IN_OURS_CSV="$SNAPSHOT_ONLY_OURS" \
+      ONLY_IN_RECORD_CSV="$SNAPSHOT_ONLY_RECORD" \
+      ./scripts/payload_replay/run_apnic_replay.sh
+    ;;
+  delta)
+    rm -rf "$DELTA_DB_DIR"
+    ROUTINATOR_RECORD_CSV="$ROUTINATOR_DELTA_RECORD_CSV" \
+      VALIDATION_TIME="$DELTA_VALIDATION_TIME" \
+      PAYLOAD_BASE_VALIDATION_TIME="$SNAPSHOT_VALIDATION_TIME" \
+      OUT_DIR="$MULTI_RIR_OUT_DIR" \
+      DB_DIR="$DELTA_DB_DIR" \
+      RUN_NAME="${RUN_NAME:-${RIR}_delta_replay}" \
+      DELTA_ROOT="$RIR_ROOT" \
+      META_JSON="$DELTA_META_JSON" \
+      RUN_LOG="$DELTA_RUN_LOG" \
+      REPORT_JSON="$DELTA_REPORT_JSON" \
+      VRPS_CSV="$DELTA_VRPS_CSV" \
+      COMPARE_SUMMARY_MD="$DELTA_REPORT_MD" \
+      ONLY_IN_OURS_CSV="$DELTA_ONLY_OURS" \
+      ONLY_IN_RECORD_CSV="$DELTA_ONLY_RECORD" \
+      ./scripts/payload_replay/run_apnic_delta_replay.sh
+    ;;
+  both)
+    rm -rf "$SNAPSHOT_DB_DIR" "$DELTA_DB_DIR"
+    ROUTINATOR_RECORD_CSV="$ROUTINATOR_BASE_RECORD_CSV" \
+      VALIDATION_TIME="$SNAPSHOT_VALIDATION_TIME" \
+      OUT_DIR="$MULTI_RIR_OUT_DIR" \
+      DB_DIR="$SNAPSHOT_DB_DIR" \
+      RUN_NAME="${RUN_NAME_SNAPSHOT:-${RIR}_snapshot_replay}" \
+      META_JSON="$SNAPSHOT_META_JSON" \
+      RUN_LOG="$SNAPSHOT_RUN_LOG" \
+      REPORT_JSON="$SNAPSHOT_REPORT_JSON" \
+      VRPS_CSV="$SNAPSHOT_VRPS_CSV" \
+      COMPARE_SUMMARY_MD="$SNAPSHOT_REPORT_MD" \
+      ONLY_IN_OURS_CSV="$SNAPSHOT_ONLY_OURS" \
+      ONLY_IN_RECORD_CSV="$SNAPSHOT_ONLY_RECORD" \
+      ./scripts/payload_replay/run_apnic_replay.sh
+    ROUTINATOR_RECORD_CSV="$ROUTINATOR_DELTA_RECORD_CSV" \
+      VALIDATION_TIME="$DELTA_VALIDATION_TIME" \
+      PAYLOAD_BASE_VALIDATION_TIME="$SNAPSHOT_VALIDATION_TIME" \
+      OUT_DIR="$MULTI_RIR_OUT_DIR" \
+      DB_DIR="$DELTA_DB_DIR" \
+      RUN_NAME="${RUN_NAME_DELTA:-${RIR}_delta_replay}" \
+      DELTA_ROOT="$RIR_ROOT" \
+      META_JSON="$DELTA_META_JSON" \
+      RUN_LOG="$DELTA_RUN_LOG" \
+      REPORT_JSON="$DELTA_REPORT_JSON" \
+      VRPS_CSV="$DELTA_VRPS_CSV" \
+      COMPARE_SUMMARY_MD="$DELTA_REPORT_MD" \
+      ONLY_IN_OURS_CSV="$DELTA_ONLY_OURS" \
+      ONLY_IN_RECORD_CSV="$DELTA_ONLY_RECORD" \
+      ./scripts/payload_replay/run_apnic_delta_replay.sh
+    python3 "$CASE_REPORT_SCRIPT" \
+      --rir "$RIR" \
+      --snapshot-meta "$SNAPSHOT_META_JSON" \
+      --snapshot-compare "$SNAPSHOT_REPORT_MD" \
+      --delta-meta "$DELTA_META_JSON" \
+      --delta-compare "$DELTA_REPORT_MD" \
+      --routinator-base-seconds "$ROUTINATOR_BASE_REPLAY_SECONDS" \
+      --routinator-delta-seconds "$ROUTINATOR_DELTA_REPLAY_SECONDS" \
+      --out-md "$CASE_REPORT_MD" \
+      --out-json "$CASE_REPORT_JSON" >/dev/null
+    echo "- case report: $CASE_REPORT_MD" >&2
+    echo "- case report json: $CASE_REPORT_JSON" >&2
+    ;;
+  *)
+    echo "unsupported mode: $MODE; expected describe|snapshot|delta|both" >&2
+    exit 2
+    ;;
+esac

scripts/payload_replay/run_multi_rir_replay_suite.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+BUNDLE_ROOT="${BUNDLE_ROOT:-$ROOT_DIR/../../rpki/target/live/20260316-112341-multi-final3}"
+SUITE_OUT_DIR="${SUITE_OUT_DIR:-$ROOT_DIR/target/live/multi_rir_replay_runs}"
+RIRS="${RIRS:-afrinic apnic arin lacnic ripe}"
+CASE_SCRIPT="$ROOT_DIR/scripts/payload_replay/run_multi_rir_replay_case.sh"
+SUMMARY_SCRIPT="$ROOT_DIR/scripts/payload_replay/write_multi_rir_summary.py"
+
+mkdir -p "$SUITE_OUT_DIR"
+
+for rir in $RIRS; do
+  MULTI_RIR_OUT_DIR="$SUITE_OUT_DIR/$rir" \
+    BUNDLE_ROOT="$BUNDLE_ROOT" \
+    "$CASE_SCRIPT" "$rir" both
+  echo "completed $rir" >&2
+  echo >&2
+ done
+
+python3 "$SUMMARY_SCRIPT" \
+  --case-root "$SUITE_OUT_DIR" \
+  --out-md "$SUITE_OUT_DIR/multi_rir_summary.md" \
+  --out-json "$SUITE_OUT_DIR/multi_rir_summary.json" \
+  --rirs $RIRS >/dev/null
+
+echo "== multi-RIR replay suite complete ==" >&2
+echo "- suite_out_dir: $SUITE_OUT_DIR" >&2
+echo "- summary_md: $SUITE_OUT_DIR/multi_rir_summary.md" >&2
+echo "- summary_json: $SUITE_OUT_DIR/multi_rir_summary.json" >&2

scripts/payload_replay/write_multi_rir_case_report.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description="Generate one multi-RIR replay case report")
+    p.add_argument("--rir", required=True)
+    p.add_argument("--snapshot-meta", required=True)
+    p.add_argument("--snapshot-compare", required=True)
+    p.add_argument("--delta-meta", required=True)
+    p.add_argument("--delta-compare", required=True)
+    p.add_argument("--routinator-base-seconds", required=True, type=float)
+    p.add_argument("--routinator-delta-seconds", required=True, type=float)
+    p.add_argument("--out-md", required=True)
+    p.add_argument("--out-json", required=True)
+    return p.parse_args()
+
+
+def read_json(path: str) -> dict:
+    return json.loads(Path(path).read_text(encoding="utf-8"))
+
+
+def parse_compare_md(path: str) -> dict:
+    lines = Path(path).read_text(encoding="utf-8").splitlines()
+    out = {}
+    for line in lines:
+        if not line.startswith("| "):
+            continue
+        parts = [p.strip() for p in line.strip("|").split("|")]
+        if len(parts) != 2:
+            continue
+        key, value = parts
+        if key in {"metric", "---"}:
+            continue
+        try:
+            out[key] = int(value)
+        except ValueError:
+            pass
+    return out
+
+
+def ratio(ours: float, baseline: float) -> float | None:
+    if baseline <= 0:
+        return None
+    return ours / baseline
+
+
+def build_report(args: argparse.Namespace) -> dict:
+    snapshot_meta = read_json(args.snapshot_meta)
+    delta_meta = read_json(args.delta_meta)
+    snapshot_compare = parse_compare_md(args.snapshot_compare)
+    delta_compare = parse_compare_md(args.delta_compare)
+
+    snapshot_ours = float(snapshot_meta["durations_secs"]["rpki_run"])
+    delta_ours = float(delta_meta["durations_secs"]["rpki_run"])
+
+    report = {
+        "rir": args.rir,
+        "snapshot": {
+            "meta_json": str(Path(args.snapshot_meta).resolve()),
+            "compare_md": str(Path(args.snapshot_compare).resolve()),
+            "ours_seconds": snapshot_ours,
+            "routinator_seconds": args.routinator_base_seconds,
+            "ratio": ratio(snapshot_ours, args.routinator_base_seconds),
+            "compare": snapshot_compare,
+            "match": snapshot_compare.get("only_in_ours", -1) == 0
+            and snapshot_compare.get("only_in_record", -1) == 0,
+            "counts": snapshot_meta.get("counts", {}),
+        },
+        "delta": {
+            "meta_json": str(Path(args.delta_meta).resolve()),
+            "compare_md": str(Path(args.delta_compare).resolve()),
+            "ours_seconds": delta_ours,
+            "routinator_seconds": args.routinator_delta_seconds,
+            "ratio": ratio(delta_ours, args.routinator_delta_seconds),
+            "compare": delta_compare,
+            "match": delta_compare.get("only_in_ours", -1) == 0
+            and delta_compare.get("only_in_record", -1) == 0,
+            "counts": delta_meta.get("counts", {}),
+        },
+    }
+    return report
+
+
+def write_md(path: Path, report: dict) -> None:
+    snapshot = report["snapshot"]
+    delta = report["delta"]
+    lines = []
+    lines.append(f"# {report['rir'].upper()} Replay Report\n\n")
+    lines.append("## Summary\n\n")
+    lines.append("| mode | match | ours_s | routinator_s | ratio | only_in_ours | only_in_record |\n")
+    lines.append("|---|---|---:|---:|---:|---:|---:|\n")
+    lines.append(
+        f"| snapshot | {str(snapshot['match']).lower()} | {snapshot['ours_seconds']:.3f} | {snapshot['routinator_seconds']:.3f} | {snapshot['ratio']:.3f} | {snapshot['compare'].get('only_in_ours', 0)} | {snapshot['compare'].get('only_in_record', 0)} |\n"
+    )
+    lines.append(
+        f"| delta | {str(delta['match']).lower()} | {delta['ours_seconds']:.3f} | {delta['routinator_seconds']:.3f} | {delta['ratio']:.3f} | {delta['compare'].get('only_in_ours', 0)} | {delta['compare'].get('only_in_record', 0)} |\n"
+    )
+    lines.append("\n## Snapshot Inputs\n\n")
+    lines.append(f"- meta_json: `{snapshot['meta_json']}`\n")
+    lines.append(f"- compare_md: `{snapshot['compare_md']}`\n")
+    lines.append("\n## Delta Inputs\n\n")
+    lines.append(f"- meta_json: `{delta['meta_json']}`\n")
+    lines.append(f"- compare_md: `{delta['compare_md']}`\n")
+    lines.append("\n## Counts\n\n")
+    lines.append("### Snapshot\n\n")
+    for k, v in sorted(snapshot.get("counts", {}).items()):
+        lines.append(f"- {k}: `{v}`\n")
+    lines.append("\n### Delta\n\n")
+    for k, v in sorted(delta.get("counts", {}).items()):
+        lines.append(f"- {k}: `{v}`\n")
+    path.write_text("".join(lines), encoding="utf-8")
+
+
+def main() -> int:
+    args = parse_args()
+    report = build_report(args)
+    out_json = Path(args.out_json)
+    out_md = Path(args.out_md)
+    out_json.parent.mkdir(parents=True, exist_ok=True)
+    out_md.parent.mkdir(parents=True, exist_ok=True)
+    out_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    write_md(out_md, report)
+    print(out_md)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/payload_replay/write_multi_rir_summary.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import math
+from pathlib import Path
+
+DEFAULT_RIRS = ["afrinic", "apnic", "arin", "lacnic", "ripe"]
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description="Aggregate per-RIR replay case reports")
+    p.add_argument("--case-root", required=True, help="directory containing <rir>/<rir>_case_report.json")
+    p.add_argument("--out-md", required=True)
+    p.add_argument("--out-json", required=True)
+    p.add_argument("--rirs", nargs="*", default=None, help="RIRs to include (default: all 5)")
+    return p.parse_args()
+
+
+def read_case(case_root: Path, rir: str) -> dict:
+    path = case_root / rir / f"{rir}_case_report.json"
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def geomean(values: list[float]) -> float:
+    vals = [v for v in values if v > 0]
+    if not vals:
+        return 0.0
+    return math.exp(sum(math.log(v) for v in vals) / len(vals))
+
+
+def build_summary(cases: list[dict]) -> dict:
+    snapshot_ratios = [c["snapshot"]["ratio"] for c in cases]
+    delta_ratios = [c["delta"]["ratio"] for c in cases]
+    return {
+        "cases": cases,
+        "summary": {
+            "snapshot_all_match": all(c["snapshot"]["match"] for c in cases),
+            "delta_all_match": all(c["delta"]["match"] for c in cases),
+            "snapshot_ratio_geomean": geomean(snapshot_ratios),
+            "delta_ratio_geomean": geomean(delta_ratios),
+            "all_ratio_geomean": geomean(snapshot_ratios + delta_ratios),
+        },
+    }
+
+
+def write_md(path: Path, data: dict) -> None:
+    lines = []
+    lines.append("# Multi-RIR Replay Summary\n\n")
+    lines.append("## Correctness + Timing\n\n")
+    lines.append("| RIR | snapshot_match | snapshot_ours_s | snapshot_routinator_s | snapshot_ratio | delta_match | delta_ours_s | delta_routinator_s | delta_ratio |\n")
+    lines.append("|---|---|---:|---:|---:|---|---:|---:|---:|\n")
+    for case in data["cases"]:
+        lines.append(
+            f"| {case['rir']} | {str(case['snapshot']['match']).lower()} | {case['snapshot']['ours_seconds']:.3f} | {case['snapshot']['routinator_seconds']:.3f} | {case['snapshot']['ratio']:.3f} | {str(case['delta']['match']).lower()} | {case['delta']['ours_seconds']:.3f} | {case['delta']['routinator_seconds']:.3f} | {case['delta']['ratio']:.3f} |\n"
+        )
+    s = data["summary"]
+    lines.append("\n## Aggregate Metrics\n\n")
+    lines.append("| metric | value |\n")
+    lines.append("|---|---:|\n")
+    lines.append(f"| snapshot_all_match | {str(s['snapshot_all_match']).lower()} |\n")
+    lines.append(f"| delta_all_match | {str(s['delta_all_match']).lower()} |\n")
+    lines.append(f"| snapshot_ratio_geomean | {s['snapshot_ratio_geomean']:.3f} |\n")
+    lines.append(f"| delta_ratio_geomean | {s['delta_ratio_geomean']:.3f} |\n")
+    lines.append(f"| all_ratio_geomean | {s['all_ratio_geomean']:.3f} |\n")
+    path.write_text("".join(lines), encoding="utf-8")
+
+
+def main() -> int:
+    args = parse_args()
+    case_root = Path(args.case_root)
+    rirs = args.rirs or DEFAULT_RIRS
+    cases = [read_case(case_root, rir) for rir in rirs]
+    data = build_summary(cases)
+    out_md = Path(args.out_md)
+    out_json = Path(args.out_json)
+    out_md.parent.mkdir(parents=True, exist_ok=True)
+    out_json.parent.mkdir(parents=True, exist_ok=True)
+    out_json.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    write_md(out_md, data)
+    print(out_md)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/periodic/compare_ccr_cir_round.sh

@@ -0,0 +1,191 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  ./scripts/periodic/compare_ccr_cir_round.sh \
+    --ours-ccr <path> \
+    --rpki-client-ccr <path> \
+    --out-dir <path> \
+    [--ours-cir <path>] \
+    [--rpki-client-cir <path>] \
+    [--trust-anchor <name>] \
+    [--sample-limit <n>] \
+    [--always-compare-cir]
+USAGE
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+OURS_CCR=""
+CLIENT_CCR=""
+OURS_CIR=""
+CLIENT_CIR=""
+OUT_DIR=""
+TRUST_ANCHOR="unknown"
+SAMPLE_LIMIT="20"
+ALWAYS_COMPARE_CIR=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --ours-ccr) OURS_CCR="$2"; shift 2 ;;
+    --rpki-client-ccr) CLIENT_CCR="$2"; shift 2 ;;
+    --ours-cir) OURS_CIR="$2"; shift 2 ;;
+    --rpki-client-cir) CLIENT_CIR="$2"; shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --trust-anchor) TRUST_ANCHOR="$2"; shift 2 ;;
+    --sample-limit) SAMPLE_LIMIT="$2"; shift 2 ;;
+    --always-compare-cir) ALWAYS_COMPARE_CIR=1; shift ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$OURS_CCR" && -n "$CLIENT_CCR" && -n "$OUT_DIR" ]] || { usage >&2; exit 2; }
+if [[ -n "$OURS_CIR" || -n "$CLIENT_CIR" ]]; then
+  [[ -n "$OURS_CIR" && -n "$CLIENT_CIR" ]] || { echo "--ours-cir and --rpki-client-cir must be provided together" >&2; exit 2; }
+fi
+
+mkdir -p "$OUT_DIR/ccr"
+
+"$ROOT_DIR/scripts/periodic/compare_ccr_round.sh" \
+  --ours-ccr "$OURS_CCR" \
+  --rpki-client-ccr "$CLIENT_CCR" \
+  --out-dir "$OUT_DIR/ccr" \
+  --trust-anchor "$TRUST_ANCHOR" >/dev/null
+
+RUN_CIR="$(python3 - <<'PY' "$OUT_DIR/ccr/compare-summary.json" "$ALWAYS_COMPARE_CIR" "$OURS_CIR" "$CLIENT_CIR"
+import json
+import sys
+summary = json.load(open(sys.argv[1], "r", encoding="utf-8"))
+always = sys.argv[2] == "1"
+has_cir = bool(sys.argv[3]) and bool(sys.argv[4])
+ccr_match = summary.get("stateDigestMatch") is True or summary.get("allMatch") is True
+print("1" if has_cir and (always or not ccr_match) else "0")
+PY
+)"
+
+if [[ "$RUN_CIR" == "1" ]]; then
+  mkdir -p "$OUT_DIR/cir"
+  "$ROOT_DIR/scripts/periodic/compare_cir_round.sh" \
+    --ours-cir "$OURS_CIR" \
+    --rpki-client-cir "$CLIENT_CIR" \
+    --out-dir "$OUT_DIR/cir" \
+    --sample-limit "$SAMPLE_LIMIT" >/dev/null
+fi
+
+python3 - <<'PY' \
+  "$OUT_DIR/ccr/compare-summary.json" \
+  "$OUT_DIR/cir/cir-compare-summary.json" \
+  "$OUT_DIR/summary.json" \
+  "$OUT_DIR/summary.md" \
+  "$ALWAYS_COMPARE_CIR" \
+  "$OURS_CIR" \
+  "$CLIENT_CIR" \
+  "$TRUST_ANCHOR" \
+  "$SAMPLE_LIMIT"
+import json
+import sys
+from pathlib import Path
+
+ccr_path = Path(sys.argv[1])
+cir_path = Path(sys.argv[2])
+summary_json = Path(sys.argv[3])
+summary_md = Path(sys.argv[4])
+always_compare_cir = sys.argv[5] == "1"
+ours_cir = sys.argv[6]
+client_cir = sys.argv[7]
+trust_anchor = sys.argv[8]
+sample_limit = int(sys.argv[9])
+
+ccr = json.load(open(ccr_path, "r", encoding="utf-8"))
+ccr_match = ccr.get("stateDigestMatch") is True or ccr.get("allMatch") is True
+cir_available = bool(ours_cir) and bool(client_cir)
+cir_compared = cir_path.exists()
+cir = json.load(open(cir_path, "r", encoding="utf-8")) if cir_compared else None
+
+if ccr_match:
+    if cir_compared and cir and cir.get("allMatch") is not True:
+        diagnosis = "ccr_state_same_but_cir_process_diff"
+    else:
+        diagnosis = "ccr_state_digest_match"
+else:
+    if not cir_available:
+        diagnosis = "ccr_mismatch_cir_not_available"
+    elif not cir_compared:
+        diagnosis = "ccr_mismatch_cir_compare_skipped"
+    elif cir.get("trustAnchors", {}).get("match") is not True:
+        diagnosis = "trust_anchor_input_difference"
+    elif cir.get("objects", {}).get("match") is not True:
+        diagnosis = "input_object_or_manifest_accepted_set_difference"
+    elif (
+        cir.get("rejects", {}).get("match") is not True
+        or cir.get("rejectListSha256Match") is not True
+    ):
+        diagnosis = "validation_reject_policy_difference"
+    else:
+        diagnosis = "ccr_projection_sorting_encoding_or_non_cir_state_difference"
+
+combined = {
+    "allMatch": bool(ccr_match and (not cir_compared or (cir and cir.get("allMatch") is True))),
+    "diagnosis": diagnosis,
+    "comparePath": ccr.get("comparePath"),
+    "stateDigestMatch": ccr_match,
+    "mismatchedStates": ccr.get("mismatchedStates", []),
+    "mismatchedComponents": ccr.get("mismatchedComponents", []),
+    "vrps": ccr.get("vrps"),
+    "vaps": ccr.get("vaps"),
+    "trustAnchor": trust_anchor,
+    "sampleLimit": sample_limit,
+    "ccr": {
+        "summaryPath": str(ccr_path),
+        "stateDigestMatch": ccr_match,
+        "comparePath": ccr.get("comparePath"),
+        "mismatchedStates": ccr.get("mismatchedStates", []),
+        "mismatchedComponents": ccr.get("mismatchedComponents", []),
+        "vrpMatch": ccr.get("vrps", {}).get("match"),
+        "vapMatch": ccr.get("vaps", {}).get("match"),
+    },
+    "cir": {
+        "available": cir_available,
+        "compared": cir_compared,
+        "summaryPath": str(cir_path) if cir_compared else None,
+        "comparePolicy": "always" if always_compare_cir else "on_ccr_mismatch",
+        "allMatch": cir.get("allMatch") if cir else None,
+        "objectsMatch": cir.get("objects", {}).get("match") if cir else None,
+        "rejectsMatch": cir.get("rejects", {}).get("match") if cir else None,
+        "trustAnchorsMatch": cir.get("trustAnchors", {}).get("match") if cir else None,
+        "talsMatch": cir.get("trustAnchors", {}).get("match") if cir else None,
+        "rejectListSha256Match": cir.get("rejectListSha256Match") if cir else None,
+        "oursObjectCount": cir.get("ours", {}).get("objectCount") if cir else None,
+        "rpkiClientObjectCount": cir.get("rpkiClient", {}).get("objectCount") if cir else None,
+        "oursTrustAnchorCount": cir.get("ours", {}).get("trustAnchorCount") if cir else None,
+        "rpkiClientTrustAnchorCount": cir.get("rpkiClient", {}).get("trustAnchorCount") if cir else None,
+        "oursRejectCount": cir.get("ours", {}).get("rejectCount") if cir else None,
+        "rpkiClientRejectCount": cir.get("rpkiClient", {}).get("rejectCount") if cir else None,
+    },
+}
+
+summary_json.write_text(json.dumps(combined, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+lines = [
+    "# CCR/CIR Compare Summary",
+    "",
+    f"- `allMatch`: `{str(combined['allMatch']).lower()}`",
+    f"- `diagnosis`: `{combined['diagnosis']}`",
+    f"- `ccrStateDigestMatch`: `{str(ccr_match).lower()}`",
+    f"- `ccrMismatchedStates`: `{','.join(combined['ccr']['mismatchedStates'])}`",
+    f"- `cirCompared`: `{str(cir_compared).lower()}`",
+]
+if cir:
+    lines.extend([
+        f"- `cirObjectsMatch`: `{str(combined['cir']['objectsMatch']).lower()}`",
+        f"- `cirRejectsMatch`: `{str(combined['cir']['rejectsMatch']).lower()}`",
+        f"- `cirTrustAnchorsMatch`: `{str(combined['cir']['trustAnchorsMatch']).lower()}`",
+        f"- `cirCounts`: ours objects `{combined['cir']['oursObjectCount']}`, rpki-client objects `{combined['cir']['rpkiClientObjectCount']}`, ours trustAnchors `{combined['cir']['oursTrustAnchorCount']}`, rpki-client trustAnchors `{combined['cir']['rpkiClientTrustAnchorCount']}`, ours rejects `{combined['cir']['oursRejectCount']}`, rpki-client rejects `{combined['cir']['rpkiClientRejectCount']}`",
+    ])
+else:
+    lines.append(f"- `cirSkippedReason`: `{'ccr matched' if ccr_match and not always_compare_cir else 'CIR inputs unavailable'}`")
+summary_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
+print(summary_json)
+PY

scripts/periodic/compare_ccr_round.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/periodic/compare_ccr_round.sh \
+    --ours-ccr <path> \
+    --rpki-client-ccr <path> \
+    --out-dir <path> \
+    [--trust-anchor <name>]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+OURS_CCR=""
+CLIENT_CCR=""
+OUT_DIR=""
+TRUST_ANCHOR="unknown"
+CCR_TO_COMPARE_VIEWS_BIN="$ROOT_DIR/target/release/ccr_to_compare_views"
+CCR_STATE_COMPARE_BIN="$ROOT_DIR/target/release/ccr_state_compare"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --ours-ccr) OURS_CCR="$2"; shift 2 ;;
+    --rpki-client-ccr) CLIENT_CCR="$2"; shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --trust-anchor) TRUST_ANCHOR="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$OURS_CCR" && -n "$CLIENT_CCR" && -n "$OUT_DIR" ]] || { usage >&2; exit 2; }
+
+mkdir -p "$OUT_DIR"
+
+if [[ ! -x "$CCR_STATE_COMPARE_BIN" || ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin ccr_state_compare --bin ccr_to_compare_views
+  )
+fi
+
+OURS_VRPS="$OUT_DIR/ours-vrps.csv"
+OURS_VAPS="$OUT_DIR/ours-vaps.csv"
+CLIENT_VRPS="$OUT_DIR/rpki-client-vrps.csv"
+CLIENT_VAPS="$OUT_DIR/rpki-client-vaps.csv"
+SUMMARY_JSON="$OUT_DIR/compare-summary.json"
+SUMMARY_MD="$OUT_DIR/compare-summary.md"
+
+"$CCR_STATE_COMPARE_BIN" \
+  --ours-ccr "$OURS_CCR" \
+  --rpki-client-ccr "$CLIENT_CCR" \
+  --out-json "$SUMMARY_JSON" \
+  --out-md "$SUMMARY_MD" \
+  --out-dir "$OUT_DIR" \
+  --trust-anchor "$TRUST_ANCHOR" \
+  --fallback-compare-views
+
+echo "$OUT_DIR"

scripts/periodic/compare_cir_round.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  ./scripts/periodic/compare_cir_round.sh \
+    --ours-cir <path> \
+    --rpki-client-cir <path> \
+    --out-dir <path> \
+    [--sample-limit <n>]
+USAGE
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+OURS_CIR=""
+CLIENT_CIR=""
+OUT_DIR=""
+SAMPLE_LIMIT="20"
+CIR_STATE_COMPARE_BIN="$ROOT_DIR/target/release/cir_state_compare"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --ours-cir) OURS_CIR="$2"; shift 2 ;;
+    --rpki-client-cir) CLIENT_CIR="$2"; shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --sample-limit) SAMPLE_LIMIT="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$OURS_CIR" && -n "$CLIENT_CIR" && -n "$OUT_DIR" ]] || { usage >&2; exit 2; }
+mkdir -p "$OUT_DIR"
+
+if [[ ! -x "$CIR_STATE_COMPARE_BIN" ]]; then
+  (cd "$ROOT_DIR" && cargo build --release --bin cir_state_compare)
+fi
+
+"$CIR_STATE_COMPARE_BIN" \
+  --ours-cir "$OURS_CIR" \
+  --rpki-client-cir "$CLIENT_CIR" \
+  --out-json "$OUT_DIR/cir-compare-summary.json" \
+  --out-md "$OUT_DIR/cir-compare-summary.md" \
+  --sample-limit "$SAMPLE_LIMIT"
+
+echo "$OUT_DIR"

scripts/periodic/run_apnic_ours_parallel_round_remote.sh

@@ -0,0 +1,158 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/periodic/run_apnic_ours_parallel_round_remote.sh \
+    --run-root <path> \
+    --round-id <round-XXX> \
+    --kind <snapshot|delta> \
+    --ssh-target <user@host> \
+    --remote-root <path> \
+    [--scheduled-at <RFC3339>] \
+    [--skip-sync]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RUN_ROOT=""
+ROUND_ID=""
+KIND=""
+SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
+REMOTE_ROOT=""
+SCHEDULED_AT=""
+SKIP_SYNC=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --run-root) RUN_ROOT="$2"; shift 2 ;;
+    --round-id) ROUND_ID="$2"; shift 2 ;;
+    --kind) KIND="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --scheduled-at) SCHEDULED_AT="$2"; shift 2 ;;
+    --skip-sync) SKIP_SYNC=1; shift 1 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$RUN_ROOT" && -n "$ROUND_ID" && -n "$KIND" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
+[[ "$KIND" == "snapshot" || "$KIND" == "delta" ]] || { echo "--kind must be snapshot or delta" >&2; exit 2; }
+
+LOCAL_OUT="$RUN_ROOT/rounds/$ROUND_ID/ours"
+REMOTE_REPO="$REMOTE_ROOT/repo"
+REMOTE_OUT="$REMOTE_ROOT/rounds/$ROUND_ID/ours"
+REMOTE_WORK_DB="$REMOTE_ROOT/state/ours/work-db"
+REMOTE_RAW_STORE="$REMOTE_ROOT/state/ours/raw-store.db"
+REMOTE_REPO_BYTES="$REMOTE_ROOT/state/ours/repo-bytes.db"
+
+mkdir -p "$LOCAL_OUT"
+
+if [[ "$SKIP_SYNC" -eq 0 ]]; then
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'"
+  rsync -a --delete \
+    --exclude target \
+    --exclude .git \
+    "$ROOT_DIR/" "$SSH_TARGET:$REMOTE_REPO/"
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_REPO/target/release' '$REMOTE_OUT' '$REMOTE_ROOT/state/ours'"
+  rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_REPO/target/release/"
+else
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_OUT' '$REMOTE_ROOT/state/ours'"
+fi
+
+ssh "$SSH_TARGET" \
+  REMOTE_REPO="$REMOTE_REPO" \
+  REMOTE_OUT="$REMOTE_OUT" \
+  REMOTE_WORK_DB="$REMOTE_WORK_DB" \
+  REMOTE_RAW_STORE="$REMOTE_RAW_STORE" \
+  REMOTE_REPO_BYTES="$REMOTE_REPO_BYTES" \
+  KIND="$KIND" \
+  ROUND_ID="$ROUND_ID" \
+  SCHEDULED_AT="$SCHEDULED_AT" \
+  'bash -s' <<'EOS'
+set -euo pipefail
+
+cd "$REMOTE_REPO"
+mkdir -p "$REMOTE_OUT"
+
+if [[ "$KIND" == "snapshot" ]]; then
+  rm -rf "$REMOTE_WORK_DB" "$REMOTE_RAW_STORE" "$REMOTE_REPO_BYTES"
+fi
+mkdir -p "$(dirname "$REMOTE_WORK_DB")"
+
+started_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+started_at_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+
+ccr_out="$REMOTE_OUT/result.ccr"
+cir_out="$REMOTE_OUT/result.cir"
+report_out="$REMOTE_OUT/report.json"
+run_log="$REMOTE_OUT/run.log"
+meta_out="$REMOTE_OUT/round-result.json"
+
+set +e
+env RPKI_PROGRESS_LOG=1 RPKI_PROGRESS_SLOW_SECS=0 target/release/rpki \
+  --db "$REMOTE_WORK_DB" \
+  --raw-store-db "$REMOTE_RAW_STORE" \
+  --repo-bytes-db "$REMOTE_REPO_BYTES" \
+  --tal-path tests/fixtures/tal/apnic-rfc7730-https.tal \
+  --ta-path tests/fixtures/ta/apnic-ta.cer \
+  --ccr-out "$ccr_out" \
+  --report-json "$report_out" \
+  --cir-enable \
+  --cir-out "$cir_out" \
+  --cir-tal-uri "https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer" \
+  >"$run_log" 2>&1
+exit_code=$?
+set -e
+
+finished_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+finished_at_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+
+python3 - <<'PY' "$meta_out" "$ROUND_ID" "$KIND" "$SCHEDULED_AT" "$started_at_iso" "$finished_at_iso" "$REMOTE_WORK_DB" "$REMOTE_RAW_STORE" "$exit_code" "$started_at_ms" "$finished_at_ms"
+import json, sys
+(
+    path,
+    round_id,
+    kind,
+    scheduled_at,
+    started_at,
+    finished_at,
+    work_db,
+    raw_store,
+    exit_code,
+    start_ms,
+    end_ms,
+) = sys.argv[1:]
+with open(path, "w", encoding="utf-8") as fh:
+    json.dump(
+        {
+            "roundId": round_id,
+            "kind": kind,
+            "scheduledAt": scheduled_at or None,
+            "startedAt": started_at,
+            "finishedAt": finished_at,
+            "durationMs": int(end_ms) - int(start_ms),
+            "remoteWorkDbPath": work_db,
+            "remoteRawStoreDbPath": raw_store,
+            "exitCode": int(exit_code),
+        },
+        fh,
+        indent=2,
+    )
+PY
+
+exit "$exit_code"
+EOS
+
+rsync -a "$SSH_TARGET:$REMOTE_OUT/" "$LOCAL_OUT/"
+echo "$LOCAL_OUT"

scripts/periodic/run_apnic_parallel_dual_rp_periodic_ccr_compare.sh

@@ -0,0 +1,284 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/periodic/run_apnic_parallel_dual_rp_periodic_ccr_compare.sh \
+    --run-root <path> \
+    [--ssh-target <user@host>] \
+    [--remote-root <path>] \
+    [--rpki-client-bin <path>] \
+    [--round-count <n>] \
+    [--interval-secs <n>] \
+    [--start-at <RFC3339>] \
+    [--dry-run]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RUN_ROOT=""
+SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
+REMOTE_ROOT=""
+RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}"
+RPKI_CLIENT_LIBTLS_PATH="${RPKI_CLIENT_LIBTLS_PATH:-/home/yuyr/dev/rpki-client-9.7/.deps/libtls/root/usr/lib/x86_64-linux-gnu/libtls.so.28}"
+ROUND_COUNT=10
+INTERVAL_SECS=600
+START_AT=""
+DRY_RUN=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --run-root) RUN_ROOT="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --rpki-client-libtls) RPKI_CLIENT_LIBTLS_PATH="$2"; shift 2 ;;
+    --round-count) ROUND_COUNT="$2"; shift 2 ;;
+    --interval-secs) INTERVAL_SECS="$2"; shift 2 ;;
+    --start-at) START_AT="$2"; shift 2 ;;
+    --dry-run) DRY_RUN=1; shift 1 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$RUN_ROOT" ]] || { usage >&2; exit 2; }
+[[ "$ROUND_COUNT" =~ ^[0-9]+$ ]] || { echo "--round-count must be an integer" >&2; exit 2; }
+[[ "$INTERVAL_SECS" =~ ^[0-9]+$ ]] || { echo "--interval-secs must be an integer" >&2; exit 2; }
+if [[ "$DRY_RUN" -ne 1 ]]; then
+  [[ -n "$REMOTE_ROOT" ]] || { echo "--remote-root is required unless --dry-run" >&2; exit 2; }
+  [[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
+  [[ -f "$RPKI_CLIENT_LIBTLS_PATH" ]] || { echo "rpki-client libtls not found: $RPKI_CLIENT_LIBTLS_PATH" >&2; exit 2; }
+fi
+
+mkdir -p "$RUN_ROOT"
+
+python3 - <<'PY' "$RUN_ROOT" "$SSH_TARGET" "$REMOTE_ROOT" "$ROUND_COUNT" "$INTERVAL_SECS" "$START_AT" "$DRY_RUN"
+import json
+import sys
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+run_root = Path(sys.argv[1]).resolve()
+ssh_target = sys.argv[2]
+remote_root = sys.argv[3]
+round_count = int(sys.argv[4])
+interval_secs = int(sys.argv[5])
+start_at_arg = sys.argv[6]
+dry_run = bool(int(sys.argv[7]))
+
+def parse_rfc3339_utc(value: str) -> datetime:
+    return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)
+
+def fmt(dt: datetime) -> str:
+    return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+base_time = parse_rfc3339_utc(start_at_arg) if start_at_arg else datetime.now(timezone.utc)
+
+(run_root / "rounds").mkdir(parents=True, exist_ok=True)
+(run_root / "state" / "ours" / "work-db").mkdir(parents=True, exist_ok=True)
+(run_root / "state" / "ours" / "raw-store.db").mkdir(parents=True, exist_ok=True)
+(run_root / "state" / "rpki-client" / "cache").mkdir(parents=True, exist_ok=True)
+(run_root / "state" / "rpki-client" / "out").mkdir(parents=True, exist_ok=True)
+
+meta = {
+    "version": 1,
+    "rir": "apnic",
+    "roundCount": round_count,
+    "intervalSecs": interval_secs,
+    "baseScheduledAt": fmt(base_time),
+    "mode": "dry_run" if dry_run else "remote_periodic",
+    "execution": {
+        "mode": "remote",
+        "sshTarget": ssh_target,
+        "remoteRoot": remote_root or None,
+    },
+}
+(run_root / "meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
+
+rounds = []
+for idx in range(round_count):
+    round_id = f"round-{idx+1:03d}"
+    kind = "snapshot" if idx == 0 else "delta"
+    scheduled_at = base_time + timedelta(seconds=interval_secs * idx)
+    round_dir = run_root / "rounds" / round_id
+    for name in ("ours", "rpki-client", "compare"):
+        (round_dir / name).mkdir(parents=True, exist_ok=True)
+    round_meta = {
+        "roundId": round_id,
+        "kind": kind,
+        "scheduledAt": fmt(scheduled_at),
+        "status": "dry_run" if dry_run else "pending",
+        "paths": {
+            "ours": f"rounds/{round_id}/ours",
+            "rpkiClient": f"rounds/{round_id}/rpki-client",
+            "compare": f"rounds/{round_id}/compare",
+        },
+    }
+    (round_dir / "round-meta.json").write_text(json.dumps(round_meta, indent=2), encoding="utf-8")
+    rounds.append(round_meta)
+
+final_summary = {
+    "version": 1,
+    "status": "dry_run" if dry_run else "pending",
+    "roundCount": round_count,
+    "allMatch": None,
+    "rounds": rounds,
+}
+(run_root / "final-summary.json").write_text(json.dumps(final_summary, indent=2), encoding="utf-8")
+PY
+
+if [[ "$DRY_RUN" -eq 1 ]]; then
+  echo "$RUN_ROOT"
+  exit 0
+fi
+
+if [[ ! -x "$ROOT_DIR/target/release/rpki" || ! -x "$ROOT_DIR/target/release/ccr_to_compare_views" ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin rpki --bin ccr_to_compare_views
+  )
+fi
+
+ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'"
+rsync -a --delete \
+  --exclude target \
+  --exclude .git \
+  "$ROOT_DIR/" "$SSH_TARGET:$REMOTE_ROOT/repo/"
+ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/repo/target/release' '$REMOTE_ROOT/bin' '$REMOTE_ROOT/lib' '$REMOTE_ROOT/rounds' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client'"
+rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_ROOT/repo/target/release/"
+rsync -a "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_ROOT/bin/rpki-client"
+rsync -aL "$RPKI_CLIENT_LIBTLS_PATH" "$SSH_TARGET:$REMOTE_ROOT/lib/libtls.so.28"
+
+for idx in $(seq 1 "$ROUND_COUNT"); do
+  ROUND_ID="$(printf 'round-%03d' "$idx")"
+  ROUND_DIR="$RUN_ROOT/rounds/$ROUND_ID"
+  SCHEDULED_AT="$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
+import json, sys
+print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['scheduledAt'])
+PY
+)"
+  python3 - <<'PY' "$SCHEDULED_AT"
+from datetime import datetime, timezone
+import sys, time
+scheduled = datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).astimezone(timezone.utc)
+delay = (scheduled - datetime.now(timezone.utc)).total_seconds()
+if delay > 0:
+    time.sleep(delay)
+PY
+
+  "$ROOT_DIR/scripts/periodic/run_apnic_ours_parallel_round_remote.sh" \
+    --run-root "$RUN_ROOT" \
+    --round-id "$ROUND_ID" \
+    --kind "$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
+import json, sys
+print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['kind'])
+PY
+)" \
+    --ssh-target "$SSH_TARGET" \
+    --remote-root "$REMOTE_ROOT" \
+    --scheduled-at "$SCHEDULED_AT" \
+    --skip-sync &
+  OURS_PID=$!
+
+  "$ROOT_DIR/scripts/periodic/run_apnic_rpki_client_round_remote.sh" \
+    --run-root "$RUN_ROOT" \
+    --round-id "$ROUND_ID" \
+    --kind "$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
+import json, sys
+print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['kind'])
+PY
+)" \
+    --ssh-target "$SSH_TARGET" \
+    --remote-root "$REMOTE_ROOT" \
+    --scheduled-at "$SCHEDULED_AT" \
+    --rpki-client-bin "$RPKI_CLIENT_BIN" \
+    --skip-sync &
+  CLIENT_PID=$!
+
+  set +e
+  wait "$OURS_PID"; OURS_STATUS=$?
+  wait "$CLIENT_PID"; CLIENT_STATUS=$?
+  set -e
+
+  rsync -az "$SSH_TARGET:$REMOTE_ROOT/rounds/$ROUND_ID/ours/" "$ROUND_DIR/ours/"
+  rsync -az "$SSH_TARGET:$REMOTE_ROOT/rounds/$ROUND_ID/rpki-client/" "$ROUND_DIR/rpki-client/"
+
+  if [[ "$OURS_STATUS" -eq 0 && "$CLIENT_STATUS" -eq 0 \
+        && -f "$ROUND_DIR/ours/result.ccr" && -f "$ROUND_DIR/rpki-client/result.ccr" ]]; then
+    "$ROOT_DIR/scripts/periodic/compare_ccr_round.sh" \
+      --ours-ccr "$ROUND_DIR/ours/result.ccr" \
+      --rpki-client-ccr "$ROUND_DIR/rpki-client/result.ccr" \
+      --out-dir "$ROUND_DIR/compare" \
+      --trust-anchor apnic >/dev/null
+  fi
+
+  python3 - <<'PY' "$ROUND_DIR/round-meta.json" "$ROUND_DIR/ours/round-result.json" "$ROUND_DIR/rpki-client/round-result.json" "$ROUND_DIR/compare/compare-summary.json"
+import json, sys
+from datetime import datetime, timezone
+from pathlib import Path
+meta_path, ours_path, client_path, compare_path = sys.argv[1:]
+meta = json.load(open(meta_path, 'r', encoding='utf-8'))
+ours = json.load(open(ours_path, 'r', encoding='utf-8'))
+client = json.load(open(client_path, 'r', encoding='utf-8'))
+scheduled = datetime.fromisoformat(meta['scheduledAt'].replace('Z', '+00:00')).astimezone(timezone.utc)
+started = [
+    datetime.fromisoformat(v.replace('Z', '+00:00')).astimezone(timezone.utc)
+    for v in [ours.get('startedAt'), client.get('startedAt')] if v
+]
+finished = [
+    datetime.fromisoformat(v.replace('Z', '+00:00')).astimezone(timezone.utc)
+    for v in [ours.get('finishedAt'), client.get('finishedAt')] if v
+]
+if started:
+    start_at = min(started)
+    meta['startedAt'] = start_at.strftime('%Y-%m-%dT%H:%M:%SZ')
+    meta['startLagMs'] = max(int((start_at - scheduled).total_seconds() * 1000), 0)
+if finished:
+    finish_at = max(finished)
+    meta['finishedAt'] = finish_at.strftime('%Y-%m-%dT%H:%M:%SZ')
+meta['status'] = 'completed' if ours.get('exitCode') == 0 and client.get('exitCode') == 0 else 'failed'
+meta['ours'] = {'exitCode': ours.get('exitCode'), 'durationMs': ours.get('durationMs')}
+meta['rpkiClient'] = {'exitCode': client.get('exitCode'), 'durationMs': client.get('durationMs')}
+if Path(compare_path).exists():
+    compare = json.load(open(compare_path, 'r', encoding='utf-8'))
+    meta['compare'] = {
+        'allMatch': compare.get('allMatch'),
+        'vrpMatch': compare.get('vrps', {}).get('match'),
+        'vapMatch': compare.get('vaps', {}).get('match'),
+        'oursVrps': compare.get('vrps', {}).get('ours'),
+        'rpkiClientVrps': compare.get('vrps', {}).get('rpkiClient'),
+        'oursVaps': compare.get('vaps', {}).get('ours'),
+        'rpkiClientVaps': compare.get('vaps', {}).get('rpkiClient'),
+    }
+json.dump(meta, open(meta_path, 'w', encoding='utf-8'), indent=2)
+PY
+
+  ssh "$SSH_TARGET" "rm -rf '$REMOTE_ROOT/rounds/$ROUND_ID'"
+done
+
+python3 - <<'PY' "$RUN_ROOT/final-summary.json" "$RUN_ROOT/rounds"
+import json, sys
+from pathlib import Path
+summary_path = Path(sys.argv[1])
+rounds_root = Path(sys.argv[2])
+rounds = []
+all_match = True
+for round_dir in sorted(rounds_root.glob('round-*')):
+    meta = json.load(open(round_dir / 'round-meta.json', 'r', encoding='utf-8'))
+    rounds.append(meta)
+    compare = meta.get('compare')
+    if compare is None or compare.get('allMatch') is not True:
+        all_match = False
+summary = {
+    'version': 1,
+    'status': 'completed',
+    'roundCount': len(rounds),
+    'allMatch': all_match,
+    'rounds': rounds,
+}
+json.dump(summary, open(summary_path, 'w', encoding='utf-8'), indent=2)
+PY
+
+echo "$RUN_ROOT"

scripts/periodic/run_apnic_rpki_client_round_remote.sh

@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/periodic/run_apnic_rpki_client_round_remote.sh \
+    --run-root <path> \
+    --round-id <round-XXX> \
+    --kind <snapshot|delta> \
+    --ssh-target <user@host> \
+    --remote-root <path> \
+    [--scheduled-at <RFC3339>] \
+    [--rpki-client-bin <local path>] \
+    [--skip-sync]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RUN_ROOT=""
+ROUND_ID=""
+KIND=""
+SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
+REMOTE_ROOT=""
+SCHEDULED_AT=""
+RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}"
+SKIP_SYNC=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --run-root) RUN_ROOT="$2"; shift 2 ;;
+    --round-id) ROUND_ID="$2"; shift 2 ;;
+    --kind) KIND="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --scheduled-at) SCHEDULED_AT="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --skip-sync) SKIP_SYNC=1; shift 1 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$RUN_ROOT" && -n "$ROUND_ID" && -n "$KIND" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
+[[ "$KIND" == "snapshot" || "$KIND" == "delta" ]] || { echo "--kind must be snapshot or delta" >&2; exit 2; }
+[[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
+
+LOCAL_OUT="$RUN_ROOT/rounds/$ROUND_ID/rpki-client"
+REMOTE_REPO="$REMOTE_ROOT/repo"
+REMOTE_BIN_DIR="$REMOTE_ROOT/bin"
+REMOTE_BIN="$REMOTE_BIN_DIR/rpki-client"
+REMOTE_OUT="$REMOTE_ROOT/rounds/$ROUND_ID/rpki-client"
+REMOTE_CACHE="$REMOTE_ROOT/state/rpki-client/cache"
+REMOTE_STATE_OUT="$REMOTE_ROOT/state/rpki-client/out"
+REMOTE_STATE_ROOT="$REMOTE_ROOT/state/rpki-client"
+
+mkdir -p "$LOCAL_OUT"
+
+if [[ "$SKIP_SYNC" -eq 0 ]]; then
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'"
+  rsync -a --delete \
+    --exclude target \
+    --exclude .git \
+    "$ROOT_DIR/" "$SSH_TARGET:$REMOTE_REPO/"
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_BIN_DIR' '$REMOTE_OUT' '$REMOTE_STATE_ROOT'"
+  rsync -a "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_BIN"
+else
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_BIN_DIR' '$REMOTE_OUT' '$REMOTE_STATE_ROOT'"
+fi
+
+ssh "$SSH_TARGET" \
+  REMOTE_ROOT="$REMOTE_ROOT" \
+  REMOTE_BIN="$REMOTE_BIN" \
+  REMOTE_OUT="$REMOTE_OUT" \
+  REMOTE_CACHE="$REMOTE_CACHE" \
+  REMOTE_STATE_OUT="$REMOTE_STATE_OUT" \
+  REMOTE_STATE_ROOT="$REMOTE_STATE_ROOT" \
+  KIND="$KIND" \
+  ROUND_ID="$ROUND_ID" \
+  SCHEDULED_AT="$SCHEDULED_AT" \
+  'bash -s' <<'EOS'
+set -euo pipefail
+
+cd "$REMOTE_ROOT"
+mkdir -p "$REMOTE_OUT"
+
+if [[ "$KIND" == "snapshot" ]]; then
+  rm -rf "$REMOTE_CACHE" "$REMOTE_STATE_OUT" "$REMOTE_STATE_ROOT/ta" "$REMOTE_STATE_ROOT/.ta"
+fi
+mkdir -p "$REMOTE_CACHE" "$REMOTE_STATE_OUT" "$REMOTE_STATE_ROOT/ta" "$REMOTE_STATE_ROOT/.ta"
+chmod -R 0777 "$REMOTE_STATE_ROOT"
+
+started_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+started_at_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+
+ccr_out="$REMOTE_OUT/result.ccr"
+run_log="$REMOTE_OUT/run.log"
+meta_out="$REMOTE_OUT/round-result.json"
+
+set +e
+(
+  cd "$REMOTE_STATE_ROOT"
+  LD_LIBRARY_PATH="$REMOTE_ROOT/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \
+    "$REMOTE_BIN" -vv -t "../../repo/tests/fixtures/tal/apnic-rfc7730-https.tal" -d "cache" "out"
+) >"$run_log" 2>&1
+exit_code=$?
+set -e
+
+if [[ -f "$REMOTE_STATE_OUT/rpki.ccr" ]]; then
+  cp "$REMOTE_STATE_OUT/rpki.ccr" "$ccr_out"
+fi
+if [[ -f "$REMOTE_STATE_OUT/openbgpd" ]]; then
+  cp "$REMOTE_STATE_OUT/openbgpd" "$REMOTE_OUT/openbgpd"
+fi
+
+finished_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+finished_at_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+
+python3 - <<'PY' "$meta_out" "$ROUND_ID" "$KIND" "$SCHEDULED_AT" "$started_at_iso" "$finished_at_iso" "$REMOTE_CACHE" "$REMOTE_STATE_OUT" "$exit_code" "$started_at_ms" "$finished_at_ms"
+import json, sys
+(
+    path,
+    round_id,
+    kind,
+    scheduled_at,
+    started_at,
+    finished_at,
+    cache_path,
+    out_path,
+    exit_code,
+    start_ms,
+    end_ms,
+) = sys.argv[1:]
+with open(path, "w", encoding="utf-8") as fh:
+    json.dump(
+        {
+            "roundId": round_id,
+            "kind": kind,
+            "scheduledAt": scheduled_at or None,
+            "startedAt": started_at,
+            "finishedAt": finished_at,
+            "durationMs": int(end_ms) - int(start_ms),
+            "remoteCachePath": cache_path,
+            "remoteOutPath": out_path,
+            "exitCode": int(exit_code),
+        },
+        fh,
+        indent=2,
+    )
+PY
+
+exit "$exit_code"
+EOS
+
+rsync -a "$SSH_TARGET:$REMOTE_OUT/" "$LOCAL_OUT/"
+echo "$LOCAL_OUT"

scripts/periodic/run_arin_dual_rp_periodic_ccr_compare.sh

@@ -0,0 +1,324 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/periodic/run_arin_dual_rp_periodic_ccr_compare.sh \
+    --run-root <path> \
+    [--ssh-target <user@host>] \
+    [--remote-root <path>] \
+    [--rpki-client-bin <path>] \
+    [--round-count <n>] \
+    [--interval-secs <n>] \
+    [--start-at <RFC3339>] \
+    [--dry-run]
+
+M1 behavior:
+  - creates the periodic run skeleton
+  - writes per-round scheduling metadata
+  - does not execute RP binaries yet
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RUN_ROOT=""
+SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
+REMOTE_ROOT=""
+RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}"
+RPKI_CLIENT_LIBTLS_PATH="${RPKI_CLIENT_LIBTLS_PATH:-/home/yuyr/dev/rpki-client-9.7/.deps/libtls/root/usr/lib/x86_64-linux-gnu/libtls.so.28}"
+ROUND_COUNT=10
+INTERVAL_SECS=600
+START_AT=""
+DRY_RUN=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --run-root) RUN_ROOT="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --rpki-client-libtls) RPKI_CLIENT_LIBTLS_PATH="$2"; shift 2 ;;
+    --round-count) ROUND_COUNT="$2"; shift 2 ;;
+    --interval-secs) INTERVAL_SECS="$2"; shift 2 ;;
+    --start-at) START_AT="$2"; shift 2 ;;
+    --dry-run) DRY_RUN=1; shift 1 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$RUN_ROOT" ]] || { usage >&2; exit 2; }
+[[ "$ROUND_COUNT" =~ ^[0-9]+$ ]] || { echo "--round-count must be an integer" >&2; exit 2; }
+[[ "$INTERVAL_SECS" =~ ^[0-9]+$ ]] || { echo "--interval-secs must be an integer" >&2; exit 2; }
+if [[ "$DRY_RUN" -ne 1 ]]; then
+  [[ -n "$REMOTE_ROOT" ]] || { echo "--remote-root is required unless --dry-run" >&2; exit 2; }
+  [[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
+  [[ -f "$RPKI_CLIENT_LIBTLS_PATH" ]] || { echo "rpki-client libtls not found: $RPKI_CLIENT_LIBTLS_PATH" >&2; exit 2; }
+fi
+
+mkdir -p "$RUN_ROOT"
+
+python3 - <<'PY' "$RUN_ROOT" "$SSH_TARGET" "$REMOTE_ROOT" "$ROUND_COUNT" "$INTERVAL_SECS" "$START_AT" "$DRY_RUN"
+import json
+import sys
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+run_root = Path(sys.argv[1]).resolve()
+ssh_target = sys.argv[2]
+remote_root = sys.argv[3]
+round_count = int(sys.argv[4])
+interval_secs = int(sys.argv[5])
+start_at_arg = sys.argv[6]
+dry_run = bool(int(sys.argv[7]))
+
+
+def parse_rfc3339_utc(value: str) -> datetime:
+    return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)
+
+
+def fmt(dt: datetime) -> str:
+    return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+base_time = (
+    parse_rfc3339_utc(start_at_arg)
+    if start_at_arg
+    else datetime.now(timezone.utc)
+)
+
+(run_root / "rounds").mkdir(parents=True, exist_ok=True)
+(run_root / "state" / "ours" / "work-db").mkdir(parents=True, exist_ok=True)
+(run_root / "state" / "ours" / "raw-store.db").mkdir(parents=True, exist_ok=True)
+(run_root / "state" / "ours" / "repo-bytes.db").mkdir(parents=True, exist_ok=True)
+(run_root / "state" / "rpki-client" / "cache").mkdir(parents=True, exist_ok=True)
+(run_root / "state" / "rpki-client" / "out").mkdir(parents=True, exist_ok=True)
+
+meta = {
+    "version": 1,
+    "rir": "arin",
+    "roundCount": round_count,
+    "intervalSecs": interval_secs,
+    "baseScheduledAt": fmt(base_time),
+    "mode": "dry_run" if dry_run else "skeleton_only",
+    "execution": {
+        "mode": "remote",
+        "sshTarget": ssh_target,
+        "remoteRoot": remote_root or None,
+    },
+    "state": {
+        "ours": {
+            "workDbPath": "state/ours/work-db",
+            "rawStoreDbPath": "state/ours/raw-store.db",
+            "repoBytesDbPath": "state/ours/repo-bytes.db",
+            "remoteWorkDbPath": "state/ours/work-db",
+            "remoteRawStoreDbPath": "state/ours/raw-store.db",
+            "remoteRepoBytesDbPath": "state/ours/repo-bytes.db",
+        },
+        "rpkiClient": {
+            "cachePath": "state/rpki-client/cache",
+            "outPath": "state/rpki-client/out",
+            "remoteCachePath": "state/rpki-client/cache",
+            "remoteOutPath": "state/rpki-client/out",
+        },
+    },
+}
+(run_root / "meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
+
+rounds = []
+for idx in range(round_count):
+    round_id = f"round-{idx+1:03d}"
+    kind = "snapshot" if idx == 0 else "delta"
+    scheduled_at = base_time + timedelta(seconds=interval_secs * idx)
+    round_dir = run_root / "rounds" / round_id
+    for name in ("ours", "rpki-client", "compare"):
+        (round_dir / name).mkdir(parents=True, exist_ok=True)
+
+    # M1 only builds the schedule skeleton, so lag is defined relative to the schedule model.
+    started_at = scheduled_at if dry_run else None
+    finished_at = scheduled_at if dry_run else None
+    round_meta = {
+        "roundId": round_id,
+        "kind": kind,
+        "scheduledAt": fmt(scheduled_at),
+        "startedAt": fmt(started_at) if started_at else None,
+        "finishedAt": fmt(finished_at) if finished_at else None,
+        "startLagMs": 0 if dry_run else None,
+        "status": "dry_run" if dry_run else "pending",
+        "paths": {
+            "ours": f"rounds/{round_id}/ours",
+            "rpkiClient": f"rounds/{round_id}/rpki-client",
+            "compare": f"rounds/{round_id}/compare",
+        },
+    }
+    (round_dir / "round-meta.json").write_text(
+        json.dumps(round_meta, indent=2), encoding="utf-8"
+    )
+    rounds.append(round_meta)
+
+final_summary = {
+    "version": 1,
+    "status": "dry_run" if dry_run else "pending",
+    "roundCount": round_count,
+    "allMatch": None,
+    "rounds": rounds,
+}
+(run_root / "final-summary.json").write_text(
+    json.dumps(final_summary, indent=2), encoding="utf-8"
+)
+PY
+
+if [[ "$DRY_RUN" -eq 1 ]]; then
+  echo "$RUN_ROOT"
+  exit 0
+fi
+
+if [[ ! -x "$ROOT_DIR/target/release/rpki" || ! -x "$ROOT_DIR/target/release/ccr_to_compare_views" ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin rpki --bin ccr_to_compare_views
+  )
+fi
+
+ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'"
+rsync -a --delete \
+  --exclude target \
+  --exclude .git \
+  "$ROOT_DIR/" "$SSH_TARGET:$REMOTE_ROOT/repo/"
+ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/repo/target/release' '$REMOTE_ROOT/bin' '$REMOTE_ROOT/lib' '$REMOTE_ROOT/rounds' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client'"
+rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_ROOT/repo/target/release/"
+rsync -a "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_ROOT/bin/rpki-client"
+rsync -aL "$RPKI_CLIENT_LIBTLS_PATH" "$SSH_TARGET:$REMOTE_ROOT/lib/libtls.so.28"
+
+for idx in $(seq 1 "$ROUND_COUNT"); do
+  ROUND_ID="$(printf 'round-%03d' "$idx")"
+  ROUND_DIR="$RUN_ROOT/rounds/$ROUND_ID"
+  SCHEDULED_AT="$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
+import json, sys
+print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['scheduledAt'])
+PY
+)"
+
+  python3 - <<'PY' "$SCHEDULED_AT"
+from datetime import datetime, timezone
+import sys, time
+scheduled = datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).astimezone(timezone.utc)
+now = datetime.now(timezone.utc)
+delay = (scheduled - now).total_seconds()
+if delay > 0:
+    time.sleep(delay)
+PY
+
+  "$ROOT_DIR/scripts/periodic/run_arin_ours_round_remote.sh" \
+    --run-root "$RUN_ROOT" \
+    --round-id "$ROUND_ID" \
+    --kind "$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
+import json, sys
+print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['kind'])
+PY
+)" \
+    --ssh-target "$SSH_TARGET" \
+    --remote-root "$REMOTE_ROOT" \
+    --scheduled-at "$SCHEDULED_AT" \
+    --skip-sync &
+  OURS_PID=$!
+
+  "$ROOT_DIR/scripts/periodic/run_arin_rpki_client_round_remote.sh" \
+    --run-root "$RUN_ROOT" \
+    --round-id "$ROUND_ID" \
+    --kind "$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
+import json, sys
+print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['kind'])
+PY
+)" \
+    --ssh-target "$SSH_TARGET" \
+    --remote-root "$REMOTE_ROOT" \
+    --scheduled-at "$SCHEDULED_AT" \
+    --rpki-client-bin "$RPKI_CLIENT_BIN" \
+    --skip-sync &
+  CLIENT_PID=$!
+
+  set +e
+  wait "$OURS_PID"
+  OURS_STATUS=$?
+  wait "$CLIENT_PID"
+  CLIENT_STATUS=$?
+  set -e
+
+  if [[ "$OURS_STATUS" -eq 0 && "$CLIENT_STATUS" -eq 0 \
+        && -f "$ROUND_DIR/ours/result.ccr" && -f "$ROUND_DIR/rpki-client/result.ccr" ]]; then
+    "$ROOT_DIR/scripts/periodic/compare_ccr_round.sh" \
+      --ours-ccr "$ROUND_DIR/ours/result.ccr" \
+      --rpki-client-ccr "$ROUND_DIR/rpki-client/result.ccr" \
+      --out-dir "$ROUND_DIR/compare"
+  fi
+
+  python3 - <<'PY' "$ROUND_DIR/round-meta.json" "$ROUND_DIR/ours/round-result.json" "$ROUND_DIR/rpki-client/round-result.json" "$ROUND_DIR/compare/compare-summary.json"
+import json, sys
+from datetime import datetime, timezone
+round_meta_path, ours_result_path, client_result_path, compare_path = sys.argv[1:]
+meta = json.load(open(round_meta_path, 'r', encoding='utf-8'))
+ours = json.load(open(ours_result_path, 'r', encoding='utf-8'))
+client = json.load(open(client_result_path, 'r', encoding='utf-8'))
+scheduled = datetime.fromisoformat(meta['scheduledAt'].replace('Z', '+00:00')).astimezone(timezone.utc)
+started_candidates = []
+for item in (ours, client):
+    if item.get('startedAt'):
+        started_candidates.append(datetime.fromisoformat(item['startedAt'].replace('Z', '+00:00')).astimezone(timezone.utc))
+finished_candidates = []
+for item in (ours, client):
+    if item.get('finishedAt'):
+        finished_candidates.append(datetime.fromisoformat(item['finishedAt'].replace('Z', '+00:00')).astimezone(timezone.utc))
+if started_candidates:
+    started_at = min(started_candidates)
+    meta['startedAt'] = started_at.strftime('%Y-%m-%dT%H:%M:%SZ')
+    lag_ms = int((started_at - scheduled).total_seconds() * 1000)
+    meta['startLagMs'] = max(lag_ms, 0)
+if finished_candidates:
+    finished_at = max(finished_candidates)
+    meta['finishedAt'] = finished_at.strftime('%Y-%m-%dT%H:%M:%SZ')
+meta['status'] = 'completed' if ours.get('exitCode') == 0 and client.get('exitCode') == 0 else 'failed'
+meta['ours'] = {
+    'exitCode': ours.get('exitCode'),
+    'durationMs': json.load(open(ours_result_path.replace('round-result.json', 'timing.json'), 'r', encoding='utf-8')).get('durationMs'),
+}
+meta['rpkiClient'] = {
+    'exitCode': client.get('exitCode'),
+    'durationMs': json.load(open(client_result_path.replace('round-result.json', 'timing.json'), 'r', encoding='utf-8')).get('durationMs'),
+}
+if compare_path and __import__('pathlib').Path(compare_path).exists():
+    compare = json.load(open(compare_path, 'r', encoding='utf-8'))
+    meta['compare'] = {
+        'allMatch': compare.get('allMatch'),
+        'vrpMatch': compare.get('vrps', {}).get('match'),
+        'vapMatch': compare.get('vaps', {}).get('match'),
+    }
+json.dump(meta, open(round_meta_path, 'w', encoding='utf-8'), indent=2)
+PY
+done
+
+python3 - <<'PY' "$RUN_ROOT/final-summary.json" "$RUN_ROOT/rounds"
+import json, sys
+from pathlib import Path
+summary_path = Path(sys.argv[1])
+rounds_root = Path(sys.argv[2])
+rounds = []
+all_match = True
+for round_dir in sorted(rounds_root.glob('round-*')):
+    meta = json.load(open(round_dir / 'round-meta.json', 'r', encoding='utf-8'))
+    rounds.append(meta)
+    compare = meta.get('compare')
+    if compare is None or compare.get('allMatch') is not True:
+        all_match = False
+summary = {
+    'version': 1,
+    'status': 'completed',
+    'roundCount': len(rounds),
+    'allMatch': all_match,
+    'rounds': rounds,
+}
+json.dump(summary, open(summary_path, 'w', encoding='utf-8'), indent=2)
+PY
+
+echo "$RUN_ROOT"

scripts/periodic/run_arin_ours_round_remote.sh

@@ -0,0 +1,158 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/periodic/run_arin_ours_round_remote.sh \
+    --run-root <path> \
+    --round-id <round-XXX> \
+    --kind <snapshot|delta> \
+    --ssh-target <user@host> \
+    --remote-root <path> \
+    [--scheduled-at <RFC3339>] \
+    [--skip-sync]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RUN_ROOT=""
+ROUND_ID=""
+KIND=""
+SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
+REMOTE_ROOT=""
+SCHEDULED_AT=""
+SKIP_SYNC=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --run-root) RUN_ROOT="$2"; shift 2 ;;
+    --round-id) ROUND_ID="$2"; shift 2 ;;
+    --kind) KIND="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --scheduled-at) SCHEDULED_AT="$2"; shift 2 ;;
+    --skip-sync) SKIP_SYNC=1; shift 1 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$RUN_ROOT" && -n "$ROUND_ID" && -n "$KIND" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
+[[ "$KIND" == "snapshot" || "$KIND" == "delta" ]] || { echo "--kind must be snapshot or delta" >&2; exit 2; }
+
+LOCAL_OUT="$RUN_ROOT/rounds/$ROUND_ID/ours"
+REMOTE_REPO="$REMOTE_ROOT/repo"
+REMOTE_OUT="$REMOTE_ROOT/rounds/$ROUND_ID/ours"
+REMOTE_WORK_DB="$REMOTE_ROOT/state/ours/work-db"
+REMOTE_RAW_STORE="$REMOTE_ROOT/state/ours/raw-store.db"
+REMOTE_REPO_BYTES="$REMOTE_ROOT/state/ours/repo-bytes.db"
+
+mkdir -p "$LOCAL_OUT"
+
+if [[ "$SKIP_SYNC" -eq 0 ]]; then
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'"
+  rsync -a --delete \
+    --exclude target \
+    --exclude .git \
+    "$ROOT_DIR/" "$SSH_TARGET:$REMOTE_REPO/"
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_REPO/target/release' '$REMOTE_OUT' '$REMOTE_ROOT/state/ours'"
+  rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_REPO/target/release/"
+else
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_OUT' '$REMOTE_ROOT/state/ours'"
+fi
+
+ssh "$SSH_TARGET" \
+  REMOTE_REPO="$REMOTE_REPO" \
+  REMOTE_OUT="$REMOTE_OUT" \
+  REMOTE_WORK_DB="$REMOTE_WORK_DB" \
+  REMOTE_RAW_STORE="$REMOTE_RAW_STORE" \
+  REMOTE_REPO_BYTES="$REMOTE_REPO_BYTES" \
+  KIND="$KIND" \
+  ROUND_ID="$ROUND_ID" \
+  SCHEDULED_AT="$SCHEDULED_AT" \
+  'bash -s' <<'EOS'
+set -euo pipefail
+
+cd "$REMOTE_REPO"
+mkdir -p "$REMOTE_OUT"
+
+if [[ "$KIND" == "snapshot" ]]; then
+  rm -rf "$REMOTE_WORK_DB" "$REMOTE_RAW_STORE" "$REMOTE_REPO_BYTES"
+fi
+mkdir -p "$(dirname "$REMOTE_WORK_DB")"
+
+started_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+started_at_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+
+ccr_out="$REMOTE_OUT/result.ccr"
+report_out="$REMOTE_OUT/report.json"
+run_log="$REMOTE_OUT/run.log"
+timing_out="$REMOTE_OUT/timing.json"
+meta_out="$REMOTE_OUT/round-result.json"
+
+set +e
+env RPKI_PROGRESS_LOG=1 target/release/rpki \
+  --db "$REMOTE_WORK_DB" \
+  --raw-store-db "$REMOTE_RAW_STORE" \
+  --repo-bytes-db "$REMOTE_REPO_BYTES" \
+  --tal-path tests/fixtures/tal/arin.tal \
+  --ta-path tests/fixtures/ta/arin-ta.cer \
+  --ccr-out "$ccr_out" \
+  --report-json "$report_out" \
+  >"$run_log" 2>&1
+exit_code=$?
+set -e
+
+finished_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+finished_at_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+
+python3 - <<'PY' "$timing_out" "$started_at_ms" "$finished_at_ms" "$started_at_iso" "$finished_at_iso" "$exit_code"
+import json, sys
+path, start_ms, end_ms, started_at, finished_at, exit_code = sys.argv[1:]
+with open(path, "w", encoding="utf-8") as fh:
+    json.dump(
+        {
+            "durationMs": int(end_ms) - int(start_ms),
+            "startedAt": started_at,
+            "finishedAt": finished_at,
+            "exitCode": int(exit_code),
+        },
+        fh,
+        indent=2,
+    )
+PY
+
+python3 - <<'PY' "$meta_out" "$ROUND_ID" "$KIND" "$SCHEDULED_AT" "$started_at_iso" "$finished_at_iso" "$REMOTE_WORK_DB" "$REMOTE_RAW_STORE" "$exit_code"
+import json, sys
+path, round_id, kind, scheduled_at, started_at, finished_at, work_db, raw_store, exit_code = sys.argv[1:]
+with open(path, "w", encoding="utf-8") as fh:
+    json.dump(
+        {
+            "roundId": round_id,
+            "kind": kind,
+            "scheduledAt": scheduled_at or None,
+            "startedAt": started_at,
+            "finishedAt": finished_at,
+            "remoteWorkDbPath": work_db,
+            "remoteRawStoreDbPath": raw_store,
+            "exitCode": int(exit_code),
+        },
+        fh,
+        indent=2,
+    )
+PY
+
+exit "$exit_code"
+EOS
+
+rsync -a "$SSH_TARGET:$REMOTE_OUT/" "$LOCAL_OUT/"
+echo "$LOCAL_OUT"

scripts/periodic/run_arin_rpki_client_round_remote.sh

@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/periodic/run_arin_rpki_client_round_remote.sh \
+    --run-root <path> \
+    --round-id <round-XXX> \
+    --kind <snapshot|delta> \
+    --ssh-target <user@host> \
+    --remote-root <path> \
+    [--scheduled-at <RFC3339>] \
+    [--rpki-client-bin <local path>] \
+    [--skip-sync]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RUN_ROOT=""
+ROUND_ID=""
+KIND=""
+SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
+REMOTE_ROOT=""
+SCHEDULED_AT=""
+RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}"
+SKIP_SYNC=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --run-root) RUN_ROOT="$2"; shift 2 ;;
+    --round-id) ROUND_ID="$2"; shift 2 ;;
+    --kind) KIND="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --scheduled-at) SCHEDULED_AT="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --skip-sync) SKIP_SYNC=1; shift 1 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$RUN_ROOT" && -n "$ROUND_ID" && -n "$KIND" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
+[[ "$KIND" == "snapshot" || "$KIND" == "delta" ]] || { echo "--kind must be snapshot or delta" >&2; exit 2; }
+[[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
+
+LOCAL_OUT="$RUN_ROOT/rounds/$ROUND_ID/rpki-client"
+REMOTE_REPO="$REMOTE_ROOT/repo"
+REMOTE_BIN_DIR="$REMOTE_ROOT/bin"
+REMOTE_BIN="$REMOTE_BIN_DIR/rpki-client"
+REMOTE_OUT="$REMOTE_ROOT/rounds/$ROUND_ID/rpki-client"
+REMOTE_CACHE="$REMOTE_ROOT/state/rpki-client/cache"
+REMOTE_STATE_OUT="$REMOTE_ROOT/state/rpki-client/out"
+REMOTE_STATE_ROOT="$REMOTE_ROOT/state/rpki-client"
+
+mkdir -p "$LOCAL_OUT"
+
+if [[ "$SKIP_SYNC" -eq 0 ]]; then
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'"
+  rsync -a --delete \
+    --exclude target \
+    --exclude .git \
+    "$ROOT_DIR/" "$SSH_TARGET:$REMOTE_REPO/"
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_BIN_DIR' '$REMOTE_OUT' '$REMOTE_STATE_ROOT'"
+  rsync -a "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_BIN"
+else
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_BIN_DIR' '$REMOTE_OUT' '$REMOTE_STATE_ROOT'"
+fi
+
+ssh "$SSH_TARGET" \
+  REMOTE_ROOT="$REMOTE_ROOT" \
+  REMOTE_REPO="$REMOTE_REPO" \
+  REMOTE_BIN="$REMOTE_BIN" \
+  REMOTE_OUT="$REMOTE_OUT" \
+  REMOTE_CACHE="$REMOTE_CACHE" \
+  REMOTE_STATE_OUT="$REMOTE_STATE_OUT" \
+  REMOTE_STATE_ROOT="$REMOTE_STATE_ROOT" \
+  KIND="$KIND" \
+  ROUND_ID="$ROUND_ID" \
+  SCHEDULED_AT="$SCHEDULED_AT" \
+  'bash -s' <<'EOS'
+set -euo pipefail
+
+cd "$REMOTE_ROOT"
+mkdir -p "$REMOTE_OUT"
+
+if [[ "$KIND" == "snapshot" ]]; then
+  rm -rf "$REMOTE_CACHE" "$REMOTE_STATE_OUT" "$REMOTE_STATE_ROOT/ta" "$REMOTE_STATE_ROOT/.ta"
+fi
+mkdir -p "$REMOTE_CACHE" "$REMOTE_STATE_OUT" "$REMOTE_STATE_ROOT/ta" "$REMOTE_STATE_ROOT/.ta"
+chmod -R 0777 "$REMOTE_STATE_ROOT"
+
+started_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+started_at_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+
+ccr_out="$REMOTE_OUT/result.ccr"
+run_log="$REMOTE_OUT/run.log"
+timing_out="$REMOTE_OUT/timing.json"
+meta_out="$REMOTE_OUT/round-result.json"
+
+set +e
+(
+  cd "$REMOTE_STATE_ROOT"
+  LD_LIBRARY_PATH="$REMOTE_ROOT/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \
+    "$REMOTE_BIN" -vv -t "../../repo/tests/fixtures/tal/arin.tal" -d "cache" "out"
+) >"$run_log" 2>&1
+exit_code=$?
+set -e
+
+if [[ -f "$REMOTE_STATE_OUT/rpki.ccr" ]]; then
+  cp "$REMOTE_STATE_OUT/rpki.ccr" "$ccr_out"
+fi
+
+finished_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+finished_at_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+
+python3 - <<'PY' "$timing_out" "$started_at_ms" "$finished_at_ms" "$started_at_iso" "$finished_at_iso" "$exit_code"
+import json, sys
+path, start_ms, end_ms, started_at, finished_at, exit_code = sys.argv[1:]
+with open(path, "w", encoding="utf-8") as fh:
+    json.dump(
+        {
+            "durationMs": int(end_ms) - int(start_ms),
+            "startedAt": started_at,
+            "finishedAt": finished_at,
+            "exitCode": int(exit_code),
+        },
+        fh,
+        indent=2,
+    )
+PY
+
+python3 - <<'PY' "$meta_out" "$ROUND_ID" "$KIND" "$SCHEDULED_AT" "$started_at_iso" "$finished_at_iso" "$REMOTE_CACHE" "$REMOTE_STATE_OUT" "$exit_code"
+import json, sys
+path, round_id, kind, scheduled_at, started_at, finished_at, cache_path, out_path, exit_code = sys.argv[1:]
+with open(path, "w", encoding="utf-8") as fh:
+    json.dump(
+        {
+            "roundId": round_id,
+            "kind": kind,
+            "scheduledAt": scheduled_at or None,
+            "startedAt": started_at,
+            "finishedAt": finished_at,
+            "remoteCachePath": cache_path,
+            "remoteOutPath": out_path,
+            "exitCode": int(exit_code),
+        },
+        fh,
+        indent=2,
+    )
+PY
+
+exit "$exit_code"
+EOS
+
+rsync -a "$SSH_TARGET:$REMOTE_OUT/" "$LOCAL_OUT/"
+echo "$LOCAL_OUT"

scripts/replay_verify/README.md

@@ -0,0 +1,45 @@
+# Replay Verify Scripts
+
+## `run_multi_rir_ccr_replay_verify.sh`
+
+用途：
+- 通用 multi-RIR CCR replay verify 入口
+- 通过 `--rir` 指定一个或多个 RIR，按顺序执行
+- 通过 `--mode` 指定 `snapshot`、`delta` 或 `both`
+- 默认每个 RIR 的 RocksDB 目录在 compare/verify 结束后自动删除；传 `--keep-db` 才保留
+- 同一次执行的所有产物都会先落到 `rpki/target/replay/<timestamp>/`
+- 该时间戳目录下再按 RIR 分目录：
+  - `<rir>_ccr_replay_<timestamp>`
+
+默认输入：
+- 历史 replay fixture root: `/home/yuyr/dev/rust_playground/routinator/bench/multi_rir_demo/runs/20260316-112341-multi-final3`
+- 每个 RIR 的 TAL / TA / validation time / record CSV 由 `scripts/payload_replay/multi_rir_case_info.py` 解析
+
+用法：
+- 单个 RIR：
+  - `./scripts/replay_verify/run_multi_rir_ccr_replay_verify.sh --rir apnic --mode both`
+  - `./scripts/replay_verify/run_multi_rir_ccr_replay_verify.sh --rir apnic --mode snapshot`
+  - `./scripts/replay_verify/run_multi_rir_ccr_replay_verify.sh --rir apnic --mode delta`
+- `./scripts/replay_verify/run_multi_rir_ccr_replay_verify.sh --rir apnic --mode both`
+- `./scripts/replay_verify/run_multi_rir_ccr_replay_verify.sh --rir apnic,ripe --mode snapshot`
+- `./scripts/replay_verify/run_multi_rir_ccr_replay_verify.sh --rir afrinic,apnic,arin,lacnic,ripe --mode both`
+- `./scripts/replay_verify/run_multi_rir_ccr_replay_verify.sh --rir apnic --mode delta --keep-db`
+
+可覆盖环境变量：
+- `BUNDLE_ROOT`
+- `OUT_ROOT`（默认：`rpki/target/replay`）
+- `RUN_TAG`
+
+主要产物：
+- 单次执行根目录：
+  - `rpki/target/replay/<timestamp>/`
+- 每个 RIR 子目录下：
+  - `<rir>_snapshot.ccr`
+  - `<rir>_delta.ccr`
+  - `<rir>_*_report.json`
+  - `<rir>_*_ccr_vrps.csv`
+  - `<rir>_*_ccr_compare_summary.md`
+  - `<rir>_*_ccr_verify.json`
+- 同次执行总汇总：
+  - `multi_rir_ccr_replay_verify_<timestamp>_summary.md`
+  - `multi_rir_ccr_replay_verify_<timestamp>_summary.json`

scripts/replay_verify/run_multi_rir_ccr_replay_verify.sh

@@ -0,0 +1,284 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  run_multi_rir_ccr_replay_verify.sh --rir <rir[,rir...]> [--mode snapshot|delta|both] [--keep-db]
+
+Options:
+  --rir <list>       Comma-separated RIR list, e.g. apnic or apnic,ripe
+  --mode <mode>      snapshot | delta | both (default: both)
+  --keep-db          Keep per-run RocksDB directories (default: remove after verify)
+  --bundle-root <p>  Override bundle root
+  --out-root <p>     Override output root (default: rpki/target/replay)
+  --run-tag <tag>    Override timestamp suffix for all RIR runs
+USAGE
+}
+
+MODE="both"
+KEEP_DB=0
+RIR_LIST=""
+BUNDLE_ROOT="${BUNDLE_ROOT:-/home/yuyr/dev/rust_playground/routinator/bench/multi_rir_demo/runs/20260316-112341-multi-final3}"
+OUT_ROOT="${OUT_ROOT:-$ROOT_DIR/target/replay}"
+RUN_TAG="${RUN_TAG:-$(date -u +%Y%m%dT%H%M%SZ)}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rir)
+      shift
+      RIR_LIST="${1:-}"
+      ;;
+    --mode)
+      shift
+      MODE="${1:-}"
+      ;;
+    --keep-db)
+      KEEP_DB=1
+      ;;
+    --bundle-root)
+      shift
+      BUNDLE_ROOT="${1:-}"
+      ;;
+    --out-root)
+      shift
+      OUT_ROOT="${1:-}"
+      ;;
+    --run-tag)
+      shift
+      RUN_TAG="${1:-}"
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "unknown argument: $1" >&2
+      usage >&2
+      exit 2
+      ;;
+  esac
+  shift || true
+done
+
+if [[ -z "$RIR_LIST" ]]; then
+  echo "--rir is required" >&2
+  usage >&2
+  exit 2
+fi
+case "$MODE" in
+  snapshot|delta|both) ;;
+  *)
+    echo "invalid --mode: $MODE" >&2
+    usage >&2
+    exit 2
+    ;;
+esac
+
+CASE_INFO_SCRIPT="$ROOT_DIR/scripts/payload_replay/multi_rir_case_info.py"
+mkdir -p "$OUT_ROOT"
+RUN_ROOT="$OUT_ROOT/$RUN_TAG"
+mkdir -p "$RUN_ROOT"
+
+cargo build --release --bin rpki --bin ccr_to_routinator_csv --bin ccr_verify >/dev/null
+
+summary_md="$RUN_ROOT/multi_rir_ccr_replay_verify_${RUN_TAG}_summary.md"
+summary_json="$RUN_ROOT/multi_rir_ccr_replay_verify_${RUN_TAG}_summary.json"
+
+python3 - <<'PY' >/dev/null
+PY
+
+summary_json_tmp="$(mktemp)"
+printf '[]' > "$summary_json_tmp"
+
+run_one_mode() {
+  local rir="$1"
+  local mode="$2"
+  local run_dir="$3"
+  local trust_anchor="$4"
+  local tal_path="$5"
+  local ta_path="$6"
+  local base_archive="$7"
+  local base_locks="$8"
+  local base_csv="$9"
+  local delta_archive="${10}"
+  local delta_locks="${11}"
+  local delta_csv="${12}"
+  local snapshot_validation_time="${13}"
+  local delta_validation_time="${14}"
+
+  local db_dir="$run_dir/${rir}_${mode}_db"
+  local report_json="$run_dir/${rir}_${mode}_report.json"
+  local run_log="$run_dir/${rir}_${mode}_run.log"
+  local ccr_path="$run_dir/${rir}_${mode}.ccr"
+  local csv_path="$run_dir/${rir}_${mode}_ccr_vrps.csv"
+  local compare_md="$run_dir/${rir}_${mode}_ccr_compare_summary.md"
+  local only_ours="$run_dir/${rir}_${mode}_ccr_only_in_ours.csv"
+  local only_record="$run_dir/${rir}_${mode}_ccr_only_in_record.csv"
+  local verify_json="$run_dir/${rir}_${mode}_ccr_verify.json"
+  local meta_json="$run_dir/${rir}_${mode}_meta.json"
+
+  rm -rf "$db_dir"
+
+  local -a cmd=(target/release/rpki --db "$db_dir" --tal-path "$tal_path" --ta-path "$ta_path")
+  if [[ "$mode" == "snapshot" ]]; then
+    cmd+=(--payload-replay-archive "$base_archive" --payload-replay-locks "$base_locks" --validation-time "$snapshot_validation_time")
+  else
+    cmd+=(
+      --payload-base-archive "$base_archive"
+      --payload-base-locks "$base_locks"
+      --payload-base-validation-time "$snapshot_validation_time"
+      --payload-delta-archive "$delta_archive"
+      --payload-delta-locks "$delta_locks"
+      --validation-time "$delta_validation_time"
+    )
+  fi
+  cmd+=(--report-json "$report_json" --ccr-out "$ccr_path")
+
+  local start_s end_s duration_s
+  start_s="$(date +%s)"
+  (
+    echo "# ${rir} ${mode} command:"
+    printf '%q ' "${cmd[@]}"
+    echo
+    echo
+    "${cmd[@]}"
+  ) 2>&1 | tee "$run_log" >/dev/null
+  end_s="$(date +%s)"
+  duration_s="$((end_s - start_s))"
+
+  target/release/ccr_to_routinator_csv \
+    --ccr "$ccr_path" \
+    --out "$csv_path" \
+    --trust-anchor "$trust_anchor" >/dev/null
+
+  local record_csv
+  if [[ "$mode" == "snapshot" ]]; then
+    record_csv="$base_csv"
+  else
+    record_csv="$delta_csv"
+  fi
+  ./scripts/payload_replay/compare_with_routinator_record.sh \
+    "$csv_path" \
+    "$record_csv" \
+    "$compare_md" \
+    "$only_ours" \
+    "$only_record" >/dev/null
+
+  target/release/ccr_verify \
+    --ccr "$ccr_path" \
+    --db "$db_dir" > "$verify_json"
+
+  python3 - "$report_json" "$meta_json" "$mode" "$duration_s" <<'PY'
+import json, sys
+from pathlib import Path
+report = json.loads(Path(sys.argv[1]).read_text(encoding='utf-8'))
+meta = {
+  'mode': sys.argv[3],
+  'duration_seconds': int(sys.argv[4]),
+  'validation_time': report.get('validation_time_rfc3339_utc'),
+  'publication_points_processed': report['tree']['instances_processed'],
+  'publication_points_failed': report['tree']['instances_failed'],
+  'vrps': len(report['vrps']),
+  'aspas': len(report['aspas']),
+}
+Path(sys.argv[2]).write_text(json.dumps(meta, ensure_ascii=False, indent=2)+'\n', encoding='utf-8')
+PY
+
+  if [[ "$KEEP_DB" -eq 0 ]]; then
+    rm -rf "$db_dir"
+  fi
+}
+
+IFS=',' read -r -a RIRS <<< "$RIR_LIST"
+for rir in "${RIRS[@]}"; do
+  rir="$(echo "$rir" | xargs)"
+  [[ -n "$rir" ]] || continue
+  eval "$(python3 "$CASE_INFO_SCRIPT" --bundle-root "$BUNDLE_ROOT" --rir "$rir" --format env)"
+
+  run_dir="$RUN_ROOT/${rir}_ccr_replay_${RUN_TAG}"
+  mkdir -p "$run_dir"
+
+  if [[ "$MODE" == "snapshot" || "$MODE" == "both" ]]; then
+    run_one_mode \
+      "$rir" snapshot "$run_dir" "$TRUST_ANCHOR" "$TAL_PATH" "$TA_PATH" \
+      "$PAYLOAD_REPLAY_ARCHIVE" "$PAYLOAD_REPLAY_LOCKS" "$ROUTINATOR_BASE_RECORD_CSV" \
+      "$PAYLOAD_DELTA_ARCHIVE" "$PAYLOAD_DELTA_LOCKS" "$ROUTINATOR_DELTA_RECORD_CSV" \
+      "$SNAPSHOT_VALIDATION_TIME" "$DELTA_VALIDATION_TIME"
+  fi
+  if [[ "$MODE" == "delta" || "$MODE" == "both" ]]; then
+    run_one_mode \
+      "$rir" delta "$run_dir" "$TRUST_ANCHOR" "$TAL_PATH" "$TA_PATH" \
+      "$PAYLOAD_REPLAY_ARCHIVE" "$PAYLOAD_REPLAY_LOCKS" "$ROUTINATOR_BASE_RECORD_CSV" \
+      "$PAYLOAD_DELTA_ARCHIVE" "$PAYLOAD_DELTA_LOCKS" "$ROUTINATOR_DELTA_RECORD_CSV" \
+      "$SNAPSHOT_VALIDATION_TIME" "$DELTA_VALIDATION_TIME"
+  fi
+
+  python3 - "$summary_json_tmp" "$run_dir" "$rir" "$MODE" <<'PY'
+import json, sys
+from pathlib import Path
+summary_path = Path(sys.argv[1])
+run_dir = Path(sys.argv[2])
+rir = sys.argv[3]
+mode = sys.argv[4]
+rows = json.loads(summary_path.read_text(encoding='utf-8'))
+for submode in ['snapshot','delta']:
+    if mode not in ('both', submode):
+        continue
+    compare = run_dir / f'{rir}_{submode}_ccr_compare_summary.md'
+    meta = run_dir / f'{rir}_{submode}_meta.json'
+    verify = run_dir / f'{rir}_{submode}_ccr_verify.json'
+    if not compare.exists() or not meta.exists() or not verify.exists():
+        continue
+    compare_text = compare.read_text(encoding='utf-8')
+    meta_obj = json.loads(meta.read_text(encoding='utf-8'))
+    verify_obj = json.loads(verify.read_text(encoding='utf-8'))
+    def metric(name):
+        prefix = f'| {name} | '
+        for line in compare_text.splitlines():
+            if line.startswith(prefix):
+                return int(line.split('|')[2].strip())
+        raise SystemExit(f'missing metric {name} in {compare}')
+    rows.append({
+        'rir': rir,
+        'mode': submode,
+        'run_dir': str(run_dir),
+        'duration_seconds': meta_obj['duration_seconds'],
+        'vrps': meta_obj['vrps'],
+        'aspas': meta_obj['aspas'],
+        'only_in_ours': metric('only_in_ours'),
+        'only_in_record': metric('only_in_record'),
+        'intersection': metric('intersection'),
+        'state_hashes_ok': verify_obj.get('state_hashes_ok'),
+    })
+summary_path.write_text(json.dumps(rows, ensure_ascii=False, indent=2)+'\n', encoding='utf-8')
+PY
+
+done
+
+python3 - "$summary_json_tmp" "$summary_json" "$summary_md" "$RUN_TAG" <<'PY'
+import json, sys
+from pathlib import Path
+rows = json.loads(Path(sys.argv[1]).read_text(encoding='utf-8'))
+out_json = Path(sys.argv[2])
+out_md = Path(sys.argv[3])
+run_tag = sys.argv[4]
+out_json.write_text(json.dumps(rows, ensure_ascii=False, indent=2)+'\n', encoding='utf-8')
+parts = []
+parts.append('# Multi-RIR CCR Replay Verify Summary\n\n')
+parts.append(f'- run_tag: `{run_tag}`\n\n')
+parts.append('| rir | mode | duration_s | vrps | aspas | only_in_ours | only_in_record | state_hashes_ok |\n')
+parts.append('|---|---|---:|---:|---:|---:|---:|---|\n')
+for row in rows:
+    parts.append(f"| {row['rir']} | {row['mode']} | {row['duration_seconds']} | {row['vrps']} | {row['aspas']} | {row['only_in_ours']} | {row['only_in_record']} | {row['state_hashes_ok']} |\n")
+out_md.write_text(''.join(parts), encoding='utf-8')
+PY
+
+rm -f "$summary_json_tmp"
+
+echo "== multi-rir replay verify complete ==" >&2
+echo "- summary: $summary_md" >&2
+echo "- summary json: $summary_json" >&2